[tor-commits] [onionoo/master] Support additional columns in GeoLite2 files.

[karsten at torproject.org]

commit 1301bcbd62663a5237c68a7126afb9812b4b410f
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Tue May 24 19:18:43 2016 +0200

    Support additional columns in GeoLite2 files.
    
    Looks like MaxMind added a tenth column "accuracy_radius" in their May
    2016 update.  We should just ignore that column and any other columns
    added in the future.
    
    Fixes #19154.
---
 .../torproject/onionoo/updater/LookupService.java  |  6 ++---
 .../onionoo/updater/LookupServiceTest.java         | 28 ++++++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java
index 6da3608..e800926 100644
--- a/src/main/java/org/torproject/onionoo/updater/LookupService.java
+++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java
@@ -122,8 +122,8 @@ public class LookupService {
           addressStringNumbers.values());
       String line = br.readLine();
       while ((line = br.readLine()) != null) {
-        String[] parts = line.split(",", 9);
-        if (parts.length != 9) {
+        String[] parts = line.split(",", -1);
+        if (parts.length < 9) {
           log.error("Illegal line '" + line + "' in "
               + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath()
               + ".");
@@ -169,7 +169,7 @@ public class LookupService {
           log.error("Number format exception while parsing line '" + line
               + "' in "
               + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath()
-              + ".");
+              + ".", e);
           return lookupResults;
         }
       }
diff --git a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
index 2b90a37..7ba3462 100644
--- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
+++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java
@@ -320,6 +320,34 @@ public class LookupServiceTest {
   }
 
   @Test()
+  public void testLookupBlocksExtraneousField() {
+    List<String> geoLite2CityBlocksIPv4Lines = new ArrayList<String>();
+    geoLite2CityBlocksIPv4Lines.add("network,geoname_id,"
+        + "registered_country_geoname_id,represented_country_geoname_id,"
+        + "is_anonymous_proxy,is_satellite_provider,postal_code,latitude,"
+        + "longitude,accuracy_radius");
+    geoLite2CityBlocksIPv4Lines.add("8.8.8.0/24,5375480,6252001,,0,"
+        + "0,94035,37.3860,-122.0838,937");
+    this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null,
+        "8.8.8.8", "us", "United States", "California", "Mountain View",
+        37.3860f, -122.0838f, "AS15169", "Google Inc.");
+  }
+
+  @Test()
+  public void testLookupBlocksThreeExtraneousFields() {
+    List<String> geoLite2CityBlocksIPv4Lines = new ArrayList<String>();
+    geoLite2CityBlocksIPv4Lines.add("network,geoname_id,"
+        + "registered_country_geoname_id,represented_country_geoname_id,"
+        + "is_anonymous_proxy,is_satellite_provider,postal_code,latitude,"
+        + "longitude,accuracy_radius,more,even_more,wow_so_many_fields");
+    geoLite2CityBlocksIPv4Lines.add("8.8.8.0/24,5375480,6252001,,0,"
+        + "0,94035,37.3860,-122.0838,937,1,2,30000000000000");
+    this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null,
+        "8.8.8.8", "us", "United States", "California", "Mountain View",
+        37.3860f, -122.0838f, "AS15169", "Google Inc.");
+  }
+
+  @Test()
   public void testLookupLocationLocIdNotANumber() {
     List<String> geoLite2CityLocationsEnLines = new ArrayList<String>();
     geoLite2CityLocationsEnLines.add("geoname_id,locale_code,"