commit 1301bcbd62663a5237c68a7126afb9812b4b410f Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue May 24 19:18:43 2016 +0200

Support additional columns in GeoLite2 files.

Looks like MaxMind added a tenth column "accuracy_radius" in their May 2016 update. We should just ignore that column and any other columns added in the future.

Fixes #19154. --- .../torproject/onionoo/updater/LookupService.java | 6 ++--- .../onionoo/updater/LookupServiceTest.java | 28 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index 6da3608..e800926 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -122,8 +122,8 @@ public class LookupService { addressStringNumbers.values()); String line = br.readLine(); while ((line = br.readLine()) != null) { - String[] parts = line.split(",", 9); - if (parts.length != 9) { + String[] parts = line.split(",", -1); + if (parts.length < 9) { log.error("Illegal line '" + line + "' in " + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() + "."); @@ -169,7 +169,7 @@ public class LookupService { log.error("Number format exception while parsing line '" + line + "' in " + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() - + "."); + + ".", e); return lookupResults; } } diff --git a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java index 2b90a37..7ba3462 100644 --- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java +++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java @@ -320,6 +320,34 @@ public class LookupServiceTest { }

@Test() + public void testLookupBlocksExtraneousField() { + List<String> geoLite2CityBlocksIPv4Lines = new ArrayList<String>(); + geoLite2CityBlocksIPv4Lines.add("network,geoname_id," + + "registered_country_geoname_id,represented_country_geoname_id," + + "is_anonymous_proxy,is_satellite_provider,postal_code,latitude," + + "longitude,accuracy_radius"); + geoLite2CityBlocksIPv4Lines.add("8.8.8.0/24,5375480,6252001,,0," + + "0,94035,37.3860,-122.0838,937"); + this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null, + "8.8.8.8", "us", "United States", "California", "Mountain View", + 37.3860f, -122.0838f, "AS15169", "Google Inc."); + } + + @Test() + public void testLookupBlocksThreeExtraneousFields() { + List<String> geoLite2CityBlocksIPv4Lines = new ArrayList<String>(); + geoLite2CityBlocksIPv4Lines.add("network,geoname_id," + + "registered_country_geoname_id,represented_country_geoname_id," + + "is_anonymous_proxy,is_satellite_provider,postal_code,latitude," + + "longitude,accuracy_radius,more,even_more,wow_so_many_fields"); + geoLite2CityBlocksIPv4Lines.add("8.8.8.0/24,5375480,6252001,,0," + + "0,94035,37.3860,-122.0838,937,1,2,30000000000000"); + this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null, + "8.8.8.8", "us", "United States", "California", "Mountain View", + 37.3860f, -122.0838f, "AS15169", "Google Inc."); + } + + @Test() public void testLookupLocationLocIdNotANumber() { List<String> geoLite2CityLocationsEnLines = new ArrayList<String>(); geoLite2CityLocationsEnLines.add("geoname_id,locale_code,"