[onionoo/master] Add unit tests for new GeoIP2 code, and fix a bug.

commit 0b09a6dd60b8d9d6465e9004274b4ec468db4346 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Feb 24 11:54:07 2014 +0100 Add unit tests for new GeoIP2 code, and fix a bug. --- src/org/torproject/onionoo/LookupService.java | 10 + test/org/torproject/onionoo/LookupServiceTest.java | 343 +++++++------------- 2 files changed, 128 insertions(+), 225 deletions(-) diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java index 928a550..3cfe3f2 100644 --- a/src/org/torproject/onionoo/LookupService.java +++ b/src/org/torproject/onionoo/LookupService.java @@ -125,6 +125,13 @@ public class LookupService { try { String startAddressString = parts[0].substring(7); /* ::ffff: */ long startIpNum = this.parseAddressString(startAddressString); + if (startIpNum < 0L) { + System.err.println("Illegal IP address in '" + line + + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() + + "."); + br.close(); + return lookupResults; + } int networkMaskLength = Integer.parseInt(parts[1]); if (networkMaskLength < 96 || networkMaskLength > 128) { System.err.println("Illegal network mask in '" + line @@ -133,6 +140,9 @@ public class LookupService { br.close(); return lookupResults; } + if (parts[2].length() == 0 && parts[3].length() == 0) { + continue; + } long endIpNum = startIpNum + (1 << (128 - networkMaskLength)) - 1; for (long addressNumber : sortedAddressNumbers. diff --git a/test/org/torproject/onionoo/LookupServiceTest.java b/test/org/torproject/onionoo/LookupServiceTest.java index 4834092..ae220f8 100644 --- a/test/org/torproject/onionoo/LookupServiceTest.java +++ b/test/org/torproject/onionoo/LookupServiceTest.java @@ -26,10 +26,8 @@ import org.torproject.onionoo.LookupService.LookupResult; public class LookupServiceTest { - private List<String> manualGeoLiteCityBlocksLines, - automaticGeoLiteCityBlocksLines, geoLiteCityBlocksLines, - geoLiteCityLocationLines, iso3166Lines, regionLines, - geoipASNum2Lines; + private List<String> geoLite2CityBlocksLines, + geoLite2CityLocationsLines, geoipASNum2Lines; private LookupService lookupService; @@ -38,29 +36,27 @@ public class LookupServiceTest { private SortedMap<String, LookupResult> lookupResults; private void populateLines() { - this.manualGeoLiteCityBlocksLines = new ArrayList<String>(); - this.manualGeoLiteCityBlocksLines.add( - "Copyright (c) 2011 MaxMind Inc. All Rights Reserved."); - this.manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - this.manualGeoLiteCityBlocksLines.add("\"134739200\",\"134744063\"," - + "\"223\""); - this.manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\"," - + "\"32191\""); - this.manualGeoLiteCityBlocksLines.add("\"134744320\",\"134751743\"," - + "\"223\""); - this.geoLiteCityLocationLines = new ArrayList<String>(); - this.geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind " - + "LLC. All Rights Reserved."); - this.geoLiteCityLocationLines.add("locId,country,region,city," - + "postalCode,latitude,longitude,metroCode,areaCode"); - this.geoLiteCityLocationLines.add("223,\"US\",\"\",\"\",\"\"," - + "38.0000,-97.0000,,"); - this.geoLiteCityLocationLines.add("32191,\"US\",\"CA\"," - + "\"Mountain View\",\"\",37.3860,-122.0838,807,650"); - this.iso3166Lines = new ArrayList<String>(); - this.iso3166Lines.add("US,\"United States\""); - this.regionLines = new ArrayList<String>(); - this.regionLines.add("US,CA,\"California\""); + this.geoLite2CityBlocksLines = new ArrayList<String>(); + this.geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.9.0,120,6252001,6252001," + + ",,38.0000,-97.0000,0,0"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,5375480,6252001," + + ",94043,37.3860,-122.0838,0,0"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.7.0,120,6252001,6252001," + + ",,38.0000,-97.0000,0,0"); + this.geoLite2CityLocationsLines = new ArrayList<String>(); + this.geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + this.geoLite2CityLocationsLines.add("6252001,NA,\"North America\",US," + + "\"United States\",,,,,"); + this.geoLite2CityLocationsLines.add("5375480,NA,\"North America\",US," + + "\"United States\",CA,California,\"Mountain View\",807," + + "America/Los_Angeles"); this.geoipASNum2Lines = new ArrayList<String>(); this.geoipASNum2Lines.add("134743296,134744063,\"AS3356 Level 3 " + "Communications\""); @@ -72,16 +68,10 @@ public class LookupServiceTest { private void writeCsvFiles() { try { - this.writeCsvFile(this.manualGeoLiteCityBlocksLines, - "Manual-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.automaticGeoLiteCityBlocksLines, - "Automatic-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityBlocksLines, - "GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityLocationLines, - "GeoLiteCity-Location.csv"); - this.writeCsvFile(this.iso3166Lines, "iso3166.csv"); - this.writeCsvFile(this.regionLines, "region.csv"); + this.writeCsvFile(this.geoLite2CityBlocksLines, + "GeoLite2-City-Blocks.csv"); + this.writeCsvFile(this.geoLite2CityLocationsLines, + "GeoLite2-City-Locations.csv"); this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); } catch (IOException e) { throw new RuntimeException(e); @@ -105,36 +95,19 @@ public class LookupServiceTest { this.lookupResults = this.lookupService.lookup(this.addressStrings); } - private void assertLookupResult( - List<String> manualGeoLiteCityBlocksLines, - List<String> automaticGeoLiteCityBlocksLines, - List<String> geoLiteCityBlocksLines, - List<String> geoLiteCityLocationLines, List<String> iso3166Lines, - List<String> regionLines, List<String> geoipASNum2Lines, - String addressString, String countryCode, String countryName, - String regionName, String cityName, String latitude, - String longitude, String aSNumber, String aSName) { + private void assertLookupResult(List<String> geoLite2CityBlocksLines, + List<String> geoLite2CityLocationsLines, + List<String> geoipASNum2Lines, String addressString, + String countryCode, String countryName, String regionName, + String cityName, String latitude, String longitude, String aSNumber, + String aSName) { this.addressStrings.add(addressString); this.populateLines(); - if (manualGeoLiteCityBlocksLines != null) { - this.manualGeoLiteCityBlocksLines = - manualGeoLiteCityBlocksLines; + if (geoLite2CityBlocksLines != null) { + this.geoLite2CityBlocksLines = geoLite2CityBlocksLines; } - if (automaticGeoLiteCityBlocksLines != null) { - this.automaticGeoLiteCityBlocksLines = - automaticGeoLiteCityBlocksLines; - } - if (geoLiteCityBlocksLines != null) { - this.geoLiteCityBlocksLines = geoLiteCityBlocksLines; - } - if (geoLiteCityLocationLines != null) { - this.geoLiteCityLocationLines = geoLiteCityLocationLines; - } - if (iso3166Lines != null) { - this.iso3166Lines = iso3166Lines; - } - if (regionLines != null) { - this.regionLines = regionLines; + if (geoLite2CityLocationsLines != null) { + this.geoLite2CityLocationsLines = geoLite2CityLocationsLines; } if (geoipASNum2Lines != null) { this.geoipASNum2Lines = geoipASNum2Lines; @@ -216,230 +189,154 @@ public class LookupServiceTest { @Test() public void testLookup8888() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.8", "us", + this.assertLookupResult(null, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); } @Test() public void testLookup8880() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.0", "us", + this.assertLookupResult(null, null, null, "8.8.8.0", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); } @Test() public void testLookup888255() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.255", "us", + this.assertLookupResult(null, null, null, "8.8.8.255", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); } @Test() public void testLookup888256() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.256", null, null, null, - null, null, null, null, null); + this.assertLookupResult(null, null, null, "8.8.8.256", null, null, + null, null, null, null, null, null); } @Test() public void testLookup888Minus1() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.-1", null, null, null, - null, null, null, null, null); + this.assertLookupResult(null, null, null, "8.8.8.-1", null, null, + null, null, null, null, null, null); } @Test() public void testLookup000() { - this.assertLookupResult(null, - null, null, null, null, null, null, "0.0.0.0", null, null, null, + this.assertLookupResult(null, null, null, "0.0.0.0", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoBlocksLines() { - this.assertLookupResult( - new ArrayList<String>(), null, null, null, null, null, null, + this.assertLookupResult(new ArrayList<String>(), null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoLocationLines() { - this.assertLookupResult(null, - null, null, new ArrayList<String>(), null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupNoIso3166Lines() { - this.assertLookupResult(null, - null, null, null, new ArrayList<String>(), null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupNoRegionLines() { - this.assertLookupResult(null, - null, null, null, null, new ArrayList<String>(), null, "8.8.8.8", - null, null, null, null, null, null, null, null); + this.assertLookupResult(null, new ArrayList<String>(), null, + "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoGeoipASNum2Lines() { - this.assertLookupResult(null, - null, null, null, null, null, new ArrayList<String>(), "8.8.8.8", - null, null, null, null, null, null, null, null); + this.assertLookupResult(null, null, new ArrayList<String>(), + "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoCorrespondingLocation() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("223,\"US\",\"\",\"\",\"\",38.0000," - + "-97.0000,,"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, "AS15169", "Google Inc."); - } - - @Test() - public void testLookupNoCorrespondingCountryName() { - List<String> iso3166Lines = new ArrayList<String>(); - iso3166Lines.add("UY,\"Uruguay\""); - this.assertLookupResult(null, - null, null, null, iso3166Lines, null, null, "8.8.8.8", "us", - null, "California", "Mountain View", "37.3860", "-122.0838", - "AS15169", "Google Inc."); - } - - @Test() - public void testLookupNoCorrespondingRegionName() { - List<String> regionLines = new ArrayList<String>(); - regionLines.add("US,CO,\"Colorado\""); - this.assertLookupResult(null, - null, null, null, null, regionLines, null, "8.8.8.8", "us", - "United States", null, "Mountain View", "37.3860", "-122.0838", + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("6252001,NA,\"North America\",US," + + "\"United States\",,,,,"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, "37.3860", "-122.0838", "AS15169", "Google Inc."); } @Test() - public void testLookupBlocksEndBeforeStart() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add("\"134739200\",\"134744063\"," - + "\"223\""); - manualGeoLiteCityBlocksLines.add("\"134744319\",\"134744064\"," - + "\"32191\""); - manualGeoLiteCityBlocksLines.add("\"134744320\",\"134751743\"," - + "\"223\""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, - "8.8.8.8", null, null, null, null, null, null, "AS15169", - "Google Inc."); - } - - @Test() public void testLookupBlocksStartNotANumber() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add("\"one\",\"134744319\"," - + "\"32191\""); + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:one,120,5375480,6252001,,94043," + + "37.3860,-122.0838,0,0"); this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() - public void testLookupBlocksStartTooLarge() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add("\"1" - + String.valueOf(Long.MAX_VALUE) + "\",\"134744319\",\"32191\""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + public void testLookupBlocksLocationX() { + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,X,X,,94043,37.3860," + + "-122.0838,0,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() - public void testLookupBlocksLocationX() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\",\"X\""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, - "8.8.8.8", null, null, null, null, null, null, null, null); + public void testLookupBlocksLocationEmpty() { + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,,,,,,,1,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, + "8.8.8.8", null, null, null, null, null, null, "AS15169", + "Google Inc."); } @Test() public void testLookupBlocksTooFewFields() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,5375480,6252001," + + ",94043,37.3860,-122.0838,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupLocationLocIdNotANumber() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("threetwoonenineone,\"US\",\"CA\"," - + "\"Mountain View\",\"\",37.3860,-122.0838,807,650"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("threetwoonenineone,NA," + + "\"North America\",US,\"United States\",CA,California," + + "\"Mountain View\",807,America/Los_Angeles"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupLocationTooFewFields() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("32191,\"US\",\"CA\",\"Mountain View\"," - + "\"\",37.3860,-122.0838,807"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupIso3166TooFewFields() { - List<String> iso3166Lines = new ArrayList<String>(); - iso3166Lines.add("US"); - this.assertLookupResult(null, - null, null, null, iso3166Lines, null, null, "8.8.8.8", null, null, - null, null, null, null, null, null); - } - - @Test() - public void testLookupRegionTooFewFields() { - List<String> regionLines = new ArrayList<String>(); - regionLines.add("US,CA"); - this.assertLookupResult(null, - null, null, null, null, regionLines, null, "8.8.8.8", null, null, - null, null, null, null, null, null); + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("5375480,NA,\"North America\",US," + + "\"United States\",CA,California,\"Mountain View\",807"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() @@ -450,8 +347,7 @@ public class LookupServiceTest { geoipASNum2Lines.add("134744319,134744064,\"AS15169 Google Inc.\""); geoipASNum2Lines.add("134744320,134750463,\"AS3356 Level 3 " + "Communications\""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", "us", + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", null, null); } @@ -460,8 +356,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2StartNotANumber() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("one,134744319,\"AS15169 Google Inc.\""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -470,8 +365,7 @@ public class LookupServiceTest { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("1" + String.valueOf(Long.MAX_VALUE) + ",134744319,\"AS15169 Google Inc.\""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -479,8 +373,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2TooFewFields() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("134744064,134744319"); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } }
participants (1)
-
karsten@torproject.org