commit 0b09a6dd60b8d9d6465e9004274b4ec468db4346 Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Feb 24 11:54:07 2014 +0100
Add unit tests for new GeoIP2 code, and fix a bug. --- src/org/torproject/onionoo/LookupService.java | 10 + test/org/torproject/onionoo/LookupServiceTest.java | 343 +++++++------------- 2 files changed, 128 insertions(+), 225 deletions(-)
diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java index 928a550..3cfe3f2 100644 --- a/src/org/torproject/onionoo/LookupService.java +++ b/src/org/torproject/onionoo/LookupService.java @@ -125,6 +125,13 @@ public class LookupService { try { String startAddressString = parts[0].substring(7); /* ::ffff: */ long startIpNum = this.parseAddressString(startAddressString); + if (startIpNum < 0L) { + System.err.println("Illegal IP address in '" + line + + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() + + "."); + br.close(); + return lookupResults; + } int networkMaskLength = Integer.parseInt(parts[1]); if (networkMaskLength < 96 || networkMaskLength > 128) { System.err.println("Illegal network mask in '" + line @@ -133,6 +140,9 @@ public class LookupService { br.close(); return lookupResults; } + if (parts[2].length() == 0 && parts[3].length() == 0) { + continue; + } long endIpNum = startIpNum + (1 << (128 - networkMaskLength)) - 1; for (long addressNumber : sortedAddressNumbers. diff --git a/test/org/torproject/onionoo/LookupServiceTest.java b/test/org/torproject/onionoo/LookupServiceTest.java index 4834092..ae220f8 100644 --- a/test/org/torproject/onionoo/LookupServiceTest.java +++ b/test/org/torproject/onionoo/LookupServiceTest.java @@ -26,10 +26,8 @@ import org.torproject.onionoo.LookupService.LookupResult;
public class LookupServiceTest {
- private List<String> manualGeoLiteCityBlocksLines, - automaticGeoLiteCityBlocksLines, geoLiteCityBlocksLines, - geoLiteCityLocationLines, iso3166Lines, regionLines, - geoipASNum2Lines; + private List<String> geoLite2CityBlocksLines, + geoLite2CityLocationsLines, geoipASNum2Lines;
private LookupService lookupService;
@@ -38,29 +36,27 @@ public class LookupServiceTest { private SortedMap<String, LookupResult> lookupResults;
private void populateLines() { - this.manualGeoLiteCityBlocksLines = new ArrayList<String>(); - this.manualGeoLiteCityBlocksLines.add( - "Copyright (c) 2011 MaxMind Inc. All Rights Reserved."); - this.manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - this.manualGeoLiteCityBlocksLines.add(""134739200","134744063"," - + ""223""); - this.manualGeoLiteCityBlocksLines.add(""134744064","134744319"," - + ""32191""); - this.manualGeoLiteCityBlocksLines.add(""134744320","134751743"," - + ""223""); - this.geoLiteCityLocationLines = new ArrayList<String>(); - this.geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind " - + "LLC. All Rights Reserved."); - this.geoLiteCityLocationLines.add("locId,country,region,city," - + "postalCode,latitude,longitude,metroCode,areaCode"); - this.geoLiteCityLocationLines.add("223,"US","","",""," - + "38.0000,-97.0000,,"); - this.geoLiteCityLocationLines.add("32191,"US","CA"," - + ""Mountain View","",37.3860,-122.0838,807,650"); - this.iso3166Lines = new ArrayList<String>(); - this.iso3166Lines.add("US,"United States""); - this.regionLines = new ArrayList<String>(); - this.regionLines.add("US,CA,"California""); + this.geoLite2CityBlocksLines = new ArrayList<String>(); + this.geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.9.0,120,6252001,6252001," + + ",,38.0000,-97.0000,0,0"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,5375480,6252001," + + ",94043,37.3860,-122.0838,0,0"); + this.geoLite2CityBlocksLines.add("::ffff:8.8.7.0,120,6252001,6252001," + + ",,38.0000,-97.0000,0,0"); + this.geoLite2CityLocationsLines = new ArrayList<String>(); + this.geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + this.geoLite2CityLocationsLines.add("6252001,NA,"North America",US," + + ""United States",,,,,"); + this.geoLite2CityLocationsLines.add("5375480,NA,"North America",US," + + ""United States",CA,California,"Mountain View",807," + + "America/Los_Angeles"); this.geoipASNum2Lines = new ArrayList<String>(); this.geoipASNum2Lines.add("134743296,134744063,"AS3356 Level 3 " + "Communications""); @@ -72,16 +68,10 @@ public class LookupServiceTest {
private void writeCsvFiles() { try { - this.writeCsvFile(this.manualGeoLiteCityBlocksLines, - "Manual-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.automaticGeoLiteCityBlocksLines, - "Automatic-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityBlocksLines, - "GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityLocationLines, - "GeoLiteCity-Location.csv"); - this.writeCsvFile(this.iso3166Lines, "iso3166.csv"); - this.writeCsvFile(this.regionLines, "region.csv"); + this.writeCsvFile(this.geoLite2CityBlocksLines, + "GeoLite2-City-Blocks.csv"); + this.writeCsvFile(this.geoLite2CityLocationsLines, + "GeoLite2-City-Locations.csv"); this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); } catch (IOException e) { throw new RuntimeException(e); @@ -105,36 +95,19 @@ public class LookupServiceTest { this.lookupResults = this.lookupService.lookup(this.addressStrings); }
- private void assertLookupResult( - List<String> manualGeoLiteCityBlocksLines, - List<String> automaticGeoLiteCityBlocksLines, - List<String> geoLiteCityBlocksLines, - List<String> geoLiteCityLocationLines, List<String> iso3166Lines, - List<String> regionLines, List<String> geoipASNum2Lines, - String addressString, String countryCode, String countryName, - String regionName, String cityName, String latitude, - String longitude, String aSNumber, String aSName) { + private void assertLookupResult(List<String> geoLite2CityBlocksLines, + List<String> geoLite2CityLocationsLines, + List<String> geoipASNum2Lines, String addressString, + String countryCode, String countryName, String regionName, + String cityName, String latitude, String longitude, String aSNumber, + String aSName) { this.addressStrings.add(addressString); this.populateLines(); - if (manualGeoLiteCityBlocksLines != null) { - this.manualGeoLiteCityBlocksLines = - manualGeoLiteCityBlocksLines; + if (geoLite2CityBlocksLines != null) { + this.geoLite2CityBlocksLines = geoLite2CityBlocksLines; } - if (automaticGeoLiteCityBlocksLines != null) { - this.automaticGeoLiteCityBlocksLines = - automaticGeoLiteCityBlocksLines; - } - if (geoLiteCityBlocksLines != null) { - this.geoLiteCityBlocksLines = geoLiteCityBlocksLines; - } - if (geoLiteCityLocationLines != null) { - this.geoLiteCityLocationLines = geoLiteCityLocationLines; - } - if (iso3166Lines != null) { - this.iso3166Lines = iso3166Lines; - } - if (regionLines != null) { - this.regionLines = regionLines; + if (geoLite2CityLocationsLines != null) { + this.geoLite2CityLocationsLines = geoLite2CityLocationsLines; } if (geoipASNum2Lines != null) { this.geoipASNum2Lines = geoipASNum2Lines; @@ -216,230 +189,154 @@ public class LookupServiceTest {
@Test() public void testLookup8888() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.8", "us", + this.assertLookupResult(null, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); }
@Test() public void testLookup8880() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.0", "us", + this.assertLookupResult(null, null, null, "8.8.8.0", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); }
@Test() public void testLookup888255() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.255", "us", + this.assertLookupResult(null, null, null, "8.8.8.255", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); }
@Test() public void testLookup888256() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.256", null, null, null, - null, null, null, null, null); + this.assertLookupResult(null, null, null, "8.8.8.256", null, null, + null, null, null, null, null, null); }
@Test() public void testLookup888Minus1() { - this.assertLookupResult(null, - null, null, null, null, null, null, "8.8.8.-1", null, null, null, - null, null, null, null, null); + this.assertLookupResult(null, null, null, "8.8.8.-1", null, null, + null, null, null, null, null, null); }
@Test() public void testLookup000() { - this.assertLookupResult(null, - null, null, null, null, null, null, "0.0.0.0", null, null, null, + this.assertLookupResult(null, null, null, "0.0.0.0", null, null, null, null, null, null, null, null); }
@Test() public void testLookupNoBlocksLines() { - this.assertLookupResult( - new ArrayList<String>(), null, null, null, null, null, null, + this.assertLookupResult(new ArrayList<String>(), null, null, "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() public void testLookupNoLocationLines() { - this.assertLookupResult(null, - null, null, new ArrayList<String>(), null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupNoIso3166Lines() { - this.assertLookupResult(null, - null, null, null, new ArrayList<String>(), null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupNoRegionLines() { - this.assertLookupResult(null, - null, null, null, null, new ArrayList<String>(), null, "8.8.8.8", - null, null, null, null, null, null, null, null); + this.assertLookupResult(null, new ArrayList<String>(), null, + "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() public void testLookupNoGeoipASNum2Lines() { - this.assertLookupResult(null, - null, null, null, null, null, new ArrayList<String>(), "8.8.8.8", - null, null, null, null, null, null, null, null); + this.assertLookupResult(null, null, new ArrayList<String>(), + "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() public void testLookupNoCorrespondingLocation() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("223,"US","","","",38.0000," - + "-97.0000,,"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, "AS15169", "Google Inc."); - } - - @Test() - public void testLookupNoCorrespondingCountryName() { - List<String> iso3166Lines = new ArrayList<String>(); - iso3166Lines.add("UY,"Uruguay""); - this.assertLookupResult(null, - null, null, null, iso3166Lines, null, null, "8.8.8.8", "us", - null, "California", "Mountain View", "37.3860", "-122.0838", - "AS15169", "Google Inc."); - } - - @Test() - public void testLookupNoCorrespondingRegionName() { - List<String> regionLines = new ArrayList<String>(); - regionLines.add("US,CO,"Colorado""); - this.assertLookupResult(null, - null, null, null, null, regionLines, null, "8.8.8.8", "us", - "United States", null, "Mountain View", "37.3860", "-122.0838", + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("6252001,NA,"North America",US," + + ""United States",,,,,"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, "37.3860", "-122.0838", "AS15169", "Google Inc."); }
@Test() - public void testLookupBlocksEndBeforeStart() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add(""134739200","134744063"," - + ""223""); - manualGeoLiteCityBlocksLines.add(""134744319","134744064"," - + ""32191""); - manualGeoLiteCityBlocksLines.add(""134744320","134751743"," - + ""223""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, - "8.8.8.8", null, null, null, null, null, null, "AS15169", - "Google Inc."); - } - - @Test() public void testLookupBlocksStartNotANumber() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add(""one","134744319"," - + ""32191""); + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:one,120,5375480,6252001,,94043," + + "37.3860,-122.0838,0,0"); this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() - public void testLookupBlocksStartTooLarge() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add(""1" - + String.valueOf(Long.MAX_VALUE) + "","134744319","32191""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + public void testLookupBlocksLocationX() { + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,X,X,,94043,37.3860," + + "-122.0838,0,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() - public void testLookupBlocksLocationX() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add(""134744064","134744319","X""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, - "8.8.8.8", null, null, null, null, null, null, null, null); + public void testLookupBlocksLocationEmpty() { + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,,,,,,,1,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, + "8.8.8.8", null, null, null, null, null, null, "AS15169", + "Google Inc."); }
@Test() public void testLookupBlocksTooFewFields() { - List<String> manualGeoLiteCityBlocksLines = new ArrayList<String>(); - manualGeoLiteCityBlocksLines.add("Copyright (c) 2011 MaxMind Inc. " - + "All Rights Reserved."); - manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - manualGeoLiteCityBlocksLines.add(""134744064","134744319""); - this.assertLookupResult( - manualGeoLiteCityBlocksLines, null, null, null, null, null, null, + List<String> geoLite2CityBlocksLines = new ArrayList<String>(); + geoLite2CityBlocksLines.add("network_start_ip," + + "network_mask_length,geoname_id,registered_country_geoname_id," + + "represented_country_geoname_id,postal_code,latitude,longitude," + + "is_anonymous_proxy,is_satellite_provider"); + geoLite2CityBlocksLines.add("::ffff:8.8.8.0,120,5375480,6252001," + + ",94043,37.3860,-122.0838,0"); + this.assertLookupResult(geoLite2CityBlocksLines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() public void testLookupLocationLocIdNotANumber() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("threetwoonenineone,"US","CA"," - + ""Mountain View","",37.3860,-122.0838,807,650"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("threetwoonenineone,NA," + + ""North America",US,"United States",CA,California," + + ""Mountain View",807,America/Los_Angeles"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() public void testLookupLocationTooFewFields() { - List<String> geoLiteCityLocationLines = new ArrayList<String>(); - geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind LLC. All " - + "Rights Reserved."); - geoLiteCityLocationLines.add("locId,country,region,city,postalCode," - + "latitude,longitude,metroCode,areaCode"); - geoLiteCityLocationLines.add("32191,"US","CA","Mountain View"," - + """,37.3860,-122.0838,807"); - this.assertLookupResult(null, - null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", - null, null, null, null, null, null, null, null); - } - - @Test() - public void testLookupIso3166TooFewFields() { - List<String> iso3166Lines = new ArrayList<String>(); - iso3166Lines.add("US"); - this.assertLookupResult(null, - null, null, null, iso3166Lines, null, null, "8.8.8.8", null, null, - null, null, null, null, null, null); - } - - @Test() - public void testLookupRegionTooFewFields() { - List<String> regionLines = new ArrayList<String>(); - regionLines.add("US,CA"); - this.assertLookupResult(null, - null, null, null, null, regionLines, null, "8.8.8.8", null, null, - null, null, null, null, null, null); + List<String> geoLite2CityLocationsLines = new ArrayList<String>(); + geoLite2CityLocationsLines.add("geoname_id,continent_code," + + "continent_name,country_iso_code,country_name," + + "subdivision_iso_code,subdivision_name,city_name,metro_code," + + "time_zone"); + geoLite2CityLocationsLines.add("5375480,NA,"North America",US," + + ""United States",CA,California,"Mountain View",807"); + this.assertLookupResult(null, geoLite2CityLocationsLines, null, + "8.8.8.8", null, null, null, null, null, null, null, null); }
@Test() @@ -450,8 +347,7 @@ public class LookupServiceTest { geoipASNum2Lines.add("134744319,134744064,"AS15169 Google Inc.""); geoipASNum2Lines.add("134744320,134750463,"AS3356 Level 3 " + "Communications""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", "us", + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", null, null); } @@ -460,8 +356,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2StartNotANumber() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("one,134744319,"AS15169 Google Inc.""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); }
@@ -470,8 +365,7 @@ public class LookupServiceTest { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("1" + String.valueOf(Long.MAX_VALUE) + ",134744319,"AS15169 Google Inc.""); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); }
@@ -479,8 +373,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2TooFewFields() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("134744064,134744319"); - this.assertLookupResult(null, - null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, + this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } }
tor-commits@lists.torproject.org
-
karsten＠torproject.org