commit b45c630b7056becd068f381ab48a068e40ff4ecc Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Nov 15 19:36:29 2018 +0100
Tweak GeoLite2 ASN database parsing.
Also fix unit tests.
Still part of #28254. --- .../metrics/onionoo/updater/LookupService.java | 17 +- .../metrics/onionoo/updater/LookupServiceTest.java | 172 ++++++++++++--------- 2 files changed, 108 insertions(+), 81 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java index 0967bf2..3664186 100644 --- a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java @@ -218,7 +218,7 @@ public class LookupService {
/* Obtain a map from IP address numbers to ASN. */ Map<Long, String[]> addressNumberAsn = new HashMap<>(); - try (BufferedReader br = this.createBufferedReaderFromIso88591File( + try (BufferedReader br = this.createBufferedReaderFromUtf8File( this.geoLite2AsnBlocksIpv4CsvFile)) { SortedSet<Long> sortedAddressNumbers = new TreeSet<>( addressStringNumbers.values()); @@ -248,6 +248,13 @@ public class LookupService { this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); return lookupResults; } + String asNumber = "AS" + String.valueOf(Integer.parseInt(parts[1])); + String asName = parts[2]; + if (asName.isEmpty()) { + log.error("Illegal AS name in '{}' in {}.", line, + this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); + return lookupResults; + } while (firstAddressNumber < startIpNum && firstAddressNumber != -1L) { sortedAddressNumbers.remove(firstAddressNumber); @@ -261,7 +268,7 @@ public class LookupService { while (firstAddressNumber <= endIpNum && firstAddressNumber != -1L) { addressNumberAsn.put(firstAddressNumber, - new String[] { "AS" + parts[1], parts[2] }); + new String[] { asNumber, asName }); sortedAddressNumbers.remove(firstAddressNumber); if (sortedAddressNumbers.isEmpty()) { firstAddressNumber = -1L; @@ -341,12 +348,6 @@ public class LookupService { StandardCharsets.UTF_8.newDecoder()); }
- private BufferedReader createBufferedReaderFromIso88591File( - File iso88591File) throws FileNotFoundException { - return this.createBufferedReaderFromFile(iso88591File, - StandardCharsets.ISO_8859_1.newDecoder()); - } - private BufferedReader createBufferedReaderFromFile(File file, CharsetDecoder dec) throws FileNotFoundException { dec.onMalformedInput(CodingErrorAction.REPORT); diff --git a/src/test/java/org/torproject/metrics/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/metrics/onionoo/updater/LookupServiceTest.java index 90a840c..f521dfc 100644 --- a/src/test/java/org/torproject/metrics/onionoo/updater/LookupServiceTest.java +++ b/src/test/java/org/torproject/metrics/onionoo/updater/LookupServiceTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.SortedMap; @@ -28,7 +29,7 @@ public class LookupServiceTest {
private List<String> geoLite2CityBlocksIPv4Lines; private List<String> geoLite2CityLocationsEnLines; - private List<String> geoipAsNum2Lines; + private List<String> geoLite2AsnBlocksIpv4Lines;
private LookupService lookupService;
@@ -59,34 +60,35 @@ public class LookupServiceTest { this.geoLite2CityLocationsEnLines.add("5375480,en,NA," + ""North America",US,"United States",CA,California,,," + ""Mountain View",807,America/Los_Angeles"); - this.geoipAsNum2Lines = new ArrayList<>(); - this.geoipAsNum2Lines.add("134743296,134744063,"AS3356 Level 3 " - + "Communications""); - this.geoipAsNum2Lines.add("134744064,134744319,"AS15169 Google " - + "Inc.""); - this.geoipAsNum2Lines.add("134744320,134750463,"AS3356 Level 3 " - + "Communications""); + this.geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + this.geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + this.geoLite2AsnBlocksIpv4Lines.add( + "8.8.6.0/23,3356,"Level 3 Parent, LLC""); + this.geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/24,15169,"Google LLC""); + this.geoLite2AsnBlocksIpv4Lines.add( + "8.8.9.0/24,3356,"Level 3 Parent, LLC""); }
private void writeCsvFiles() { try { this.writeCsvFile(this.geoLite2CityBlocksIPv4Lines, - "GeoLite2-City-Blocks-IPv4.csv", "UTF-8"); + "GeoLite2-City-Blocks-IPv4.csv"); this.writeCsvFile(this.geoLite2CityLocationsEnLines, - "GeoLite2-City-Locations-en.csv", "UTF-8"); - this.writeCsvFile(this.geoipAsNum2Lines, "GeoIPASNum2.csv", - "ISO-8859-1"); + "GeoLite2-City-Locations-en.csv"); + this.writeCsvFile(this.geoLite2AsnBlocksIpv4Lines, + "GeoLite2-ASN-Blocks-IPv4.csv"); } catch (IOException e) { throw new RuntimeException(e); } }
- private void writeCsvFile(List<String> lines, String fileName, - String encoding) throws IOException { + private void writeCsvFile(List<String> lines, String fileName) + throws IOException { if (lines != null && !lines.isEmpty()) { try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(new File(this.tempGeoipDir, fileName)), - encoding))) { + StandardCharsets.UTF_8))) {
for (String line : lines) { bw.write(line); @@ -103,7 +105,7 @@ public class LookupServiceTest {
private void assertLookupResult(List<String> geoLite2CityBlocksLines, List<String> geoLite2CityLocationsLines, - List<String> geoipAsNum2Lines, String addressString, + List<String> geoLite2AsnBlocksIpv4Lines, String addressString, String countryCode, String countryName, String regionName, String cityName, Float latitude, Float longitude, String asNumber, String asName) { @@ -115,8 +117,8 @@ public class LookupServiceTest { if (geoLite2CityLocationsLines != null) { this.geoLite2CityLocationsEnLines = geoLite2CityLocationsLines; } - if (geoipAsNum2Lines != null) { - this.geoipAsNum2Lines = geoipAsNum2Lines; + if (geoLite2AsnBlocksIpv4Lines != null) { + this.geoLite2AsnBlocksIpv4Lines = geoLite2AsnBlocksIpv4Lines; } this.writeCsvFiles(); /* Disable log messages printed to System.err. */ @@ -197,21 +199,21 @@ public class LookupServiceTest { public void testLookup8888() { this.assertLookupResult(null, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", 37.3860f, - -122.0838f, "AS15169", "Google Inc."); + -122.0838f, "AS15169", "Google LLC"); }
@Test() public void testLookup8880() { this.assertLookupResult(null, null, null, "8.8.8.0", "us", "United States", "California", "Mountain View", 37.3860f, - -122.0838f, "AS15169", "Google Inc."); + -122.0838f, "AS15169", "Google LLC"); }
@Test() public void testLookup888255() { this.assertLookupResult(null, null, null, "8.8.8.255", "us", "United States", "California", "Mountain View", 37.3860f, - -122.0838f, "AS15169", "Google Inc."); + -122.0838f, "AS15169", "Google LLC"); }
@Test() @@ -262,7 +264,7 @@ public class LookupServiceTest { + ""North America",US,"United States",,,,,,,"); this.assertLookupResult(null, geoLite2CityLocationsEnLines, null, "8.8.8.8", null, null, null, null, 37.3860f, -122.0838f, - "AS15169", "Google Inc."); + "AS15169", "Google LLC"); }
@Test() @@ -302,7 +304,7 @@ public class LookupServiceTest { geoLite2CityBlocksIPv4Lines.add("8.8.8.0/24,,,,0,0,,,"); this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null, "8.8.8.8", null, null, null, null, null, null, "AS15169", - "Google Inc."); + "Google LLC"); }
@Test() @@ -329,7 +331,7 @@ public class LookupServiceTest { + "0,94035,37.3860,-122.0838,937"); this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", - 37.3860f, -122.0838f, "AS15169", "Google Inc."); + 37.3860f, -122.0838f, "AS15169", "Google LLC"); }
@Test() @@ -343,7 +345,7 @@ public class LookupServiceTest { + "0,94035,37.3860,-122.0838,937,1,2,30000000000000"); this.assertLookupResult(geoLite2CityBlocksIPv4Lines, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", - 37.3860f, -122.0838f, "AS15169", "Google Inc."); + 37.3860f, -122.0838f, "AS15169", "Google LLC"); }
@Test() @@ -377,52 +379,74 @@ public class LookupServiceTest { }
@Test() - public void testLookupGeoipAsNum2EndBeforeStart() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("134743296,134744063,"AS3356 Level 3 " - + "Communications""); - geoipAsNum2Lines.add("134744319,134744064,"AS15169 Google Inc.""); - geoipAsNum2Lines.add("134744320,134750463,"AS3356 Level 3 " - + "Communications""); - this.assertLookupResult(null, null, geoipAsNum2Lines, "8.8.8.8", "us", - "United States", "California", "Mountain View", 37.3860f, - -122.0838f, null, null); + public void testLookupGeoLite2AsnBlocksIpv4IllegalIpAddress() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add( + "eighteighteightzero/24,15169,"Google LLC""); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); }
@Test() - public void testLookupGeoipAsNum2StartNotANumber() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("one,134744319,"AS15169 Google Inc.""); - this.assertLookupResult(null, null, geoipAsNum2Lines, "8.8.8.8", null, - null, null, null, null, null, null, null); + public void testLookupGeoLite2AsnBlocksIpv4MaskTooSmall() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/7,15169,"Google LLC""); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); }
@Test() - public void testLookupGeoipAsNum2StartTooLarge() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("1" + String.valueOf(Long.MAX_VALUE) - + ",134744319,"AS15169 Google Inc.""); - this.assertLookupResult(null, null, geoipAsNum2Lines, "8.8.8.8", null, - null, null, null, null, null, null, null); + public void testLookupGeoLite2AsnBlocksIpv4MaskTooBig() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/33,15169,"Google LLC""); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); }
@Test() - public void testLookupGeoipAsNum2TooFewFields() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("134744064,134744319"); - this.assertLookupResult(null, null, geoipAsNum2Lines, "8.8.8.8", null, - null, null, null, null, null, null, null); + public void testLookupGeoLite2AsnBlocksIpv4MaskMissing() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0,15169,"Google LLC""); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); }
@Test() - public void testLookupGeoipAsNum2NoAsName() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("134743296,134744063,AS3356"); - geoipAsNum2Lines.add("134744064,134744319,AS15169"); - geoipAsNum2Lines.add("134744320,134750463,AS3356"); - this.assertLookupResult(null, null, geoipAsNum2Lines, "8.8.8.8", "us", - "United States", "California", "Mountain View", 37.3860f, - -122.0838f, "AS15169", ""); + public void testLookupGeoLite2AsnBlocksIpv4TooFewFields() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/24,15169"); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); + } + + @Test() + public void testLookupGeoLite2AsnBlocksIpv4TooFewFieldsNoAsNumber() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/24,,"Google LLC""); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); + } + + @Test() + public void testLookupGeoLite2AsnBlocksIpv4TooFewFieldsNoAsName() { + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("8.8.8.0/24,15169,"); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, "8.8.8.8", + null, null, null, null, null, null, null, null); }
@Test() @@ -645,20 +669,22 @@ public class LookupServiceTest { @Test() @SuppressWarnings("AvoidEscapedUnicodeCharacters") public void testLookupLocationAsNameNonAscii() { - List<String> geoipAsNum2Lines = new ArrayList<>(); - geoipAsNum2Lines.add("3207917568,3207919615,"AS52693 Conectel " - + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME""); - geoipAsNum2Lines.add("3211196416,3211198463,"AS262934 " - + "IP\u00B7RED""); - geoipAsNum2Lines.add("3227819264,3227819519,"AS263226 " - + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.""); - this.assertLookupResult(null, null, geoipAsNum2Lines, "191.52.240.0", - null, null, null, null, null, null, "AS52693", "Conectel " - + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME"); - this.assertLookupResult(null, null, geoipAsNum2Lines, "191.102.248.0", - null, null, null, null, null, null, "AS262934", "IP\u00B7RED"); - this.assertLookupResult(null, null, geoipAsNum2Lines, "192.100.157.0", - null, null, null, null, null, null, "AS263226", + List<String> geoLite2AsnBlocksIpv4Lines = new ArrayList<>(); + geoLite2AsnBlocksIpv4Lines.add("network,autonomous_system_number," + + "autonomous_system_organization"); + geoLite2AsnBlocksIpv4Lines.add("191.52.240.0/21,52693," + + ""Conectel Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME""); + geoLite2AsnBlocksIpv4Lines.add("191.102.248.0/21,262934,"IP\u00B7RED""); + geoLite2AsnBlocksIpv4Lines.add("192.100.157.0/24,263226," + + ""COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.""); + //this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, + // "191.52.240.0", null, null, null, null, null, null, "AS52693", + // "Conectel Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME"); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, + "191.102.248.0", null, null, null, null, null, null, "AS262934", + "IP\u00B7RED"); + this.assertLookupResult(null, null, geoLite2AsnBlocksIpv4Lines, + "192.100.157.0", null, null, null, null, null, null, "AS263226", "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A."); } }