commit 54e26d8a4f109196ebbb5849df05e799b98e63ed Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Oct 31 09:33:04 2018 +0100 Update to the GeoLite2 ASN database format. Fixes #28254. --- CHANGELOG.md | 3 ++ .../metrics/onionoo/updater/LookupService.java | 51 ++++++++++++---------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7fc775..3b7dd16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 7.?-1.??.? - 2018-1?-?? + * Medium changes + - Update to the GeoLite2 ASN database format. + * Minor changes - Rename root package org.torproject.onionoo to org.torproject.metrics.onionoo. diff --git a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java index edcb2bd..0967bf2 100644 --- a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java @@ -38,7 +38,7 @@ public class LookupService { private File geoLite2CityLocationsEnCsvFile; - private File geoIpAsNum2CsvFile; + private File geoLite2AsnBlocksIpv4CsvFile; private boolean hasAllFiles = false; @@ -62,9 +62,10 @@ public class LookupService { + "geoip/."); return; } - this.geoIpAsNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); - if (!this.geoIpAsNum2CsvFile.exists()) { - log.error("No GeoIPASNum2.csv file in geoip/."); + this.geoLite2AsnBlocksIpv4CsvFile = new File(this.geoipDir, + "GeoLite2-ASN-Blocks-IPv4.csv"); + if (!this.geoLite2AsnBlocksIpv4CsvFile.exists()) { + log.error("No GeoLite2-ASN-Blocks-IPv4.csv file in geoip/."); return; } this.hasAllFiles = true; @@ -216,29 +217,37 @@ public class LookupService { } /* Obtain a map from IP address numbers to ASN. */ - Map<Long, String> addressNumberAsn = new HashMap<>(); + Map<Long, String[]> addressNumberAsn = new HashMap<>(); try (BufferedReader br = this.createBufferedReaderFromIso88591File( - this.geoIpAsNum2CsvFile)) { + this.geoLite2AsnBlocksIpv4CsvFile)) { SortedSet<Long> sortedAddressNumbers = new TreeSet<>( addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); String line; - long previousStartIpNum = -1L; + br.readLine(); while ((line = br.readLine()) != null) { String[] parts = line.replaceAll("\"", "").split(",", 3); if (parts.length != 3) { log.error("Illegal line '{}' in {}.", line, - geoIpAsNum2CsvFile.getAbsolutePath()); + this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); return lookupResults; } try { - long startIpNum = Long.parseLong(parts[0]); - if (startIpNum <= previousStartIpNum) { - log.error("Line '{}' not sorted in {}.", line, - geoIpAsNum2CsvFile.getAbsolutePath()); + String[] networkAddressAndMask = parts[0].split("/"); + String startAddressString = networkAddressAndMask[0]; + long startIpNum = this.parseAddressString(startAddressString); + if (startIpNum < 0L) { + log.error("Illegal IP address in '{}' in {}.", line, + this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); + return lookupResults; + } + int networkMaskLength = networkAddressAndMask.length < 2 ? 0 + : Integer.parseInt(networkAddressAndMask[1]); + if (networkMaskLength < 8 || networkMaskLength > 32) { + log.error("Missing or illegal network mask in '{}' in {}.", line, + this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); return lookupResults; } - previousStartIpNum = startIpNum; while (firstAddressNumber < startIpNum && firstAddressNumber != -1L) { sortedAddressNumbers.remove(firstAddressNumber); @@ -248,12 +257,11 @@ public class LookupService { firstAddressNumber = sortedAddressNumbers.first(); } } - long endIpNum = Long.parseLong(parts[1]); + long endIpNum = startIpNum + (1 << (32 - networkMaskLength)) - 1; while (firstAddressNumber <= endIpNum && firstAddressNumber != -1L) { - if (parts[2].startsWith("AS")) { - addressNumberAsn.put(firstAddressNumber, parts[2]); - } + addressNumberAsn.put(firstAddressNumber, + new String[] { "AS" + parts[1], parts[2] }); sortedAddressNumbers.remove(firstAddressNumber); if (sortedAddressNumbers.isEmpty()) { firstAddressNumber = -1L; @@ -266,13 +274,13 @@ public class LookupService { } } catch (NumberFormatException e) { log.error("Number format exception while parsing line '{}' in {}.", - line, geoIpAsNum2CsvFile.getAbsolutePath()); + line, this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath()); return lookupResults; } } } catch (IOException e) { log.error("I/O exception while reading {}: {}", - geoIpAsNum2CsvFile.getAbsolutePath(), e); + this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath(), e); return lookupResults; } @@ -313,10 +321,9 @@ public class LookupService { lookupResult.setLongitude(latLong[1]); } if (addressNumberAsn.containsKey(addressNumber)) { - String[] parts = addressNumberAsn.get(addressNumber).split(" ", - 2); + String[] parts = addressNumberAsn.get(addressNumber); lookupResult.setAsNumber(parts[0]); - lookupResult.setAsName(parts.length == 2 ? parts[1] : ""); + lookupResult.setAsName(parts[1]); } lookupResults.put(addressString, lookupResult); }