commit 2d8eb09d1e37af4efd1776acf73ff94b98ca8bbd Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Feb 24 10:50:54 2015 +0100 Fix character encoding of ASN database file. Looks like MaxMind's ASN database files are ISO-8859-1 encoded, not UTF-8 encoded... --- .../torproject/onionoo/updater/LookupService.java | 2 +- .../onionoo/updater/LookupServiceTest.java | 32 ++++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index f88e752..a48e721 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -228,7 +228,7 @@ public class LookupService { addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoIPASNum2CsvFile), "UTF-8")); + new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); String line; long previousStartIpNum = -1L; while ((line = br.readLine()) != null) { diff --git a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java index 9275fbf..5673d9a 100644 --- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java +++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java @@ -72,21 +72,22 @@ public class LookupServiceTest { private void writeCsvFiles() { try { this.writeCsvFile(this.geoLite2CityBlocksIPv4Lines, - "GeoLite2-City-Blocks-IPv4.csv"); + "GeoLite2-City-Blocks-IPv4.csv", "UTF-8"); this.writeCsvFile(this.geoLite2CityLocationsEnLines, - "GeoLite2-City-Locations-en.csv"); - this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); + "GeoLite2-City-Locations-en.csv", "UTF-8"); + this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv", + "ISO-8859-1"); } catch (IOException e) { throw new RuntimeException(e); } } - private void writeCsvFile(List<String> lines, String fileName) - throws IOException { + private void writeCsvFile(List<String> lines, String fileName, + String encoding) throws IOException { if (lines != null && !lines.isEmpty()) { BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(new File(this.tempGeoipDir, fileName)), - "UTF-8")); + encoding)); for (String line : lines) { bw.write(line + "\n"); } @@ -592,5 +593,24 @@ public class LookupServiceTest { "Pomeranian Voivodeship", "Gda\u0144sk", 54.3608f, 18.6583f, null, null); } + + @Test() + public void testLookupLocationASNameNonAscii() { + List<String> geoipASNum2Lines = new ArrayList<String>(); + geoipASNum2Lines.add("3207917568,3207919615,\"AS52693 Conectel " + + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME\""); + geoipASNum2Lines.add("3211196416,3211198463,\"AS262934 " + + "IP\u00B7RED\""); + geoipASNum2Lines.add("3227819264,3227819519,\"AS263226 " + + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.\""); + this.assertLookupResult(null, null, geoipASNum2Lines, "191.52.240.0", + null, null, null, null, null, null, "AS52693", "Conectel " + + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME"); + this.assertLookupResult(null, null, geoipASNum2Lines, "191.102.248.0", + null, null, null, null, null, null, "AS262934", "IP\u00B7RED"); + this.assertLookupResult(null, null, geoipASNum2Lines, "192.100.157.0", + null, null, null, null, null, null, "AS263226", + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A."); + } }