commit 9398d6999022b4d03a2305b8dbaf66bfd52801be Author: iwakeh <iwakeh@users.ourproject.org> Date: Thu Jan 22 21:00:00 2015 +0000 Do not proceed with lookup if an encoding error is reported and log file encoding errors. --- .../torproject/onionoo/updater/LookupService.java | 48 +++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index a48e721..16dd625 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -5,8 +5,13 @@ package org.torproject.onionoo.updater; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -114,9 +119,8 @@ public class LookupService { try { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(this.geoLite2CityBlocksIPv4CsvFile), - "UTF-8")); + BufferedReader br = this.createBufferedReaderFromUtf8File( + this.geoLite2CityBlocksIPv4CsvFile); String line = br.readLine(); while ((line = br.readLine()) != null) { String[] parts = line.split(",", 9); @@ -177,7 +181,8 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() + "."); + + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() + + ": " + e); return lookupResults; } @@ -186,9 +191,8 @@ public class LookupService { try { Set<Long> blockNumbers = new HashSet<Long>( addressNumberBlocks.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(this.geoLite2CityLocationsEnCsvFile), - "UTF-8")); + BufferedReader br = this.createBufferedReaderFromUtf8File( + this.geoLite2CityLocationsEnCsvFile); String line = br.readLine(); while ((line = br.readLine()) != null) { String[] parts = line.replaceAll("\"", "").split(",", 13); @@ -217,7 +221,8 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + this.geoLite2CityLocationsEnCsvFile.getAbsolutePath() + "."); + + this.geoLite2CityLocationsEnCsvFile.getAbsolutePath() + + ": " + e); return lookupResults; } @@ -227,8 +232,8 @@ public class LookupService { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); + BufferedReader br = this.createBufferedReaderFromIso88591File( + this.geoIPASNum2CsvFile); String line; long previousStartIpNum = -1L; while ((line = br.readLine()) != null) { @@ -285,7 +290,7 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIPASNum2CsvFile.getAbsolutePath() + ": " + e); return lookupResults; } @@ -339,6 +344,27 @@ public class LookupService { return lookupResults; } + private BufferedReader createBufferedReaderFromUtf8File(File utf8File) + throws FileNotFoundException, CharacterCodingException { + CharsetDecoder dec = StandardCharsets.UTF_8.newDecoder(); + dec.onMalformedInput(CodingErrorAction.REPORT); + dec.onUnmappableCharacter(CodingErrorAction.REPORT); + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(utf8File), dec)); + return br; + } + + private BufferedReader createBufferedReaderFromIso88591File( + File iso88591File) throws FileNotFoundException, + CharacterCodingException { + CharsetDecoder dec = StandardCharsets.ISO_8859_1.newDecoder(); + dec.onMalformedInput(CodingErrorAction.REPORT); + dec.onUnmappableCharacter(CodingErrorAction.REPORT); + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(iso88591File), dec)); + return br; + } + private int addressesLookedUp = 0, addressesResolved = 0; public String getStatsString() {