February 2015 - tor-commits - lists.torproject.org

[onionoo/master] Fix character encoding of ASN database file.
by karsten＠torproject.org 25 Feb '15

25 Feb '15

commit 2d8eb09d1e37af4efd1776acf73ff94b98ca8bbd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Feb 24 10:50:54 2015 +0100 Fix character encoding of ASN database file. Looks like MaxMind's ASN database files are ISO-8859-1 encoded, not UTF-8 encoded... --- .../torproject/onionoo/updater/LookupService.java | 2 +- .../onionoo/updater/LookupServiceTest.java | 32 ++++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --… [View More]git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index f88e752..a48e721 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -228,7 +228,7 @@ public class LookupService { addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoIPASNum2CsvFile), "UTF-8")); + new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); String line; long previousStartIpNum = -1L; while ((line = br.readLine()) != null) { diff --git a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java index 9275fbf..5673d9a 100644 --- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java +++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java @@ -72,21 +72,22 @@ public class LookupServiceTest { private void writeCsvFiles() { try { this.writeCsvFile(this.geoLite2CityBlocksIPv4Lines, - "GeoLite2-City-Blocks-IPv4.csv"); + "GeoLite2-City-Blocks-IPv4.csv", "UTF-8"); this.writeCsvFile(this.geoLite2CityLocationsEnLines, - "GeoLite2-City-Locations-en.csv"); - this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); + "GeoLite2-City-Locations-en.csv", "UTF-8"); + this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv", + "ISO-8859-1"); } catch (IOException e) { throw new RuntimeException(e); } } - private void writeCsvFile(List<String> lines, String fileName) - throws IOException { + private void writeCsvFile(List<String> lines, String fileName, + String encoding) throws IOException { if (lines != null && !lines.isEmpty()) { BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(new File(this.tempGeoipDir, fileName)), - "UTF-8")); + encoding)); for (String line : lines) { bw.write(line + "\n"); } @@ -592,5 +593,24 @@ public class LookupServiceTest { "Pomeranian Voivodeship", "Gda\u0144sk", 54.3608f, 18.6583f, null, null); } + + @Test() + public void testLookupLocationASNameNonAscii() { + List<String> geoipASNum2Lines = new ArrayList<String>(); + geoipASNum2Lines.add("3207917568,3207919615,\"AS52693 Conectel " + + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME\""); + geoipASNum2Lines.add("3211196416,3211198463,\"AS262934 " + + "IP\u00B7RED\""); + geoipASNum2Lines.add("3227819264,3227819519,\"AS263226 " + + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A.\""); + this.assertLookupResult(null, null, geoipASNum2Lines, "191.52.240.0", + null, null, null, null, null, null, "AS52693", "Conectel " + + "Telecomunica\u00E7\u00F5es e Inform\u00E1tica Ltda ME"); + this.assertLookupResult(null, null, geoipASNum2Lines, "191.102.248.0", + null, null, null, null, null, null, "AS262934", "IP\u00B7RED"); + this.assertLookupResult(null, null, geoipASNum2Lines, "192.100.157.0", + null, null, null, null, null, null, "AS263226", + "COMPA\u00D1\u00CDA FINANCIERA ARGENTINA S.A."); + } } [View Less]

1 0

[onionoo/master] Do not proceed with lookup if an encoding error is reported and log file encoding errors.
by karsten＠torproject.org 25 Feb '15

25 Feb '15

commit 9398d6999022b4d03a2305b8dbaf66bfd52801be Author: iwakeh <iwakeh(a)users.ourproject.org> Date: Thu Jan 22 21:00:00 2015 +0000 Do not proceed with lookup if an encoding error is reported and log file encoding errors. --- .../torproject/onionoo/updater/LookupService.java | 48 +++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/… [View More]LookupService.java index a48e721..16dd625 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -5,8 +5,13 @@ package org.torproject.onionoo.updater; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -114,9 +119,8 @@ public class LookupService { try { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(this.geoLite2CityBlocksIPv4CsvFile), - "UTF-8")); + BufferedReader br = this.createBufferedReaderFromUtf8File( + this.geoLite2CityBlocksIPv4CsvFile); String line = br.readLine(); while ((line = br.readLine()) != null) { String[] parts = line.split(",", 9); @@ -177,7 +181,8 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() + "."); + + this.geoLite2CityBlocksIPv4CsvFile.getAbsolutePath() + + ": " + e); return lookupResults; } @@ -186,9 +191,8 @@ public class LookupService { try { Set<Long> blockNumbers = new HashSet<Long>( addressNumberBlocks.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(this.geoLite2CityLocationsEnCsvFile), - "UTF-8")); + BufferedReader br = this.createBufferedReaderFromUtf8File( + this.geoLite2CityLocationsEnCsvFile); String line = br.readLine(); while ((line = br.readLine()) != null) { String[] parts = line.replaceAll("\"", "").split(",", 13); @@ -217,7 +221,8 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + this.geoLite2CityLocationsEnCsvFile.getAbsolutePath() + "."); + + this.geoLite2CityLocationsEnCsvFile.getAbsolutePath() + + ": " + e); return lookupResults; } @@ -227,8 +232,8 @@ public class LookupService { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); + BufferedReader br = this.createBufferedReaderFromIso88591File( + this.geoIPASNum2CsvFile); String line; long previousStartIpNum = -1L; while ((line = br.readLine()) != null) { @@ -285,7 +290,7 @@ public class LookupService { br.close(); } catch (IOException e) { log.error("I/O exception while reading " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIPASNum2CsvFile.getAbsolutePath() + ": " + e); return lookupResults; } @@ -339,6 +344,27 @@ public class LookupService { return lookupResults; } + private BufferedReader createBufferedReaderFromUtf8File(File utf8File) + throws FileNotFoundException, CharacterCodingException { + CharsetDecoder dec = StandardCharsets.UTF_8.newDecoder(); + dec.onMalformedInput(CodingErrorAction.REPORT); + dec.onUnmappableCharacter(CodingErrorAction.REPORT); + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(utf8File), dec)); + return br; + } + + private BufferedReader createBufferedReaderFromIso88591File( + File iso88591File) throws FileNotFoundException, + CharacterCodingException { + CharsetDecoder dec = StandardCharsets.ISO_8859_1.newDecoder(); + dec.onMalformedInput(CodingErrorAction.REPORT); + dec.onUnmappableCharacter(CodingErrorAction.REPORT); + BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(iso88591File), dec)); + return br; + } + private int addressesLookedUp = 0, addressesResolved = 0; public String getStatsString() { [View Less]

1 0

[tor/master] Remove tor_strclear(), and replace previous calls with memwipe().
by nickm＠torproject.org 25 Feb '15

25 Feb '15

commit cbd26157c52e27af16efcb474861eb4126085283 Author: Yawning Angel <yawning(a)schwanenlied.me> Date: Tue Feb 17 18:53:33 2015 +0000 Remove tor_strclear(), and replace previous calls with memwipe(). --- src/common/util.c | 10 ---------- src/common/util.h | 1 - src/or/rendservice.c | 4 ++-- src/test/test_util.c | 21 --------------------- 4 files changed, 2 insertions(+), 34 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index 442d57a..2c3a1a1 … [View More]100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -771,16 +771,6 @@ fast_memcmpstart(const void *mem, size_t memlen, return fast_memcmp(mem, prefix, plen); } -/** Given a nul-terminated string s, set every character before the nul - * to zero. */ -void -tor_strclear(char *s) -{ - while (*s) { - *s++ = '\0'; - } -} - /** Return a pointer to the first char of s that is not whitespace and * not a comment, or to the terminating NUL if no such character exists. */ diff --git a/src/common/util.h b/src/common/util.h index ea774bd..783c2a1 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -209,7 +209,6 @@ int strcasecmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int strcmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int strcasecmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int fast_memcmpstart(const void *mem, size_t memlen, const char *prefix); -void tor_strclear(char *s); void tor_strstrip(char *s, const char *strip) ATTR_NONNULL((1,2)); long tor_parse_long(const char *s, int base, long min, diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 6c934c8..0368696 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -173,7 +173,7 @@ rend_authorized_client_free(rend_authorized_client_t *client) return; if (client->client_key) crypto_pk_free(client->client_key); - tor_strclear(client->client_name); + memwipe(client->client_name, 0, strlen(client->client_name)); tor_free(client->client_name); memwipe(client->descriptor_cookie, 0, sizeof(client->descriptor_cookie)); tor_free(client); @@ -1052,7 +1052,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) abort_writing_to_file(open_hfile); done: if (client_keys_str) { - tor_strclear(client_keys_str); + memwipe(client_keys_str, 0, strlen(client_keys_str)); tor_free(client_keys_str); } strmap_free(parsed_clients, rend_authorized_client_strmap_item_free); diff --git a/src/test/test_util.c b/src/test/test_util.c index 9939bf4..41dd917 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4111,26 +4111,6 @@ test_util_laplace(void *arg) ; } -static void -test_util_strclear(void *arg) -{ - static const char *vals[] = { "", "a", "abcdef", "abcdefgh", NULL }; - int i; - char *v = NULL; - (void)arg; - - for (i = 0; vals[i]; ++i) { - size_t n; - v = tor_strdup(vals[i]); - n = strlen(v); - tor_strclear(v); - tt_assert(tor_mem_is_zero(v, n+1)); - tor_free(v); - } - done: - tor_free(v); -} - #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -4351,7 +4331,6 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(di_ops), UTIL_TEST(round_to_next_multiple_of, 0), UTIL_TEST(laplace, 0), - UTIL_TEST(strclear, 0), UTIL_TEST(find_str_at_start_of_line, 0), UTIL_TEST(string_is_C_identifier, 0), UTIL_TEST(asprintf, 0), [View Less]

1 0

[tor/master] Merge remote-tracking branch 'yawning/bug14922'
by nickm＠torproject.org 25 Feb '15

25 Feb '15

commit 0dde4d6fa265b3f861900447c2cdcf23335072cf Merge: cc6fd98 cbd2615 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Feb 25 08:56:34 2015 -0500 Merge remote-tracking branch 'yawning/bug14922' src/common/util.c | 10 ---------- src/common/util.h | 1 - src/or/rendservice.c | 4 ++-- src/test/test_util.c | 21 --------------------- 4 files changed, 2 insertions(+), 34 deletions(-)

1 0

[tor/master] changes file for 14922
by nickm＠torproject.org 25 Feb '15

25 Feb '15

commit 320a68026ede1f5707b1b48f1b8c3bf34e11d931 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Feb 25 08:57:13 2015 -0500 changes file for 14922 --- changes/bug14922 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/bug14922 b/changes/bug14922 new file mode 100644 index 0000000..65396e4 --- /dev/null +++ b/changes/bug14922 @@ -0,0 +1,3 @@ + o Removed code: + - Remove the 'tor_strclear()' function; use memwipe() instead. + Closes ticket 14922. \ … [View More]

1 0

[onionoo/master] Fix bandwidth graphs of relays running 0.2.6.3 and later.
by karsten＠torproject.org 25 Feb '15

25 Feb '15

commit 26fdd26de0d34794415313556b7092bf2a274d31 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Feb 22 13:11:47 2015 +0100 Fix bandwidth graphs of relays running 0.2.6.3 and later. Relays running tor 0.2.6.3 or later report bandwidth histories for four-hour intervals rather than fifteen minutes. But our most detailed bandwidth graphs use a fifteen-minute interval and don't handle the reported four-hours intervals very well. The fix is to … [View More]exclude reported data points if the reported interval is longer than the graphed interval. These data points will be included in another graph covering a longer time period, which at the same time uses a longer interval for graphed data points. Fixes #14984. --- .../torproject/onionoo/writer/BandwidthDocumentWriter.java | 12 ++++++++++-- web/protocol.html | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 9bad965..551fc71 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -107,6 +107,12 @@ public class BandwidthDocumentWriter implements DocumentWriter { if (endMillis < intervalStartMillis) { continue; } + if (endMillis - startMillis > dataPointInterval) { + /* This history interval is too long for this graph's data point + * interval. Maybe the next graph will contain it, but not this + * one. */ + continue; + } while ((intervalStartMillis / dataPointInterval) != (endMillis / dataPointInterval)) { dataPoints.add(totalMillis * 5L < dataPointInterval @@ -142,11 +148,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { long firstDataPointMillis = (((this.now - graphInterval) / dataPointInterval) + firstNonNullIndex) * dataPointInterval + dataPointInterval / 2L; - if (i > 0 && + if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= this.now - graphIntervals[i - 1]) { /* Skip bandwidth history object, because it doesn't contain * anything new that wasn't already contained in the last - * bandwidth history object(s). */ + * bandwidth history object(s). Unless we did not include any of + * the previous bandwidth history objects for other reasons, in + * which case we should include this one. */ continue; } long lastDataPointMillis = firstDataPointMillis diff --git a/web/protocol.html b/web/protocol.html index f19d0d8..2a13be8 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -1613,6 +1613,8 @@ when the bandwidth document was published. A graph history object is only contained if the time period it covers is not already contained in another graph history object with shorter time period and higher data resolution. +Similarly, a graph history object is excluded if the relay did not provide +bandwidth histories on the required level of detail. The unit is bytes per second. Contained graph history objects may contain null values if the relay did not provide any bandwidth data or only data for less than 20% of a given [View Less]

1 0

[onionoo/master] Raise version to 2.2.1 for #14984 fix.
by karsten＠torproject.org 25 Feb '15

25 Feb '15

commit 9b45b309d9bd504ac8f4402c9c80d76db29adfc2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 23 18:13:28 2015 +0100 Raise version to 2.2.1 for #14984 fix. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index fbb3837..a3f105e 100644 --- a/build.xml +++ b/build.xml @@ -2,7 +2,7 @@ <property name="onionoo.protocol.version" value="2.2"/> <property name="release.version" - value=… [View More]

1 0

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 25 Feb '15

25 Feb '15

commit ccbc28ccf377a9cca3469413bede00a1c9325fd8 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Feb 25 13:45:43 2015 +0000 Update translations for tails-misc_completed --- hr_HR.po | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hr_HR.po b/hr_HR.po index 4c90f6c..3f7ac5a 100644 --- a/hr_HR.po +++ b/hr_HR.po @@ -11,8 +11,8 @@ msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" -"POT-… [View More]

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 25 Feb '15

25 Feb '15

commit fa007bb2cc323aade14ecf8f5dbd7bc76aa2c61f Author: Translation commit bot <translation(a)torproject.org> Date: Wed Feb 25 13:45:39 2015 +0000 Update translations for tails-misc --- hr_HR.po | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hr_HR.po b/hr_HR.po index 6ca8a6d..3f7ac5a 100644 --- a/hr_HR.po +++ b/hr_HR.po @@ -12,8 +12,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2015-02-23 16:… [View More]

1 0

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 25 Feb '15

25 Feb '15

commit 1b31499c748cd23f4be7c855abf3e18632d1c39f Author: Translation commit bot <translation(a)torproject.org> Date: Wed Feb 25 13:15:48 2015 +0000 Update translations for tails-misc_completed --- tr.po | 2 +- zh_CN.po | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tr.po b/tr.po index 0802484..6cbfadc 100644 --- a/tr.po +++ b/tr.po @@ -17,7 +17,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: … [View More]

1 0