October 2017 - tor-commits - lists.torproject.org

[collector/release] Prepare for 1.4.1 release.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 56a303ecdb9c7f12fe117731a5511667e9763d31 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:51:30 2017 +0200 Prepare for 1.4.1 release. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index c4f4bd0..fb87709 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.4.0-dev" /> + <property name="release.version" value="1.4.1" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.1.1" />

1 0

[collector/release] Handle bridge descriptors with unusual line order.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit a38f7c277142e5300f3d7f43567b38706d727010 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 17:29:40 2017 +0200 Handle bridge descriptors with unusual line order. Typically, the "published" line appears before the "fingerprint" line. However, an alternative Tor implementation orders these two lines differently, which is valid due to the spec. We need to handle this case by accepting lines in either order. Fixes #23981. --- .../bridgedescs/SanitizedBridgesWriter.java | 77 +++++++++++++++++----- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index f53a3d1..441eb01 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -679,6 +679,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { new String(data, "US-ASCII"))); StringBuilder scrubbed = new StringBuilder(); String line = null; + byte[] fingerprintBytes = null; String hashedBridgeIdentity = null; String address = null; String routerLine = null; @@ -734,6 +735,20 @@ public class SanitizedBridgesWriter extends CollecTorMain { this.haveWarnedAboutInterval = true; } } + if (null != fingerprintBytes) { + /* We have read both published and fingerprint lines that we need to + * scrub the bridge's address which we might need to scrub reject + * lines. */ + try { + scrubbedAddress = scrubIpv4Address(address, fingerprintBytes, + published); + } catch (IOException e) { + /* There's a persistence problem, so we shouldn't scrub more + * IP addresses in this execution. */ + this.persistenceProblemWithSecrets = true; + return; + } + } scrubbed.append(line + "\n"); /* Parse the fingerprint to determine the hashed bridge @@ -743,13 +758,51 @@ public class SanitizedBridgesWriter extends CollecTorMain { String fingerprint = line.substring(line.startsWith("opt ") ? "opt fingerprint".length() : "fingerprint".length()) .replaceAll(" ", "").toLowerCase(); - byte[] fingerprintBytes = Hex.decodeHex( - fingerprint.toCharArray()); + fingerprintBytes = Hex.decodeHex(fingerprint.toCharArray()); hashedBridgeIdentity = DigestUtils.sha1Hex(fingerprintBytes) .toLowerCase(); + if (null != published) { + /* We have read both published and fingerprint lines that we need to + * scrub the bridge's address which we might need to scrub reject + * lines. */ + try { + scrubbedAddress = scrubIpv4Address(address, fingerprintBytes, + published); + } catch (IOException e) { + /* There's a persistence problem, so we shouldn't scrub more + * IP addresses in this execution. */ + this.persistenceProblemWithSecrets = true; + return; + } + } + scrubbed.append((line.startsWith("opt ") ? "opt " : "") + + "fingerprint"); + for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) { + scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i, + 4 * (i + 1)).toUpperCase()); + } + scrubbed.append("\n"); + + /* Replace the contact line (if present) with a generic one. */ + } else if (line.startsWith("contact ")) { + scrubbed.append("contact somebody\n"); + + /* When we reach the signature, we're done. Write the sanitized + * descriptor to disk below. */ + } else if (line.startsWith("router-signature")) { + + /* Write the scrubbed "router" line now based on the "router", + * "fingerprint", and "published" lines that we read before. Also + * scrub any "or-address" lines. */ + if (null == routerLine || null == fingerprintBytes + || null == published) { + logger.warn("Missing either of the following lines that are " + + "required to sanitize this server bridge descriptor: " + + "\"router\", \"fingerprint\", \"published\". Skipping " + + "descriptor."); + return; + } try { - scrubbedAddress = scrubIpv4Address(address, fingerprintBytes, - published); if (orAddresses != null) { scrubbedOrAddresses = new ArrayList<>(); for (String orAddress : orAddresses) { @@ -781,21 +834,9 @@ public class SanitizedBridgesWriter extends CollecTorMain { this.persistenceProblemWithSecrets = true; return; } - scrubbed.append((line.startsWith("opt ") ? "opt " : "") - + "fingerprint"); - for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) { - scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i, - 4 * (i + 1)).toUpperCase()); - } - scrubbed.append("\n"); - /* Replace the contact line (if present) with a generic one. */ - } else if (line.startsWith("contact ")) { - scrubbed.append("contact somebody\n"); - - /* When we reach the signature, we're done. Write the sanitized - * descriptor to disk below. */ - } else if (line.startsWith("router-signature")) { + /* Put together the scrubbed descriptor from "router" to the newline + * before the original "router-signature" line. */ scrubbedDesc = scrubbedRouterLine; if (scrubbedOrAddresses != null) { for (String scrubbedOrAddress : scrubbedOrAddresses) {

1 0

[collector/release] Add change log entries for the two #23981 changes.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 7cfb69ad036052d0d62a2c64efcd01df8bcb5145 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:50:29 2017 +0200 Add change log entries for the two #23981 changes. --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b12b33b..c924f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# Changes in version 1.4.1 - 2017-10-26 + + * Medium changes + - Handle bridge descriptors with an unusual order of "published" + and "fingerprint" lines. + - Retain "bridge-distribution-request" lines when sanitizing + descriptors. + + # Changes in version 1.4.0 - 2017-10-09 * Major changes

1 0

[collector/release] Bump version to 1.4.0-dev.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit ebb13b12d0ae492ef7f30638c4aaea29cea3af45 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Oct 17 21:22:04 2017 +0200 Bump version to 1.4.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 7cbff44..c4f4bd0 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.4.0" /> + <property name="release.version" value="1.4.0-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.1.1" />

1 0

[collector/release] Add test that will fail #23981.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 3a95892de3bb4728a45812b2cc2ed9d24a7de33e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 16:45:38 2017 +0200 Add test that will fail #23981. --- .../collector/bridgedescs/SanitizedBridgesWriterTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java index 0e62573..3afe9cb 100644 --- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -440,6 +440,20 @@ public class SanitizedBridgesWriterTest { } @Test + public void testServerDescriptorPublishedLineAfterFingerprintLine() + throws Exception { + this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(), + "true"); + String fingerprintLine = + "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B"; + this.defaultServerDescriptorBuilder.removeLine(fingerprintLine); + this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith( + "published ", Arrays.asList(fingerprintLine)); + this.runTest(); + assertFalse(this.parsedServerDescriptors.isEmpty()); + } + + @Test public void testExtraInfoDescriptorDefault() throws Exception { this.runTest(); List<String> expectedLines = Arrays.asList(

1 0

[collector/release] Retain "bridge-distribution-request" lines.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit e54bca3f9588c3253d9f244d10819f82399b4539 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:49:18 2017 +0200 Retain "bridge-distribution-request" lines. --- .../org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 441eb01..12b42ac 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -946,7 +946,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { || line.equals("opt allow-single-hop-exits") || line.equals("allow-single-hop-exits") || line.startsWith("ipv6-policy ") - || line.equals("tunnelled-dir-server")) { + || line.equals("tunnelled-dir-server") + || line.startsWith("bridge-distribution-request ")) { scrubbed.append(line + "\n"); /* Replace node fingerprints in the family line with their hashes

1 0

[collector/release] Update metrics-base.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 3a0ba1baba1e77780803a7ed0559be122a6f01b6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:50:00 2017 +0200 Update metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 78e533f..4b34756 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 78e533fdc84e20769a8a155d1426cbea59c6e7cc +Subproject commit 4b34756ddd71ccaf0fc30e5f5bf0a813a297d4a3

1 0

[collector/master] Add test that will fail #23981.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 3a95892de3bb4728a45812b2cc2ed9d24a7de33e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 16:45:38 2017 +0200 Add test that will fail #23981. --- .../collector/bridgedescs/SanitizedBridgesWriterTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java index 0e62573..3afe9cb 100644 --- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -440,6 +440,20 @@ public class SanitizedBridgesWriterTest { } @Test + public void testServerDescriptorPublishedLineAfterFingerprintLine() + throws Exception { + this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(), + "true"); + String fingerprintLine = + "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B"; + this.defaultServerDescriptorBuilder.removeLine(fingerprintLine); + this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith( + "published ", Arrays.asList(fingerprintLine)); + this.runTest(); + assertFalse(this.parsedServerDescriptors.isEmpty()); + } + + @Test public void testExtraInfoDescriptorDefault() throws Exception { this.runTest(); List<String> expectedLines = Arrays.asList(

1 0

[collector/master] Add change log entries for the two #23981 changes.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit 7cfb69ad036052d0d62a2c64efcd01df8bcb5145 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:50:29 2017 +0200 Add change log entries for the two #23981 changes. --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b12b33b..c924f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# Changes in version 1.4.1 - 2017-10-26 + + * Medium changes + - Handle bridge descriptors with an unusual order of "published" + and "fingerprint" lines. + - Retain "bridge-distribution-request" lines when sanitizing + descriptors. + + # Changes in version 1.4.0 - 2017-10-09 * Major changes

1 0

[collector/master] Retain "bridge-distribution-request" lines.
by karsten＠torproject.org 26 Oct '17

26 Oct '17

commit e54bca3f9588c3253d9f244d10819f82399b4539 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 25 20:49:18 2017 +0200 Retain "bridge-distribution-request" lines. --- .../org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 441eb01..12b42ac 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -946,7 +946,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { || line.equals("opt allow-single-hop-exits") || line.equals("allow-single-hop-exits") || line.startsWith("ipv6-policy ") - || line.equals("tunnelled-dir-server")) { + || line.equals("tunnelled-dir-server") + || line.startsWith("bridge-distribution-request ")) { scrubbed.append(line + "\n"); /* Replace node fingerprints in the family line with their hashes

1 0