June 2017 - tor-commits - lists.torproject.org

[tor/maint-0.3.1] Try a little harder to make sure we never call tor_compress_process wrong.
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit b546d8bc2bfa0e20bdd14181a29a91ba31df2f0e Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jun 26 09:39:59 2017 -0400 Try a little harder to make sure we never call tor_compress_process wrong. --- src/or/buffers.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 3692ed4..12a6c02 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2092,7 +2092,7 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, - int done) + const int done) { char *next; size_t old_avail, avail; @@ -2114,8 +2114,10 @@ write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, case TOR_COMPRESS_ERROR: return -1; case TOR_COMPRESS_OK: - if (data_len == 0) + if (data_len == 0) { + tor_assert_nonfatal(!done); over = 1; + } break; case TOR_COMPRESS_BUFFER_FULL: if (avail) { @@ -2124,6 +2126,11 @@ write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, * whether were going to or not. */ need_new_chunk = 1; } + if (data_len == 0 && !done) { + /* We've consumed all the input data, though, so there's no + * point in forging ahead right now. */ + over = 1; + } break; } buf->datalen += old_avail - avail;

1 0

[tor/maint-0.3.1] Merge branch 'bug22719_031' into maint-0.3.1
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit fd16dd2608ba3a1f97fb13bf7ae3b6640e90376a Merge: 3259601 50be460 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:31:33 2017 -0400 Merge branch 'bug22719_031' into maint-0.3.1 changes/bug22719 | 7 +++++++ src/or/buffers.c | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-)

1 0

[tor/master] changes file for bug 22719
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit 50be4600ae14687b9f4901b4d38a5dce644f5648 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:31:03 2017 -0400 changes file for bug 22719 --- changes/bug22719 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changes/bug22719 b/changes/bug22719 new file mode 100644 index 0000000..bfcda0a --- /dev/null +++ b/changes/bug22719 @@ -0,0 +1,7 @@ + o Minor bugfixes (compression): + - When spooling compressed data to an output buffer, don't try to + spool more data when there is no more data to spool and we are + not trying to flush the input. Previously, we would sometimes + launch compression requests with nothing to do, which interferes + with our 22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha. +

1 0

[tor/master] Merge branch 'bug22719_031' into maint-0.3.1
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit fd16dd2608ba3a1f97fb13bf7ae3b6640e90376a Merge: 3259601 50be460 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:31:33 2017 -0400 Merge branch 'bug22719_031' into maint-0.3.1 changes/bug22719 | 7 +++++++ src/or/buffers.c | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.3.1'
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit 22f441d4ee40a6d1c9615aeb2b08b872549abfd0 Merge: f6df433 fd16dd2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:32:50 2017 -0400 Merge branch 'maint-0.3.1' changes/bug22719 | 7 +++++++ src/or/buffers.c | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-)

1 0

[tor/release-0.3.1] changes file for bug 22719
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit 50be4600ae14687b9f4901b4d38a5dce644f5648 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:31:03 2017 -0400 changes file for bug 22719 --- changes/bug22719 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changes/bug22719 b/changes/bug22719 new file mode 100644 index 0000000..bfcda0a --- /dev/null +++ b/changes/bug22719 @@ -0,0 +1,7 @@ + o Minor bugfixes (compression): + - When spooling compressed data to an output buffer, don't try to + spool more data when there is no more data to spool and we are + not trying to flush the input. Previously, we would sometimes + launch compression requests with nothing to do, which interferes + with our 22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha. +

1 0

[tor/release-0.3.1] Try a little harder to make sure we never call tor_compress_process wrong.
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit b546d8bc2bfa0e20bdd14181a29a91ba31df2f0e Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jun 26 09:39:59 2017 -0400 Try a little harder to make sure we never call tor_compress_process wrong. --- src/or/buffers.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 3692ed4..12a6c02 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2092,7 +2092,7 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, - int done) + const int done) { char *next; size_t old_avail, avail; @@ -2114,8 +2114,10 @@ write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, case TOR_COMPRESS_ERROR: return -1; case TOR_COMPRESS_OK: - if (data_len == 0) + if (data_len == 0) { + tor_assert_nonfatal(!done); over = 1; + } break; case TOR_COMPRESS_BUFFER_FULL: if (avail) { @@ -2124,6 +2126,11 @@ write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, * whether were going to or not. */ need_new_chunk = 1; } + if (data_len == 0 && !done) { + /* We've consumed all the input data, though, so there's no + * point in forging ahead right now. */ + over = 1; + } break; } buf->datalen += old_avail - avail;

1 0

[tor/release-0.3.1] Merge branch 'bug22719_031' into maint-0.3.1
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit fd16dd2608ba3a1f97fb13bf7ae3b6640e90376a Merge: 3259601 50be460 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:31:33 2017 -0400 Merge branch 'bug22719_031' into maint-0.3.1 changes/bug22719 | 7 +++++++ src/or/buffers.c | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-)

1 0

[metrics-web/master] Tweak solution diffs: w/o the path 'src/' the patches can be more easily copied
by karsten＠torproject.org 27 Jun '17

27 Jun '17

commit e9be3f36c35aed29e731568d50556ee630db0ed0 Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Jun 22 13:44:10 2017 +0000 Tweak solution diffs: w/o the path 'src/' the patches can be more easily copied and directly used as patch files avoiding the patch dir level command line option. And, the true location in metrics-lib is different from src, i.e., src/main/resources/examples. Also modernized the result examples. Final touches to task-22295. --- .../src/main/resources/web/WEB-INF/metrics-lib.jsp | 47 +++++++++++----------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp index 4794235..7d17734 100644 --- a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp +++ b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp @@ -188,14 +188,15 @@ java -cp .:lib/\*:generated/dist/signed/\* ConsensusWeightByVersion There will be some log statements, and the final output should now contain lines like the following: <pre> -0.2.4 -> 4.2% +0.2.4 -> 3.2% 0.2.5 -> 9.4% -0.2.6 -> 6.2% -0.2.7 -> 8.3% -0.2.8 -> 15.0% -0.2.9 -> 46.7% -0.3.0 -> 9.4% -0.3.1 -> 0.8% +0.2.6 -> 3.2% +0.2.7 -> 7.3% +0.2.8 -> 6.4% +0.2.9 -> 48.2% +0.3.0 -> 20.8% +0.3.1 -> 1.2% +0.3.2 -> 0.3% </pre> These are the numbers we were looking for. Now you should know what to do to extract interesting data from consensuses. Want to give that another try and filter relays with the <code>Exit</code> flag to learn about exit capacity by Tor version? Hint: You'll want to check for <code>entry.getFlags().contains("Exit")</code>. Of course, you could as well continue with the next tutorial below. (Or you could scroll down to the bottom of this page to see the diff.) @@ -278,15 +279,15 @@ java -cp .:lib/\*:generated/dist/signed/\* PluggableTransports The output should contain lines like the following: <pre> - fte -> 11.7% + fte -> 2.3% meek -> 0.2% - obfs2 -> 0.8% - obfs3 -> 35.7% + obfs2 -> 0.7% + obfs3 -> 20.8% obfs3_websocket -> 0.0% - obfs4 -> 72.8% - scramblesuit -> 29.6% - snowflake -> 0.0% - websocket -> 0.8% + obfs4 -> 77.0% + scramblesuit -> 17.3% + snowflake -> 0.1% + websocket -> 0.7% </pre> As above, we'll leave it up to you to further expand this code. For example, how does the result change if you count transport combinations rather than transports? Hint: you won't need anything else from metrics-lib, but you'll need to add some code to order transport names and write them to a string. (And if you'd rather look up the solution, scroll down a bit to see the diff.) @@ -301,9 +302,9 @@ java -cp .:lib/\*:generated/dist/signed/\* PluggableTransports Scrolled down just to see where we're hiding the solutions of the three little riddles above? Here are the diffs: -<pre><code class="language-diff">diff -Nur src/DownloadConsensuses.java src/DownloadConsensuses.java ---- src/DownloadConsensuses.java 2017-03-07 17:48:35.000000000 +0100 -+++ src/DownloadConsensuses.java 2017-03-10 23:02:51.000000000 +0100 +<pre><code class="language-diff">diff -Nur DownloadConsensuses.java DownloadConsensuses.java +--- DownloadConsensuses.java 2017-03-07 17:48:35.000000000 +0100 ++++ DownloadConsensuses.java 2017-03-10 23:02:51.000000000 +0100 @@ -11,7 +11,7 @@ // Download from Tor's main CollecTor instance, "https://collector.torproject.org", @@ -315,9 +316,9 @@ java -cp .:lib/\*:generated/dist/signed/\* PluggableTransports // write to the local directory called descriptors/, </code></pre> -<pre><code class="language-diff">diff -Nur src/ConsensusWeightByVersion.java src/ConsensusWeightByVersion.java ---- src/ConsensusWeightByVersion.java 2017-03-10 23:00:40.000000000 +0100 -+++ src/ConsensusWeightByVersion.java 2017-03-10 23:03:18.000000000 +0100 +<pre><code class="language-diff">diff -Nur ConsensusWeightByVersion.java ConsensusWeightByVersion.java +--- ConsensusWeightByVersion.java 2017-03-10 23:00:40.000000000 +0100 ++++ ConsensusWeightByVersion.java 2017-03-10 23:03:18.000000000 +0100 @@ -25,6 +25,9 @@ } RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; @@ -330,9 +331,9 @@ java -cp .:lib/\*:generated/dist/signed/\* PluggableTransports // We're only interested in a.b.c type versions for this example. </code></pre> -<pre><code class="language-diff">diff -Nur src/PluggableTransports.java src/PluggableTransports.java ---- src/PluggableTransports.java 2017-03-10 23:01:43.000000000 +0100 -+++ src/PluggableTransports.java 2017-03-10 23:03:43.000000000 +0100 +<pre><code class="language-diff">diff -Nur PluggableTransports.java PluggableTransports.java +--- PluggableTransports.java 2017-03-10 23:01:43.000000000 +0100 ++++ PluggableTransports.java 2017-03-10 23:03:43.000000000 +0100 @@ -20,12 +22,11 @@ BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; String fingerprint = extraInfo.getFingerprint();

1 0

[metrics-web/master] Update metrics-lib tutorials for version 2.0.0.
by karsten＠torproject.org 27 Jun '17

27 Jun '17

commit c2839918b2942d216e0d4de568affe968663a35b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jun 22 15:44:09 2017 +0200 Update metrics-lib tutorials for version 2.0.0. Implements #22295. --- .../src/main/resources/web/WEB-INF/metrics-lib.jsp | 126 +++++++++------------ 1 file changed, 56 insertions(+), 70 deletions(-) diff --git a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp index 3c74c9a..4794235 100644 --- a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp +++ b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp @@ -140,35 +140,26 @@ public class ConsensusWeightByVersion { // Read descriptors from disk. DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - - // Add the directory with descriptors to the descriptor reader. - descriptorReader.addDirectory(new File("descriptors/recent/relay-descriptors/consensuses")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { // Iterate over all descriptor files found. - DescriptorFile descriptorFile = descriptorFiles.next(); - - // Now, iterate over the descriptors contained in the file. - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof RelayNetworkStatusConsensus)) { - // We're only interested in consensuses. + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/relay-descriptors/consensuses"))) { + if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + // We're only interested in consensuses. + continue; + } + RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { + String version = entry.getVersion(); + if (!version.startsWith("Tor ") || version.length() < 9) { + // We're only interested in a.b.c type versions for this example. continue; } - RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; - for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { - String version = entry.getVersion(); - if (!version.startsWith("Tor ") || version.length() < 9) { - // We're only interested in a.b.c type versions for this example. - continue; - } - // Remove the 'Tor ' prefix and anything starting at the patch level. - version = version.substring(4, 9); - long bandwidth = entry.getBandwidth(); - totalBandwidth += bandwidth; - if (bandwidthByVersion.containsKey(version)) { - bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); - } else { - bandwidthByVersion.put(version, bandwidth); - } + // Remove the 'Tor ' prefix and anything starting at the patch level. + version = version.substring(4, 9); + long bandwidth = entry.getBandwidth(); + totalBandwidth += bandwidth; + if (bandwidthByVersion.containsKey(version)) { + bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); + } else { + bandwidthByVersion.put(version, bandwidth); } } } @@ -246,23 +237,18 @@ public class PluggableTransports { SortedMap<String, Integer> countedTransports = new TreeMap<>(); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(new File("descriptors/recent/bridge-descriptors/extra-infos")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { - continue; - } - BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; - String fingerprint = extraInfo.getFingerprint(); - if (observedFingerprints.add(fingerprint)) { - for (String transport : extraInfo.getTransports()) { - if (countedTransports.containsKey(transport)) { - countedTransports.put(transport, 1 + countedTransports.get(transport)); - } else { - countedTransports.put(transport, 1); - } + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/bridge-descriptors/extra-infos"))) { + if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { + continue; + } + BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; + String fingerprint = extraInfo.getFingerprint(); + if (observedFingerprints.add(fingerprint)) { + for (String transport : extraInfo.getTransports()) { + if (countedTransports.containsKey(transport)) { + countedTransports.put(transport, 1 + countedTransports.get(transport)); + } else { + countedTransports.put(transport, 1); } } } @@ -332,39 +318,39 @@ java -cp .:lib/\*:generated/dist/signed/\* PluggableTransports <pre><code class="language-diff">diff -Nur src/ConsensusWeightByVersion.java src/ConsensusWeightByVersion.java --- src/ConsensusWeightByVersion.java 2017-03-10 23:00:40.000000000 +0100 +++ src/ConsensusWeightByVersion.java 2017-03-10 23:03:18.000000000 +0100 -@@ -31,6 +31,9 @@ - } - RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; - for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { -+ if (!entry.getFlags().contains("Exit")) { -+ continue; -+ } - String version = entry.getVersion(); - if (!version.startsWith("Tor ") || version.length() < 9) { - // We're only interested in a.b.c type versions for this example. +@@ -25,6 +25,9 @@ + } + RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { ++ if (!entry.getFlags().contains("Exit")) { ++ continue; ++ } + String version = entry.getVersion(); + if (!version.startsWith("Tor ") || version.length() < 9) { + // We're only interested in a.b.c type versions for this example. </code></pre> <pre><code class="language-diff">diff -Nur src/PluggableTransports.java src/PluggableTransports.java --- src/PluggableTransports.java 2017-03-10 23:01:43.000000000 +0100 +++ src/PluggableTransports.java 2017-03-10 23:03:43.000000000 +0100 -@@ -24,12 +24,11 @@ - BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; - String fingerprint = extraInfo.getFingerprint(); - if (observedFingerprints.add(fingerprint)) { -- for (String transport : extraInfo.getTransports()) { -- if (countedTransports.containsKey(transport)) { -- countedTransports.put(transport, 1 + countedTransports.get(transport)); -- } else { -- countedTransports.put(transport, 1); -- } -+ String transports = new TreeSet<>(extraInfo.getTransports()).toString(); -+ if (countedTransports.containsKey(transports)) { -+ countedTransports.put(transports, 1 + countedTransports.get(transports)); -+ } else { -+ countedTransports.put(transports, 1); - } +@@ -20,12 +22,11 @@ + BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; + String fingerprint = extraInfo.getFingerprint(); + if (observedFingerprints.add(fingerprint)) { +- for (String transport : extraInfo.getTransports()) { +- if (countedTransports.containsKey(transport)) { +- countedTransports.put(transport, 1 + countedTransports.get(transport)); +- } else { +- countedTransports.put(transport, 1); +- } ++ String transports = new TreeSet<>(extraInfo.getTransports()).toString(); ++ if (countedTransports.containsKey(transports)) { ++ countedTransports.put(transports, 1 + countedTransports.get(transports)); ++ } else { ++ countedTransports.put(transports, 1); } } + } </code></pre> </div>

1 0