April 2016 - tor-commits - lists.torproject.org

[metrics-lib/master] Recognize Torperf DATEPERCx fields for 0 <= x <= 100.
by karsten＠torproject.org 26 Apr '16

26 Apr '16

commit a62f7cd45013bb237c18175f2d66092c6ba9991e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 15 17:04:52 2016 +0200 Recognize Torperf DATEPERCx fields for 0 <= x <= 100. And don't break if x is non-numeric. Spotted by iwakeh. --- CHANGELOG.md | 2 ++ src/org/torproject/descriptor/TorperfResult.java | 6 ++-- .../descriptor/impl/TorperfResultImpl.java | 32 ++++++++++------------ .../descriptor/impl/TorperfResultImplTest.java | 13 ++++++++- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b9b9a..7d2a246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Also accept \r\n as newline in Torperf results files. - Make unrecognized keys of Torperf results available together with the corresponding values, rather than just the whole line. + - In Torperf results, recognize all percentiles of expected bytes + read for 0 <= x <= 100 rather than just x = { 10, 20, ..., 90 }. * Minor changes - Include a Torperf results line with more than one unrecognized diff --git a/src/org/torproject/descriptor/TorperfResult.java b/src/org/torproject/descriptor/TorperfResult.java index 0c75cb2..9c47a61 100644 --- a/src/org/torproject/descriptor/TorperfResult.java +++ b/src/org/torproject/descriptor/TorperfResult.java @@ -55,9 +55,9 @@ public interface TorperfResult extends Descriptor { * if the torperf line didn't contain that information. */ public Boolean didTimeout(); - /* Return the times when x% of expected bytes were read for x = { 10, - * 20, 30, 40, 50, 60, 70, 80, 90 }, or null if the torperf line didn't - * contain that information. */ + /* Return the times when x% of expected bytes were read for + * 0 <= x <= 100, or null if the torperf line didn't contain that + * information. */ public SortedMap<Integer, Long> getDataPercentiles(); /* Return the time when the circuit was launched, or -1 if the torperf diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index 94e756e..62cf68d 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -275,22 +275,28 @@ public class TorperfResultImpl extends DescriptorImpl } } - private Set<String> unparsedPercentiles = new HashSet<>( - Arrays.asList("10,20,30,40,50,60,70,80,90".split(","))); private void parseDataPercentile(String value, String keyValue, String line) throws DescriptorParseException { String key = keyValue.substring(0, keyValue.indexOf("=")); String percentileString = key.substring("DATAPERC".length()); - if (!this.unparsedPercentiles.contains(percentileString)) { + int percentile = -1; + try { + percentile = Integer.parseInt(percentileString); + } catch (NumberFormatException e) { + /* Treat key as unrecognized below. */ + percentile = -1; + } + if (percentile < 0 || percentile > 100) { if (this.unrecognizedKeys == null) { this.unrecognizedKeys = new TreeMap<>(); } this.unrecognizedKeys.put(key, value); } else { - this.unparsedPercentiles.remove(percentileString); - int decileIndex = (Integer.parseInt(percentileString) / 10) - 1; long timestamp = this.parseTimestamp(value, keyValue, line); - this.dataDeciles[decileIndex] = timestamp; + if (this.dataPercentiles == null) { + this.dataPercentiles = new TreeMap<>(); + } + this.dataPercentiles.put(percentile, timestamp); } } @@ -466,18 +472,10 @@ public class TorperfResultImpl extends DescriptorImpl return this.didTimeout; } - private Long[] dataDeciles = new Long[9]; + private SortedMap<Integer, Long> dataPercentiles; public SortedMap<Integer, Long> getDataPercentiles() { - if (this.dataDeciles == null) { - return null; - } - SortedMap<Integer, Long> result = new TreeMap<>(); - for (int i = 0; i < dataDeciles.length; i++) { - if (dataDeciles[i] > 0L) { - result.put(10 * (i + 1), dataDeciles[i]); - } - } - return result; + return this.dataPercentiles == null ? null + : new TreeMap<>(this.dataPercentiles); } private long launchMillis = -1L; diff --git a/test/org/torproject/descriptor/impl/TorperfResultImplTest.java b/test/org/torproject/descriptor/impl/TorperfResultImplTest.java index be9221f..b5cde0a 100644 --- a/test/org/torproject/descriptor/impl/TorperfResultImplTest.java +++ b/test/org/torproject/descriptor/impl/TorperfResultImplTest.java @@ -44,7 +44,7 @@ public class TorperfResultImplTest { } @Test() - public void testAllAnnotatedInput() throws Exception{ + public void testAllAnnotatedInput() throws Exception { byte[] asciiBytes = (torperfAnnotation + input + torperfAnnotation + input + torperfAnnotation + input).getBytes("US-ASCII"); @@ -82,5 +82,16 @@ public class TorperfResultImplTest { + "READBYTES=51416 REQUEST=1441065601.86 RESPONSE=1441065602.38 " + "SOCKET=1441065601.86 SOURCE=moria START=1441065601.86 " + "TIMEOUT=1500 USED_AT=1441065603.40 USED_BY=2475 WRITEBYTES=75\n"; + + @Test() + public void testDatapercNonNumeric() throws Exception { + List<Descriptor> result = TorperfResultImpl.parseTorperfResults( + ("DATAPERMILLE=2.0 " + input).getBytes(), false); + assertEquals(1, result.size()); + TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0); + assertEquals(1, torperfResult.getUnrecognizedKeys().size()); + assertEquals("DATAPERMILLE", + torperfResult.getUnrecognizedKeys().firstKey()); + } }

1 0

[metrics-lib/master] Make unrecognized keys of Torperf results available.
by karsten＠torproject.org 26 Apr '16

26 Apr '16

commit 45ad8a0d2294242ad4e3f98f29f8f17ec2a588cb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 15 16:58:06 2016 +0200 Make unrecognized keys of Torperf results available. --- CHANGELOG.md | 2 ++ src/org/torproject/descriptor/TorperfResult.java | 6 ++++- .../descriptor/impl/TorperfResultImpl.java | 29 ++++++++++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863bffd..d5b9b9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Include the hostname in directory source entries of consensuses and votes. - Also accept \r\n as newline in Torperf results files. + - Make unrecognized keys of Torperf results available together with + the corresponding values, rather than just the whole line. * Minor changes - Include a Torperf results line with more than one unrecognized diff --git a/src/org/torproject/descriptor/TorperfResult.java b/src/org/torproject/descriptor/TorperfResult.java index 016a043..0c75cb2 100644 --- a/src/org/torproject/descriptor/TorperfResult.java +++ b/src/org/torproject/descriptor/TorperfResult.java @@ -1,4 +1,4 @@ -/* Copyright 2012--2015 The Tor Project +/* Copyright 2012--2016 The Tor Project * See LICENSE for licensing information */ package org.torproject.descriptor; @@ -7,6 +7,10 @@ import java.util.SortedMap; public interface TorperfResult extends Descriptor { + /* Return all unrecognized keys together with their values, or null if + * all keys were recognized. */ + public SortedMap<String, String> getUnrecognizedKeys(); + /* Return the configured name of the data source. */ public String getSource(); diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index b162b18..94e756e 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -153,6 +153,10 @@ public class TorperfResultImpl extends DescriptorImpl throw new DescriptorParseException("Unrecognized key '" + key + "' in line '" + line + "'."); } else { + if (this.unrecognizedKeys == null) { + this.unrecognizedKeys = new TreeMap<>(); + } + this.unrecognizedKeys.put(key, value); if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); } @@ -275,16 +279,19 @@ public class TorperfResultImpl extends DescriptorImpl Arrays.asList("10,20,30,40,50,60,70,80,90".split(","))); private void parseDataPercentile(String value, String keyValue, String line) throws DescriptorParseException { - String percentileString = keyValue.substring("DATAPERC".length(), - keyValue.indexOf("=")); + String key = keyValue.substring(0, keyValue.indexOf("=")); + String percentileString = key.substring("DATAPERC".length()); if (!this.unparsedPercentiles.contains(percentileString)) { - throw new DescriptorParseException("Illegal value in '" + keyValue - + "' in line '" + line + "'."); + if (this.unrecognizedKeys == null) { + this.unrecognizedKeys = new TreeMap<>(); + } + this.unrecognizedKeys.put(key, value); + } else { + this.unparsedPercentiles.remove(percentileString); + int decileIndex = (Integer.parseInt(percentileString) / 10) - 1; + long timestamp = this.parseTimestamp(value, keyValue, line); + this.dataDeciles[decileIndex] = timestamp; } - this.unparsedPercentiles.remove(percentileString); - int decileIndex = (Integer.parseInt(percentileString) / 10) - 1; - long timestamp = this.parseTimestamp(value, keyValue, line); - this.dataDeciles[decileIndex] = timestamp; } private void parseLaunch(String value, String keyValue, String line) @@ -383,6 +390,12 @@ public class TorperfResultImpl extends DescriptorImpl } } + private SortedMap<String, String> unrecognizedKeys; + public SortedMap<String, String> getUnrecognizedKeys() { + return this.unrecognizedKeys == null ? null + : new TreeMap<>(this.unrecognizedKeys); + } + private String source; public String getSource() { return this.source;

1 0

[metrics-lib/master] Handle Torperf lines with unrecognized keys correctly.
by karsten＠torproject.org 26 Apr '16

26 Apr '16

commit 6f561bef6648b2a1b1aea216508f60b2174a7cab Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 15 16:43:59 2016 +0200 Handle Torperf lines with unrecognized keys correctly. Previously, we would have added as many copies of a Torperf line as it contains keys we don't recognize, which is not really usable. --- CHANGELOG.md | 4 ++++ src/org/torproject/descriptor/impl/TorperfResultImpl.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 091d35d..863bffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ and votes. - Also accept \r\n as newline in Torperf results files. + * Minor changes + - Include a Torperf results line with more than one unrecognized + key only once in the unrecognized lines. + # Changes in version 1.1.0 - 2015-12-28 diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index cefbe94..b162b18 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -156,7 +156,9 @@ public class TorperfResultImpl extends DescriptorImpl if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); } - this.unrecognizedLines.add(line); + if (!this.unrecognizedLines.contains(line)) { + this.unrecognizedLines.add(line); + } } } }

1 0

[metrics-lib/master] Also accept \r\n as newline in Torperf results files.
by karsten＠torproject.org 26 Apr '16

26 Apr '16

commit 0d6205af5dca0d652f1ee73025346ebf643831c7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 15 16:39:16 2016 +0200 Also accept \r\n as newline in Torperf results files. --- CHANGELOG.md | 1 + src/org/torproject/descriptor/impl/TorperfResultImpl.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d427621..091d35d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Medium changes - Include the hostname in directory source entries of consensuses and votes. + - Also accept \r\n as newline in Torperf results files. # Changes in version 1.1.0 - 2015-12-28 diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index 35eb356..cefbe94 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -26,7 +26,7 @@ public class TorperfResultImpl extends DescriptorImpl } List<Descriptor> parsedDescriptors = new ArrayList<>(); String descriptorString = new String(rawDescriptorBytes); - Scanner s = new Scanner(descriptorString).useDelimiter("\n"); + Scanner s = new Scanner(descriptorString).useDelimiter("\r?\n"); String typeAnnotation = ""; while (s.hasNext()) { String line = s.next();

1 0

[metrics-lib/master] Make some trivial tweaks to TorperfResultsImpl.
by karsten＠torproject.org 26 Apr '16

26 Apr '16

commit 9657de4a8fd7f55496277d58e311ea02e259c463 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 15 16:35:22 2016 +0200 Make some trivial tweaks to TorperfResultsImpl. --- src/org/torproject/descriptor/impl/TorperfResultImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java index 26e0e4a..35eb356 100644 --- a/src/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -1,4 +1,4 @@ -/* Copyright 2012--2015 The Tor Project +/* Copyright 2012--2016 The Tor Project * See LICENSE for licensing information */ package org.torproject.descriptor.impl; @@ -355,8 +355,8 @@ public class TorperfResultImpl extends DescriptorImpl } } if (timestamp < 0L) { - throw new DescriptorParseException("Illegal timestamp '" + value + "' in '" - + keyValue + "' in line '" + line + "'."); + throw new DescriptorParseException("Illegal timestamp '" + value + + "' in '" + keyValue + "' in line '" + line + "'."); } return timestamp; } @@ -367,7 +367,7 @@ public class TorperfResultImpl extends DescriptorImpl return Integer.parseInt(value); } catch (NumberFormatException e) { throw new DescriptorParseException("Illegal value in '" + keyValue - + " in line '" + line + "'."); + + "' in line '" + line + "'."); } }

1 0

[torspec/master] Merge remote-tracking branch 'yawning-schwanenleid/feature18685'
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 92ff8bac286432141843691be7a4d3a7cf4664b0 Merge: 1bd12a4 1632d6e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 26 13:41:44 2016 -0400 Merge remote-tracking branch 'yawning-schwanenleid/feature18685' control-spec.txt | 11 +++++++++++ 1 file changed, 11 insertions(+)

1 0

[torspec/master] Document the `HIBERNATION_STATUS` `SERVER_STATUS` sub-event.
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 1632d6e31013f54286a5b14e4409399efdd96f00 Author: Yawning Angel <yawning(a)schwanenlied.me> Date: Wed Mar 30 20:32:20 2016 +0000 Document the `HIBERNATION_STATUS` `SERVER_STATUS` sub-event. --- control-spec.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/control-spec.txt b/control-spec.txt index a3c1647..757ae58 100644 --- a/control-spec.txt +++ b/control-spec.txt @@ -2383,6 +2383,17 @@ {This event could affect the controller's idea of server status. The controller should warn the admin and suggest reasonable steps to take.} + HIBERNATION_STATUS + "STATUS=" "AWAKE" | "SOFT" | "HARD" + Our bandwidth based accounting status has changed, and we are now + relaying traffic/rejecting new connections/hibernating. + + {This event could affect the controller's idea of server status. The + controller MAY inform the admin, though presumably the accounting was + explicitly enabled for a reason.} + + [This event was added in tor 0.2.9.0-alpha.] + 4.1.11. Our set of guard nodes has changed Syntax:

1 0

[tor/master] mention 18685 in changes file
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit f197d189a6c310ee6725e3648ceabc1322dd24fc Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 26 13:40:21 2016 -0400 mention 18685 in changes file --- changes/feature18685 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/feature18685 b/changes/feature18685 index a6d3dcd..bc0d1be 100644 --- a/changes/feature18685 +++ b/changes/feature18685 @@ -1,3 +1,3 @@ o Minor features (controller): - Fire a `STATUS_SERVER` event whenever the hibernation status changes - between "awake"/"soft"/"hard". + between "awake"/"soft"/"hard". Closes ticket 18685.

1 0

[tor/master] Issue a `STATUS_SERVER` event on meaningful hibernation state changes.
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit a19f4192dabe62ab74e3bf3048fa4209bff595ce Author: Yawning Angel <yawning(a)schwanenlied.me> Date: Wed Mar 30 20:19:11 2016 +0000 Issue a `STATUS_SERVER` event on meaningful hibernation state changes. Implements feature #18685. --- changes/feature18685 | 3 +++ src/or/hibernate.c | 51 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/changes/feature18685 b/changes/feature18685 new file mode 100644 index 0000000..a6d3dcd --- /dev/null +++ b/changes/feature18685 @@ -0,0 +1,3 @@ + o Minor features (controller): + - Fire a `STATUS_SERVER` event whenever the hibernation status changes + between "awake"/"soft"/"hard". diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9408925..3666abb 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -28,6 +28,7 @@ hibernating, phase 2: #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "control.h" #include "hibernate.h" #include "main.h" #include "router.h" @@ -111,11 +112,34 @@ static int cfg_start_day = 0, cfg_start_min = 0; /** @} */ +static const char *hibernate_state_to_string(hibernate_state_t state); static void reset_accounting(time_t now); static int read_bandwidth_usage(void); static time_t start_of_accounting_period_after(time_t now); static time_t start_of_accounting_period_containing(time_t now); static void accounting_set_wakeup_time(void); +static void on_hibernate_state_change(hibernate_state_t prev_state); + +/** + * Return the human-readable name for the hibernation state <b>state</b> + */ +static const char * +hibernate_state_to_string(hibernate_state_t state) +{ + static char buf[64]; + switch (state) { + case HIBERNATE_STATE_EXITING: return "EXITING"; + case HIBERNATE_STATE_LOWBANDWIDTH: return "SOFT"; + case HIBERNATE_STATE_DORMANT: return "HARD"; + case HIBERNATE_STATE_INITIAL: + case HIBERNATE_STATE_LIVE: + return "AWAKE"; + default: + log_warn(LD_BUG, "unknown hibernate state %d", state); + tor_snprintf(buf, sizeof(buf), "unknown [%d]", state); + return buf; + } +} /* ************ * Functions for bandwidth accounting. @@ -935,6 +959,7 @@ consider_hibernation(time_t now) { int accounting_enabled = get_options()->AccountingMax != 0; char buf[ISO_TIME_LEN+1]; + hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval * elapses. */ @@ -990,6 +1015,10 @@ consider_hibernation(time_t now) hibernate_end_time_elapsed(now); } } + + /* Dispatch a controller event if the hibernation state changed. */ + if (hibernate_state != prev_state) + on_hibernate_state_change(prev_state); } /** Helper function: called when we get a GETINFO request for an @@ -1007,12 +1036,8 @@ getinfo_helper_accounting(control_connection_t *conn, if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { - if (hibernate_state == HIBERNATE_STATE_DORMANT) - *answer = tor_strdup("hard"); - else if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH) - *answer = tor_strdup("soft"); - else - *answer = tor_strdup("awake"); + *answer = tor_strdup(hibernate_state_to_string(hibernate_state)); + tor_strlower(*answer); } else if (!strcmp(question, "accounting/bytes")) { tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, U64_PRINTF_ARG(n_bytes_read_in_interval), @@ -1062,6 +1087,20 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +/** + * Helper function: called when the hibernation state changes, and sends a + * SERVER_STATUS event to notify interested controllers of the accounting + * state change. + */ +static void +on_hibernate_state_change(hibernate_state_t prev_state) +{ + (void)prev_state; /* Should we do something with this? */ + control_event_server_status(LOG_NOTICE, + "HIBERNATION_STATUS STATUS=%s", + hibernate_state_to_string(hibernate_state)); +} + #ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit

1 0

[tor/master] Merge remote-tracking branch 'yawning-schwanenleid/feature18685'
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 4a44e2d6f1152c6d431f21dca26c84a8a28172a5 Merge: bff53aa a19f419 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 26 13:39:50 2016 -0400 Merge remote-tracking branch 'yawning-schwanenleid/feature18685' changes/feature18685 | 3 +++ src/or/hibernate.c | 51 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 6 deletions(-)

1 0