tor-commits May 2017

tor-commits@lists.torproject.org

16 participants
1135 discussions

[exonerator/master] Update to metrics-lib 1.7.0.
by karsten＠torproject.org 17 May '17

17 May '17

commit aa296fd431db27e5246bdbad7a4b7959c1e77408 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 17 15:30:15 2017 +0200 Update to metrics-lib 1.7.0. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index f5b2767..111423f 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="webxmlfile" value="src/main/webapp/web.xml"/> <property name="warfile" value="${dist}/exonerator.war"/> <property name="source-and-target-java-version" value="1.7" /> - <property name="descriptorversion" value="1.6.0" /> + <property name="descriptorversion" value="1.7.0" /> <path id="classpath"> <pathelement path="${classes}"/> <fileset dir="${libs}">

1 0

[metrics-lib/master] Bump version to 1.7.0-dev.
by karsten＠torproject.org 17 May '17

17 May '17

commit bd52bcaeb549b83cdb95238cbd904a3ebe094ebf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 17 14:00:41 2017 +0200 Bump version to 1.7.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 950b5c7..0d6608f 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.7.0" /> + <property name="release.version" value="1.7.0-dev" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[metrics-lib/release] Add tutorial link and examples.
by karsten＠torproject.org 17 May '17

17 May '17

commit b1ea6416271a53d394dd4c7eb4de8dbc73d5dbca Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 13 20:18:21 2017 +0100 Add tutorial link and examples. --- README.md | 8 +++ .../examples/ConsensusWeightByVersion.java | 62 ++++++++++++++++++++++ .../resources/examples/DownloadConsensuses.java | 25 +++++++++ .../resources/examples/PluggableTransports.java | 48 +++++++++++++++++ 4 files changed, 143 insertions(+) diff --git a/README.md b/README.md index 2208379..0cc93d8 100644 --- a/README.md +++ b/README.md @@ -89,3 +89,11 @@ jarsigner -verify descriptor-1.0.0.jar jarsigner -verify descriptor-1.0.0-sources.jar ``` + +Tutorial +-------- + +The Metrics website has a tutorial for getting started with metrics-lib: + +https://metrics.torproject.org/metrics-lib.html + diff --git a/src/main/resources/examples/ConsensusWeightByVersion.java b/src/main/resources/examples/ConsensusWeightByVersion.java new file mode 100644 index 0000000..3b8495d --- /dev/null +++ b/src/main/resources/examples/ConsensusWeightByVersion.java @@ -0,0 +1,62 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +import org.torproject.descriptor.*; + +import java.io.File; +import java.util.*; + +public class ConsensusWeightByVersion { + public static void main(String[] args) { + + // Download consensuses. + DescriptorCollector descriptorCollector = DescriptorSourceFactory.createDescriptorCollector(); + descriptorCollector.collectDescriptors("https://collector.torproject.org", new String[] { "/recent/relay-descriptors/consensuses/" }, 0L, new File("descriptors"), false); + + // Keep local counters for extracted descriptor data. + long totalBandwidth = 0L; + SortedMap<String, Long> bandwidthByVersion = new TreeMap<>(); + + // Read descriptors from disk. + DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); + + // Add the directory with descriptors to the descriptor reader. + descriptorReader.addDirectory(new File("descriptors/recent/relay-descriptors/consensuses")); + Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { // Iterate over all descriptor files found. + DescriptorFile descriptorFile = descriptorFiles.next(); + + // Now, iterate over the descriptors contained in the file. + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + // We're only interested in consensuses. + continue; + } + RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { + String version = entry.getVersion(); + if (!version.startsWith("Tor ") || version.length() < 9) { + // We're only interested in a.b.c type versions for this example. + continue; + } + // Remove the 'Tor ' prefix and anything starting at the patch level. + version = version.substring(4, 9); + long bandwidth = entry.getBandwidth(); + totalBandwidth += bandwidth; + if (bandwidthByVersion.containsKey(version)) { + bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); + } else { + bandwidthByVersion.put(version, bandwidth); + } + } + } + } + + // Print out fractions of consensus weight by Tor version. + if (totalBandwidth > 0L) { + for (Map.Entry<String, Long> e : bandwidthByVersion.entrySet()) { + System.out.printf("%s -> %4.1f%%%n", e.getKey(), (100.0 * (double) e.getValue() / (double) totalBandwidth)); + } + } + } +} diff --git a/src/main/resources/examples/DownloadConsensuses.java b/src/main/resources/examples/DownloadConsensuses.java new file mode 100644 index 0000000..a592928 --- /dev/null +++ b/src/main/resources/examples/DownloadConsensuses.java @@ -0,0 +1,25 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +import org.torproject.descriptor.*; + +import java.io.File; + +public class DownloadConsensuses { + public static void main(String[] args) { + + // Download consensuses published in the last 72 hours, which will take up to five minutes and require several hundred MB on the local disk. + DescriptorCollector descriptorCollector = DescriptorSourceFactory.createDescriptorCollector(); + descriptorCollector.collectDescriptors( + // Download from Tor's main CollecTor instance, + "https://collector.torproject.org", + // include only network status consensuses + new String[] { "/recent/relay-descriptors/consensuses/" }, + // regardless of last-modified time, + 0L, + // write to the local directory called descriptors/, + new File("descriptors"), + // and don't delete extraneous files that do not exist remotely anymore. + false); + } +} diff --git a/src/main/resources/examples/PluggableTransports.java b/src/main/resources/examples/PluggableTransports.java new file mode 100644 index 0000000..88e7eb6 --- /dev/null +++ b/src/main/resources/examples/PluggableTransports.java @@ -0,0 +1,48 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +import org.torproject.descriptor.*; + +import java.io.File; +import java.util.*; + +public class PluggableTransports { + public static void main(String[] args) { + + DescriptorCollector descriptorCollector = DescriptorSourceFactory.createDescriptorCollector(); + descriptorCollector.collectDescriptors("https://collector.torproject.org", new String[] { "/recent/bridge-descriptors/extra-infos/" }, 0L, new File("descriptors"), false); + + Set<String> observedFingerprints = new HashSet<>(); + SortedMap<String, Integer> countedTransports = new TreeMap<>(); + + DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); + descriptorReader.addDirectory(new File("descriptors/recent/bridge-descriptors/extra-infos")); + Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { + continue; + } + BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; + String fingerprint = extraInfo.getFingerprint(); + if (observedFingerprints.add(fingerprint)) { + for (String transport : extraInfo.getTransports()) { + if (countedTransports.containsKey(transport)) { + countedTransports.put(transport, 1 + countedTransports.get(transport)); + } else { + countedTransports.put(transport, 1); + } + } + } + } + } + + if (!observedFingerprints.isEmpty()) { + double totalObservedFingerprints = observedFingerprints.size(); + for (Map.Entry<String, Integer> e : countedTransports.entrySet()) { + System.out.printf("%20s -> %4.1f%%%n", e.getKey(), (100.0 * (double) e.getValue() / totalObservedFingerprints)); + } + } + } +}

1 0

[metrics-lib/release] Extended JavaDoc explanation. Part of task-22190.
by karsten＠torproject.org 17 May '17

17 May '17

commit 3e3636b8d8f0aac26dd05ea513040674baf3f8f5 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue May 9 15:17:07 2017 +0000 Extended JavaDoc explanation. Part of task-22190. --- src/main/java/org/torproject/descriptor/DescriptorCollector.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/DescriptorCollector.java b/src/main/java/org/torproject/descriptor/DescriptorCollector.java index 1ec0a62..fb0668c 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorCollector.java +++ b/src/main/java/org/torproject/descriptor/DescriptorCollector.java @@ -45,8 +45,10 @@ public interface DescriptorCollector { * e.g., {@code "https://collector.torproject.org"} * @param remoteDirectories Remote directories to collect descriptors * from, e.g., - * {@code "/recent/relay-descriptors/server-descriptors/"}, without - * processing subdirectories unless they are explicitly listed + * {@code "recent/relay-descriptors/server-descriptors"}, without + * processing subdirectories unless they are explicitly listed. + * Leading and trailing slashes will be ignored, i.e., {@code "/abc/xyz/"} + * results in the same downloads as {@code "abc/xyz"}. * @param minLastModified Minimum last-modified time in milliseconds of * files to be collected, or 0 for collecting all files * @param localDirectory Directory where collected files will be written

1 0

[metrics-lib/release] Don't skip unrecognized lines in certain cases.
by karsten＠torproject.org 17 May '17

17 May '17

commit 8f69d784be6830b62bcaa9739e39043e240ebb4c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 7 15:19:44 2017 +0200 Don't skip unrecognized lines in certain cases. When we started using Java 7's switch-on-String in 2b4d773, we broke unrecognized line parsing in extra-info descriptors. Namely, when we reached the end of a crypto block we didn't reset the list for collecting crypto lines. So far so good, but any following unrecognized lines would be collected as crypto lines and later discarded, rather than being added to the unrecognized-lines list and later reported. This only affects relay descriptors, because sanitized bridge descriptors don't contain crypto blocks. And it only affects relay descriptors with crypto blocks, like "identity-ed25519", whereas relay extra-info descriptors published by older versions were not affected. Fixes #21890. --- CHANGELOG.md | 7 +++++++ .../org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java | 4 ++-- .../torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java | 8 ++++++++ .../org/torproject/descriptor/impl/ServerDescriptorImplTest.java | 8 ++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b85361..0cb2acd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version ??? + + # Medium changes + - Fix a bug where unrecognized lines in extra-info descriptors + below crypto blocks were silently skipped. + + # Changes in version 1.6.0 - 2017-02-17 * Major changes diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index d4be77a..136f31e 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -270,9 +270,9 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl } this.unrecognizedLines.addAll(cryptoLines); } - cryptoLines = null; - nextCrypto = ""; } + cryptoLines = null; + nextCrypto = ""; break; default: if (cryptoLines != null) { diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index 096314e..c29df8c 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -1805,6 +1805,14 @@ public class ExtraInfoDescriptorImplTest { + ROUTER_SIG_ED25519_LINE); } + @Test(expected = DescriptorParseException.class) + public void testEd25519FollowedbyUnrecognizedLine() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + + "\nunrecognized-line 1"); + } + @Test() public void testExtraInfoDigestSha256Relay() throws DescriptorParseException { diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 4370321..80dc815 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -1652,6 +1652,14 @@ public class ServerDescriptorImplTest { + ROUTER_SIG_ED25519_LINE); } + @Test(expected = DescriptorParseException.class) + public void testEd25519FollowedbyUnrecognizedLine() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + + "\nunrecognized-line 1"); + } + private static final String ONION_KEY_CROSSCERT_LINES = "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n"

1 0

[metrics-lib/release] Bump version to 1.6.0-dev.
by karsten＠torproject.org 17 May '17

17 May '17

commit 5b1db5d72b1d4de1eb5d41dda5aeac8ca5c69791 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 17 16:45:13 2017 +0100 Bump version to 1.6.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 1522f12..fb926c2 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.6.0" /> + <property name="release.version" value="1.6.0-dev" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[metrics-lib/release] Update to latest metrics-base.
by karsten＠torproject.org 17 May '17

17 May '17

commit a57b1f7698bb03abe023ce2aa95cb9d0c8144a1d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Mar 14 09:29:07 2017 +0100 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 6940759..e130139 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 6940759436bbd3bdb117572886f6104f78c3b469 +Subproject commit e130139e1a551bcf3e6437a85799590600f4b096

1 0

[metrics-lib/release] Accept extra arguments in extra-info descriptors.
by karsten＠torproject.org 17 May '17

17 May '17

commit 71885a8e694b2a37b8a868389e4ea6d1ac037b30 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 3 09:41:10 2017 +0200 Accept extra arguments in extra-info descriptors. According to the specification it's valid to add extra arguments to descriptor lines unless they are tagged with "[No extra arguments]". This is not the case for any of the statistics-related lines in extra-info descriptors, so we should allow extra arguments there. Fixes #21934. --- CHANGELOG.md | 4 + .../descriptor/impl/BandwidthHistoryImpl.java | 4 +- .../descriptor/impl/ExtraInfoDescriptorImpl.java | 12 +- .../torproject/descriptor/impl/ParseHelper.java | 16 --- .../impl/ExtraInfoDescriptorImplTest.java | 127 +++++++++++++++++++++ .../descriptor/impl/ServerDescriptorImplTest.java | 6 +- 6 files changed, 142 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da9cb37..a7f0aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ below crypto blocks were silently skipped. - Add support for six new key-value pairs added by OnionPerf. + * Minor changes + - Accept extra arguments in statistics-related extra-info + descriptor lines, as permitted by dir-spec.txt. + # Changes in version 1.6.0 - 2017-02-17 diff --git a/src/main/java/org/torproject/descriptor/impl/BandwidthHistoryImpl.java b/src/main/java/org/torproject/descriptor/impl/BandwidthHistoryImpl.java index 8f18b62..ef29a3b 100644 --- a/src/main/java/org/torproject/descriptor/impl/BandwidthHistoryImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BandwidthHistoryImpl.java @@ -15,7 +15,7 @@ public class BandwidthHistoryImpl implements BandwidthHistory { String[] partsNoOpt) throws DescriptorParseException { boolean isValid = false; this.line = line; - if (partsNoOpt.length == 5 || partsNoOpt.length == 6) { + if (partsNoOpt.length >= 5) { try { this.historyEndMillis = ParseHelper.parseTimestampAtIndex(line, partsNoOpt, 1, 2); @@ -32,7 +32,7 @@ public class BandwidthHistoryImpl implements BandwidthHistory { && partsNoOpt[4].equals("s)")) { /* There are no bandwidth values to parse. */ isValid = true; - } else if (partsNoOpt.length == 6) { + } else if (partsNoOpt.length >= 6) { /* There are bandwidth values to parse. */ values = partsNoOpt[5].split(",", -1); } else if (partsNoOpt[4].length() > 2) { diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 136f31e..bbaae19 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -324,7 +324,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseGeoipDbDigestLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - if (partsNoOpt.length != 2) { + if (partsNoOpt.length < 2) { throw new DescriptorParseException("Illegal line '" + line + "' in extra-info descriptor."); } @@ -334,7 +334,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseGeoip6DbDigestLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - if (partsNoOpt.length != 2) { + if (partsNoOpt.length < 2) { throw new DescriptorParseException("Illegal line '" + line + "' in extra-info descriptor."); } @@ -344,7 +344,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseGeoipStartTimeLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - if (partsNoOpt.length != 3) { + if (partsNoOpt.length < 3) { throw new DescriptorParseException("Illegal line '" + line + "' in extra-info descriptor."); } @@ -369,7 +369,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private long[] parseStatsEndLine(String line, String[] partsNoOpt, int partsNoOptExpectedLength) throws DescriptorParseException { - if (partsNoOpt.length != partsNoOptExpectedLength + if (partsNoOpt.length < partsNoOptExpectedLength || partsNoOpt[3].length() < 2 || !partsNoOpt[3].startsWith("(") || !partsNoOpt[4].equals("s)")) { throw new DescriptorParseException("Illegal line '" + line + "'."); @@ -424,7 +424,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private double parseShareLine(String line, String[] partsNoOpt) throws DescriptorParseException { double share = -1.0; - if (partsNoOpt.length == 2 && partsNoOpt[1].length() >= 2 + if (partsNoOpt.length >= 2 && partsNoOpt[1].length() >= 2 && partsNoOpt[1].endsWith("%")) { String shareString = partsNoOpt[1]; shareString = shareString.substring(0, shareString.length() - 1); @@ -550,7 +550,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { int circuits = -1; - if (partsNoOpt.length == 2) { + if (partsNoOpt.length >= 2) { try { circuits = Integer.parseInt(partsNoOpt[1]); } catch (NumberFormatException e) { diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java index f73a591..fa6ca75 100644 --- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java @@ -341,10 +341,6 @@ public class ParseHelper { if (partsNoOpt.length < index) { throw new DescriptorParseException("Line '" + line + "' does not " + "contain a key-value list at index " + index + "."); - } else if (partsNoOpt.length > index + 1 ) { - throw new DescriptorParseException("Line '" + line + "' contains " - + "unrecognized values beyond the expected key-value list at " - + "index " + index + "."); } else if (partsNoOpt.length > index) { if (!commaSeparatedKeyValueListPatterns.containsKey(keyLength)) { String keyPattern = "[0-9a-zA-Z?<>\\-_]" @@ -390,10 +386,6 @@ public class ParseHelper { if (partsNoOpt.length < index) { throw new DescriptorParseException("Line '" + line + "' does not " + "contain a key-value list at index " + index + "."); - } else if (partsNoOpt.length > index + 1 ) { - throw new DescriptorParseException("Line '" + line + "' contains " - + "unrecognized values beyond the expected key-value list at " - + "index " + index + "."); } else if (partsNoOpt.length > index) { String[] listElements = partsNoOpt[index].split(",", -1); for (String listElement : listElements) { @@ -428,10 +420,6 @@ public class ParseHelper { throw new DescriptorParseException("Line '" + line + "' does not " + "contain a comma-separated value list at index " + index + "."); - } else if (partsNoOpt.length > index + 1) { - throw new DescriptorParseException("Line '" + line + "' contains " - + "unrecognized values beyond the expected comma-separated " - + "value list at index " + index + "."); } else if (partsNoOpt.length > index) { String[] listElements = partsNoOpt[index].split(",", -1); result = new Integer[listElements.length]; @@ -456,10 +444,6 @@ public class ParseHelper { throw new DescriptorParseException("Line '" + line + "' does not " + "contain a comma-separated value list at index " + index + "."); - } else if (partsNoOpt.length > index + 1) { - throw new DescriptorParseException("Line '" + line + "' contains " - + "unrecognized values beyond the expected comma-separated " - + "value list at index " + index + "."); } else if (partsNoOpt.length > index) { String[] listElements = partsNoOpt[index].split(",", -1); result = new Double[listElements.length]; diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index c29df8c..118ada8 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -1093,6 +1093,14 @@ public class ExtraInfoDescriptorImplTest { + "81281024,,60625920,67922944"); } + @Test() + public void testDirreqWriteHistoryExtraArg() throws DescriptorParseException { + DescriptorBuilder.createWithDirreqWriteHistoryLine( + "dirreq-write-history " + + "2012-02-11 09:03:39 (900 s) 81281024,64996352,60625920," + + "67922944 bin_size=1024"); + } + @Test(expected = DescriptorParseException.class) public void testDirreqReadHistoryMissingBytesEnd() throws DescriptorParseException { @@ -1131,6 +1139,17 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testGeoipDbDigestExtraArg() throws DescriptorParseException { + ExtraInfoDescriptor descriptor = DescriptorBuilder + .createWithGeoip6DbDigestLine("geoip-db-digest " + + "916A3CA8B7DF61473D5AE5B21711F35F301CE9E8 " + + "yblgrXtEgF3glaKv5ZvHhRREUI1t1c37SxparXSmYR4Q1yiK5zg4HE8eT9ILPRW9" + + "3I5W/pZGQxL8Bu42dGjnAQ"); + assertEquals("916A3CA8B7DF61473D5AE5B21711F35F301CE9E8", + descriptor.getGeoipDbDigest()); + } + + @Test() public void testGeoip6DbDigestValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = DescriptorBuilder .createWithGeoip6DbDigestLine("geoip6-db-digest " @@ -1140,6 +1159,17 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testGeoip6DbDigestExtraArg() throws DescriptorParseException { + ExtraInfoDescriptor descriptor = DescriptorBuilder + .createWithGeoip6DbDigestLine("geoip6-db-digest " + + "916A3CA8B7DF61473D5AE5B21711F35F301CE9E8 " + + "yblgrXtEgF3glaKv5ZvHhRREUI1t1c37SxparXSmYR4Q1yiK5zg4HE8eT9ILPRW9" + + "3I5W/pZGQxL8Bu42dGjnAQ"); + assertEquals("916A3CA8B7DF61473D5AE5B21711F35F301CE9E8", + descriptor.getGeoip6DbDigest()); + } + + @Test() public void testGeoipStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = GeoipStatsBuilder .createWithDefaultLines(); @@ -1242,6 +1272,14 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testGeoipClientOriginsExtraArg() + throws DescriptorParseException { + GeoipStatsBuilder.createWithGeoipClientOriginsLine( + "geoip-client-origins de=1152,de=952,cn=896,us=712,it=504 " + + "ru=352 fr=208 gb=208 ir=200"); + } + + @Test() public void testDirreqStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = DirreqStatsBuilder .createWithDefaultLines(); @@ -1279,6 +1317,13 @@ public class ExtraInfoDescriptorImplTest { + "2012-02-11 00:59:53 (172800 s)"); } + @Test() + public void testDirreqStatsExtraArg() + throws DescriptorParseException { + DirreqStatsBuilder.createWithDirreqStatsEndLine("dirreq-stats-end " + + "2012-02-11 00:59:53 (172800 s) XXXXXXXXXXXXXXXXXXXXXXXXXXX"); + } + @Test(expected = DescriptorParseException.class) public void testDirreqV3IpsThreeLetterCountry() throws DescriptorParseException { @@ -1287,6 +1332,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testDirreqV3IpsExtraArg() + throws DescriptorParseException { + DirreqStatsBuilder.createWithDirreqV3IpsLine("dirreq-v3-ips " + + "ab=12,cd=34 ef=56"); + } + + @Test() public void testDirreqV2IpsDigitCountry() throws DescriptorParseException { DirreqStatsBuilder.createWithDirreqV2IpsLine("dirreq-v2-ips 00=8"); @@ -1312,6 +1364,13 @@ public class ExtraInfoDescriptorImplTest { + "ok==10848"); } + @Test() + public void testDirreqV3RespExtraArg() + throws DescriptorParseException { + DirreqStatsBuilder.createWithDirreqV3RespLine("dirreq-v3-resp " + + "ok=1084 4801=ko"); + } + @Test(expected = DescriptorParseException.class) public void testDirreqV2RespNull() throws DescriptorParseException { @@ -1333,6 +1392,13 @@ public class ExtraInfoDescriptorImplTest { + "0.37"); } + @Test() + public void testDirreqV3ShareExtraArg() + throws DescriptorParseException { + DirreqStatsBuilder.createWithDirreqV3ShareLine("dirreq-v3-share " + + "0.37% 123456"); + } + @Test(expected = DescriptorParseException.class) public void testDirreqV3DirectDlSpace() throws DescriptorParseException { @@ -1362,6 +1428,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testDirreqV3TunneledDlExtraArg() + throws DescriptorParseException { + DirreqStatsBuilder.createWithDirreqV2TunneledDlLine( + "dirreq-v2-tunneled-dl complete=-8 incomplete=1/-8"); + } + + @Test() public void testEntryStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = EntryStatsBuilder .createWithDefaultLines(); @@ -1443,6 +1516,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testCellCircuitsPerDecileExtraArg() + throws DescriptorParseException { + CellStatsBuilder.createWithCellCircuitsPerDecileLine( + "cell-circuits-per-decile 866 866 866 866 866"); + } + + @Test() public void testConnBiDirectValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = DescriptorBuilder @@ -1465,6 +1545,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testConnBiDirectStatsExtraArg() + throws DescriptorParseException { + DescriptorBuilder.createWithConnBiDirectLine("conn-bi-direct " + + "2012-02-11 01:59:39 (86400 s) 42173,1591,1310,1744 +1"); + } + + @Test() public void testExitStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = ExitStatsBuilder .createWithDefaultLines(); @@ -1541,6 +1628,12 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testExitStatsStreamsExtraArg() throws DescriptorParseException { + ExitStatsBuilder.createWithExitStreamsOpenedLine( + "exit-streams-opened 25=21474 3648"); + } + + @Test() public void testBridgeStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = BridgeStatsBuilder .createWithDefaultLines(); @@ -1570,6 +1663,13 @@ public class ExtraInfoDescriptorImplTest { + "2012-02-11 01:59:39 (0 s)"); } + @Test() + public void testBridgeStatsEndExtraArg() + throws DescriptorParseException { + BridgeStatsBuilder.createWithBridgeStatsEndLine("bridge-stats-end " + + "2012-02-11 01:59:39 (86400 s) 99999999999999999999999999999999"); + } + @Test(expected = DescriptorParseException.class) public void testBridgeIpsDouble() throws DescriptorParseException { @@ -1585,6 +1685,12 @@ public class ExtraInfoDescriptorImplTest { 0x69, 0x70, 0x73 }, false); // "ips" (no newline) } + @Test() + public void testBridgeIpsExtraArg() + throws DescriptorParseException { + BridgeStatsBuilder.createWithBridgeIpsLine("bridge-ips ir=24 5"); + } + @Test(expected = DescriptorParseException.class) public void testBridgeIpVersionsDouble() throws DescriptorParseException { @@ -1592,6 +1698,13 @@ public class ExtraInfoDescriptorImplTest { "bridge-ip-versions v4=24.5"); } + @Test() + public void testBridgeIpVersionsExtraArg() + throws DescriptorParseException { + BridgeStatsBuilder.createWithBridgeIpVersionsLine( + "bridge-ip-versions v4=24 5"); + } + @Test(expected = DescriptorParseException.class) public void testBridgeIpTransportsDouble() throws DescriptorParseException { @@ -1607,6 +1720,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testBridgeIpTransportsExtraArg() + throws DescriptorParseException { + BridgeStatsBuilder.createWithBridgeIpTransportsLine( + "bridge-ip-transports obfs2=24 5"); + } + + @Test() public void testHidservStatsValid() throws DescriptorParseException { ExtraInfoDescriptor descriptor = HidservStatsBuilder .createWithDefaultLines(); @@ -1670,6 +1790,13 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testHidservDirOnionsSeenExtraArg() + throws DescriptorParseException { + HidservStatsBuilder.createWithHidservDirOnionsSeenLine( + "hidserv-dir-onions-seen -3 delta_f=8 epsilon=0.30 bin_size=8 pi=3"); + } + + @Test() public void testRouterSignatureOpt() throws DescriptorParseException { DescriptorBuilder.createWithRouterSignatureLines("opt " diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 80dc815..b713dd6 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -1245,11 +1245,11 @@ public class ServerDescriptorImplTest { "write-history 2012-01-01 03:51:44 (900 "); } - @Test(expected = DescriptorParseException.class) - public void testWriteHistoryTrailingNumber() + @Test() + public void testWriteHistoryExtraArg() throws DescriptorParseException { DescriptorBuilder.createWithWriteHistoryLine("write-history " - + "2012-01-01 03:51:44 (900 s) 4345856 1"); + + "2012-01-01 03:51:44 (900 s) 4345856 bin_size=1024"); } @Test()

1 0

[metrics-lib/release] Tweak change log a bit.
by karsten＠torproject.org 17 May '17

17 May '17

commit 2ed7d618b1260cb838d24a5b0cedad4cea14e59f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue May 9 17:17:06 2017 +0200 Tweak change log a bit. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7f0aac..9ee333a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -# Changes in version ??? +# Changes in version 1.7.0 - ??? - # Medium changes + * Medium changes - Fix a bug where unrecognized lines in extra-info descriptors below crypto blocks were silently skipped. - Add support for six new key-value pairs added by OnionPerf.

1 0

[metrics-lib/release] Add support for six new key-value pairs added by OnionPerf.
by karsten＠torproject.org 17 May '17

17 May '17

commit ee696b09f02ae59e7dccf8a5eb68ca484051040e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon May 1 20:38:55 2017 +0200 Add support for six new key-value pairs added by OnionPerf. OnionPerf adds six new key-value pairs to the .tpf format that Torperf/CollecTor did not produce: ENDPOINTLOCAL, ENDPOINTPROXY, ENDPOINTREMOTE, HOSTNAMELOCAL, HOSTNAMEREMOTE, and SOURCEADDRESS. We should add support for these keys to metrics-lib, so that we can start using their values. Implements #22122. --- CHANGELOG.md | 1 + .../org/torproject/descriptor/TorperfResult.java | 62 ++++++++++++++++ .../descriptor/impl/TorperfResultImpl.java | 84 ++++++++++++++++++++++ .../descriptor/impl/TorperfResultImplTest.java | 42 +++++++++++ 4 files changed, 189 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cb2acd..da9cb37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ # Medium changes - Fix a bug where unrecognized lines in extra-info descriptors below crypto blocks were silently skipped. + - Add support for six new key-value pairs added by OnionPerf. # Changes in version 1.6.0 - 2017-02-17 diff --git a/src/main/java/org/torproject/descriptor/TorperfResult.java b/src/main/java/org/torproject/descriptor/TorperfResult.java index 1f4e50c..1958ed2 100644 --- a/src/main/java/org/torproject/descriptor/TorperfResult.java +++ b/src/main/java/org/torproject/descriptor/TorperfResult.java @@ -211,5 +211,67 @@ public interface TorperfResult extends Descriptor { * @since 1.0.0 */ public int getUsedBy(); + + /** + * Return the hostname, IP address, and port that the TGen client used to + * connect to the local tor SOCKS port, formatted as + * <code>hostname:ip:port</code>, which may be <code>"NULL:0.0.0.0:0"</code> + * if TGen was not able to find this information or <code>null</code> if the + * OnionPerf line didn't contain this information. + * + * @since 1.7.0 + */ + public String getEndpointLocal(); + + /** + * Return the hostname, IP address, and port that the TGen client used to + * connect to the SOCKS proxy server that tor runs, formatted as + * <code>hostname:ip:port</code>, which may be <code>"NULL:0.0.0.0:0"</code> + * if TGen was not able to find this information or <code>null</code> if the + * OnionPerf line didn't contain this information. + * + * @since 1.7.0 + */ + public String getEndpointProxy(); + + /** + * Return the hostname, IP address, and port that the TGen client used to + * connect to the remote server, formatted as <code>hostname:ip:port</code>, + * which may be <code>"NULL:0.0.0.0:0"</code> if TGen was not able to find + * this information or <code>null</code> if the OnionPerf line didn't contain + * this information. + * + * @since 1.7.0 + */ + public String getEndpointRemote(); + + /** + * Return the client machine hostname, which may be <code>"(NULL)"</code> if + * the TGen client was not able to find this information or <code>null</code> + * if the OnionPerf line didn't contain this information. + * + * @since 1.7.0 + */ + public String getHostnameLocal(); + + /** + * Return the server machine hostname, which may be <code>"(NULL)"</code> if + * the TGen server was not able to find this information or <code>null</code> + * if the OnionPerf line didn't contain this information. + * + * @since 1.7.0 + */ + public String getHostnameRemote(); + + /** + * Return the public IP address of the OnionPerf host obtained by connecting + * to well-known servers and finding the IP address in the result, which may + * be <code>"unknown"</code> if OnionPerf was not able to find this + * information or <code>null</code> if the OnionPerf line didn't contain this + * information. + * + * @since 1.7.0 + */ + public String getSourceAddress(); } diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java index f5fe892..16274c7 100644 --- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -147,6 +147,24 @@ public class TorperfResultImpl extends DescriptorImpl case "USED_BY": this.parseUsedBy(value, keyValue, line); break; + case "ENDPOINTLOCAL": + this.parseEndpointLocal(value, keyValue, line); + break; + case "ENDPOINTPROXY": + this.parseEndpointProxy(value, keyValue, line); + break; + case "ENDPOINTREMOTE": + this.parseEndpointRemote(value, keyValue, line); + break; + case "HOSTNAMELOCAL": + this.parseHostnameLocal(value, keyValue, line); + break; + case "HOSTNAMEREMOTE": + this.parseHostnameRemote(value, keyValue, line); + break; + case "SOURCEADDRESS": + this.parseSourceAddress(value, keyValue, line); + break; default: if (key.startsWith("DATAPERC")) { this.parseDataPercentile(value, keyValue, line); @@ -359,6 +377,30 @@ public class TorperfResultImpl extends DescriptorImpl this.usedBy = this.parseInt(value, keyValue, line); } + private void parseEndpointLocal(String value, String keyValue, String line) { + this.endpointLocal = value; + } + + private void parseEndpointProxy(String value, String keyValue, String line) { + this.endpointProxy = value; + } + + private void parseEndpointRemote(String value, String keyValue, String line) { + this.endpointRemote = value; + } + + private void parseHostnameLocal(String value, String keyValue, String line) { + this.hostnameLocal = value; + } + + private void parseHostnameRemote(String value, String keyValue, String line) { + this.hostnameRemote = value; + } + + private void parseSourceAddress(String value, String keyValue, String line) { + this.sourceAddress = value; + } + private long parseTimestamp(String value, String keyValue, String line) throws DescriptorParseException { long timestamp = -1L; @@ -570,5 +612,47 @@ public class TorperfResultImpl extends DescriptorImpl public int getUsedBy() { return this.usedBy; } + + private String endpointLocal; + + @Override + public String getEndpointLocal() { + return this.endpointLocal; + } + + private String endpointProxy; + + @Override + public String getEndpointProxy() { + return this.endpointProxy; + } + + private String endpointRemote; + + @Override + public String getEndpointRemote() { + return this.endpointRemote; + } + + private String hostnameLocal; + + @Override + public String getHostnameLocal() { + return this.hostnameLocal; + } + + private String hostnameRemote; + + @Override + public String getHostnameRemote() { + return this.hostnameRemote; + } + + private String sourceAddress; + + @Override + public String getSourceAddress() { + return this.sourceAddress; + } } diff --git a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java index 080d16d..640b8a3 100644 --- a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java @@ -5,8 +5,10 @@ package org.torproject.descriptor.impl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorParseException; import org.junit.Test; @@ -95,5 +97,45 @@ public class TorperfResultImplTest { assertEquals("DATAPERMILLE", torperfResult.getUnrecognizedKeys().firstKey()); } + + private static final String input2 = + "BUILDTIMES=0.490000009537,0.610000133514,0.75 CIRC_ID=8522 " + + "CONNECT=1493397365.14 DATACOMPLETE=1493397368.13 " + + "DATAPERC0=1493397367.67 DATAPERC10=1493397367.81 " + + "DATAPERC100=1493397368.13 DATAPERC20=1493397367.87 " + + "DATAPERC30=1493397367.87 DATAPERC40=1493397367.90 " + + "DATAPERC50=1493397367.91 DATAPERC60=1493397367.97 " + + "DATAPERC70=1493397367.97 DATAPERC80=1493397367.99 " + + "DATAPERC90=1493397367.99 DATAREQUEST=1493397367.33 " + + "DATARESPONSE=1493397367.67 DIDTIMEOUT=0 " + + "ENDPOINTLOCAL=localhost:127.0.0.1:42436 " + + "ENDPOINTPROXY=localhost:127.0.0.1:52265 " + + "ENDPOINTREMOTE=4jocm7xwo4ltrvzp.onion:0.0.0.0:80 FILESIZE=51200 " + + "HOSTNAMELOCAL=op-us HOSTNAMEREMOTE=op-us LAUNCH=1493396168.1 " + + "NEGOTIATE=1493397365.14 " + + "PATH=$F69BED36177ED727706512BA6A97755025EEA0FB," + + "$91D23D8A539B83D2FB56AA67ECD4D75CC093AC55," + + "$4DD902046E7155BBE79C34EE6D53BF7408B98CE4 QUANTILE=0.8 " + + "READBYTES=51269 REQUEST=1493397365.14 RESPONSE=1493397367.32 " + + "SOCKET=1493397365.14 SOURCE=op-us SOURCEADDRESS=199.119.112.144 " + + "START=1493397365.14 TIMEOUT=1500 USED_AT=1493397368.14 " + + "USED_BY=17429 WRITEBYTES=54"; + + @Test() + public void testEndpointsHostnamesSourceAddress() + throws DescriptorParseException { + List<Descriptor> result = TorperfResultImpl.parseTorperfResults( + input2.getBytes(), true); + assertEquals(1, result.size()); + TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0); + assertNull(torperfResult.getUnrecognizedKeys()); + assertEquals("localhost:127.0.0.1:42436", torperfResult.getEndpointLocal()); + assertEquals("localhost:127.0.0.1:52265", torperfResult.getEndpointProxy()); + assertEquals("4jocm7xwo4ltrvzp.onion:0.0.0.0:80", + torperfResult.getEndpointRemote()); + assertEquals("op-us", torperfResult.getHostnameLocal()); + assertEquals("op-us", torperfResult.getHostnameRemote()); + assertEquals("199.119.112.144", torperfResult.getSourceAddress()); + } }

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits May 2017