October 2017 - tor-commits - lists.torproject.org

[collector/release] Bump version to 1.2.1-dev.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit d162d90299a1d27fc8750832b56e60f3a5c97b66 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Aug 18 15:03:03 2017 +0200 Bump version to 1.2.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index ec0eb41..279cd2f 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.2.1" /> + <property name="release.version" value="1.2.1-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.0.0" />

1 0

[collector/release] Prepare for 1.4.0 release.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 95051e9480a8a77badea6b6f48a8016bf24f6446 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 9 14:23:52 2017 +0200 Prepare for 1.4.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4973f7e..b12b33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.?.? - 2017-??-?? +# Changes in version 1.4.0 - 2017-10-09 * Major changes - Rename "Onionperf*" configuration options in collector.properties diff --git a/build.xml b/build.xml index d60350b..7cbff44 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.3.0-dev" /> + <property name="release.version" value="1.4.0" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.1.1" />

1 0

[collector/release] Prepare for 1.3.0 release.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 18006f9561f4ad03a1fae35b580ff66222296519 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 15 11:36:04 2017 +0200 Prepare for 1.3.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d70bbd..fb94bd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.?.? - 2017-??-?? +# Changes in version 1.3.0 - 2017-09-15 * Major changes - Update to metrics-lib 2.1.0 and to Java 8. diff --git a/build.xml b/build.xml index 3e162f0..9ebac4a 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.2.1-dev" /> + <property name="release.version" value="1.3.0" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.1.0" />

1 0

[collector/release] Changed test data, wich makes some tests fail.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 1042a7683c913a0e30d998fa9f7900d97046ac40 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Aug 18 13:03:04 2017 +0000 Changed test data, wich makes some tests fail. Part of task-23215. --- src/test/resources/2016-09-20-13-00-00-consensus | 2 +- ...03580E3B66A1707A00E60F2D15B-60ADC6BEC262AE921A1037D54C8A3976367DBE87 | 2 +- src/test/resources/2016-10-02-17-00-00-consensus-microdesc | 2 +- .../resources/20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/2016-09-20-13-00-00-consensus b/src/test/resources/2016-09-20-13-00-00-consensus index 6511263..a61a811 100644 --- a/src/test/resources/2016-09-20-13-00-00-consensus +++ b/src/test/resources/2016-09-20-13-00-00-consensus @@ -1,4 +1,4 @@ -@type network-status-consensus-3 1.0 +@type network-status-consensus-3 1.17 network-status-version 3 vote-status consensus consensus-method 20 diff --git a/src/test/resources/2016-09-20-13-00-00-vote-49015F787433103580E3B66A1707A00E60F2D15B-60ADC6BEC262AE921A1037D54C8A3976367DBE87 b/src/test/resources/2016-09-20-13-00-00-vote-49015F787433103580E3B66A1707A00E60F2D15B-60ADC6BEC262AE921A1037D54C8A3976367DBE87 index c65ff66..02ee2bc 100644 --- a/src/test/resources/2016-09-20-13-00-00-vote-49015F787433103580E3B66A1707A00E60F2D15B-60ADC6BEC262AE921A1037D54C8A3976367DBE87 +++ b/src/test/resources/2016-09-20-13-00-00-vote-49015F787433103580E3B66A1707A00E60F2D15B-60ADC6BEC262AE921A1037D54C8A3976367DBE87 @@ -1,4 +1,4 @@ -@type network-status-vote-3 1.0 +@type network-status-vote-3 1.17 network-status-version 3 vote-status vote consensus-methods 13 14 15 16 17 18 19 20 22 diff --git a/src/test/resources/2016-10-02-17-00-00-consensus-microdesc b/src/test/resources/2016-10-02-17-00-00-consensus-microdesc index 705f01b..563708f 100644 --- a/src/test/resources/2016-10-02-17-00-00-consensus-microdesc +++ b/src/test/resources/2016-10-02-17-00-00-consensus-microdesc @@ -1,4 +1,4 @@ -@type network-status-microdesc-consensus-3 1.0 +@type network-status-microdesc-consensus-3 1.17 network-status-version 3 microdesc vote-status consensus consensus-method 20 diff --git a/src/test/resources/20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 b/src/test/resources/20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 index c85732b..6171b44 100644 --- a/src/test/resources/20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 +++ b/src/test/resources/20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 @@ -1,4 +1,4 @@ -@type bridge-network-status 1.2 +@type bridge-network-status 1.17 published 2016-09-20 06:38:16 flag-thresholds stable-uptime=1293451 stable-mtbf=1209559 fast-speed=55000 guard-wfu=98.000% guard-tk=691200 guard-bw-inc-exits=336000 guard-bw-exc-exits=337000 enough-mtbf=1 ignoring-advertised-bws=0 fingerprint 1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1

1 0

[collector/release] Adapt and add tests for OnionPerf sync-runs.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 4b3c2fef58363c7135bad21a883e9f17a57d001a Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 12 18:47:46 2017 +0000 Adapt and add tests for OnionPerf sync-runs. --- .../java/org/torproject/collector/conf/ConfigurationTest.java | 2 +- .../java/org/torproject/collector/sync/SyncPersistenceTest.java | 9 ++++++++- src/test/resources/op-nl-1048576-2017-04-11.tpf | 8 ++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index bbe7832..634bca5 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -40,7 +40,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 44, Key.values().length); + 45, Key.values().length); } @Test() diff --git a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java index fe47f26..f0d30bc 100644 --- a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java +++ b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java @@ -43,6 +43,12 @@ public class SyncPersistenceTest { Integer.valueOf(1), // expected recent count of descs files Integer.valueOf(1)}, // expected output count of descs files + {"torperf/op-nl-1048576-2017-04-11.tpf", + new String[]{"torperf/2017/04/11/op-nl-1048576-2017-04-11.tpf"}, + "op-nl-1048576-2017-04-11.tpf", + Integer.valueOf(1), + Integer.valueOf(1)}, + {"relay-descriptors/server-descriptors/" + "2016-10-05-19-06-17-server-descriptors", new String[]{"relay-descriptors/server-descriptor/2016/10/e/3/" @@ -273,8 +279,9 @@ public class SyncPersistenceTest { List<String> content = Files.readAllLines(recentList.get(0).toPath(), StandardCharsets.UTF_8); List<String> expContent = linesFromResource(filename); - assertEquals(expContent.size(), content.size()); int contentSize = content.size(); + assertEquals("Number of lines incorrect for file '" + filename + "': " + + content, expContent.size(), contentSize); for (String line : expContent) { assertTrue("Couln't remove " + line + " from " + recentList.get(0), content.remove(line)); diff --git a/src/test/resources/op-nl-1048576-2017-04-11.tpf b/src/test/resources/op-nl-1048576-2017-04-11.tpf new file mode 100644 index 0000000..2396c2e --- /dev/null +++ b/src/test/resources/op-nl-1048576-2017-04-11.tpf @@ -0,0 +1,8 @@ +@type torperf 1.1 +BUILDTIMES=0.150000095367,0.230000019073,0.319999933243 CIRC_ID=664 CONNECT=1491891870.51 DATACOMPLETE=1491891877.51 DATAPERC0=1491891873.58 DATAPERC10=1491891874.32 DATAPERC100=1491891877.51 DATAPERC20=1491891874.68 DATAPERC30=1491891874.94 DATAPERC40=1491891875.37 DATAPERC50=1491891875.60 DATAPERC60=1491891875.95 DATAPERC70=1491891876.29 DATAPERC80=1491891876.68 DATAPERC90=1491891876.96 DATAREQUEST=1491891873.04 DATARESPONSE=1491891873.58 DIDTIMEOUT=0 ENDPOINTLOCAL=localhost:127.0.0.1:48934 ENDPOINTPROXY=localhost:127.0.0.1:43531 ENDPOINTREMOTE=5klxwn5wiwnd4xkm.onion:0.0.0.0:80 FILESIZE=1048576 HOSTNAMELOCAL=op-nl HOSTNAMEREMOTE=op-nl LAUNCH=1491891573.04 NEGOTIATE=1491891870.52 PATH=$34068126B7A8CE0247FA89A2F72F6C4A4DDD8F17,$3A6A156A01891A13DFAB19A16D2DBABB84F1869B,$75D4656446FE6625B6B773DA1E45222A902E3E8B QUANTILE=0.8 READBYTES=1048644 REQUEST=1491891870.52 RESPONSE=1491891873.04 SOCKET=1491891870.51 SOURCE=op-nl SOURCEADDRESS=unknown START=1491891870.51 TIMEOUT=1500 USED_AT=149 1891877.63 USED_BY=1406 WRITEBYTES=54 +@type torperf 1.1 +BUILDTIMES=0.0499999523163,0.0699999332428,0.120000123978 CIRC_ID=669 CONNECT=1491893670.52 DATACOMPLETE=1491893673.18 DATAPERC0=1491893671.99 DATAPERC10=1491893672.22 DATAPERC100=1491893673.18 DATAPERC20=1491893672.36 DATAPERC30=1491893672.45 DATAPERC40=1491893672.55 DATAPERC50=1491893672.65 DATAPERC60=1491893672.76 DATAPERC70=1491893672.87 DATAPERC80=1491893672.90 DATAPERC90=1491893673.06 DATAREQUEST=1491893671.83 DATARESPONSE=1491893671.99 DIDTIMEOUT=0 ENDPOINTLOCAL=localhost:127.0.0.1:49000 ENDPOINTPROXY=localhost:127.0.0.1:43531 ENDPOINTREMOTE=5klxwn5wiwnd4xkm.onion:0.0.0.0:80 FILESIZE=1048576 HOSTNAMELOCAL=op-nl HOSTNAMEREMOTE=op-nl LAUNCH=1491891871.04 NEGOTIATE=1491893670.52 PATH=$766A343C7623BE12976DDE361E47A3FD15C24ED9,$4FA9D001ED3BB02717C419DCC11644F2F2B2A592,$931BE618172446D534478988C01EE1CBD1874A97 QUANTILE=0.8 READBYTES=1048644 REQUEST=1491893670.52 RESPONSE=1491893671.83 SOCKET=1491893670.52 SOURCE=op-nl SOURCEADDRESS=unknown START=1491893670.52 TIMEOUT=1500 USED_AT=1 491893673.24 USED_BY=1436 WRITEBYTES=54 +@type torperf 1.1 +BUILDTIMES=0.00999999046326,0.039999961853,0.0799999237061 CIRC_ID=961 CONNECT=1491926070.57 DATACOMPLETE=1491926078.34 DATAPERC0=1491926072.18 DATAPERC10=1491926073.54 DATAPERC100=1491926078.34 DATAPERC20=1491926074.80 DATAPERC30=1491926075.81 DATAPERC40=1491926076.39 DATAPERC50=1491926076.85 DATAPERC60=1491926077.18 DATAPERC70=1491926077.53 DATAPERC80=1491926077.75 DATAPERC90=1491926077.97 DATAREQUEST=1491926071.80 DATARESPONSE=1491926072.18 DIDTIMEOUT=0 ENDPOINTLOCAL=localhost:127.0.0.1:50180 ENDPOINTPROXY=localhost:127.0.0.1:43531 ENDPOINTREMOTE=5klxwn5wiwnd4xkm.onion:0.0.0.0:80 FILESIZE=1048576 HOSTNAMELOCAL=op-nl HOSTNAMEREMOTE=op-nl LAUNCH=1491925174.04 NEGOTIATE=1491926070.57 PATH=$1A0AD1B2BAE9FEC4970186692B556B4BB7C57B3B,$86C281AD135058238D7A337D546C902BE8505DDE,$F1A98CA0FA1B3DE0AD71B97A6F5003A6BACA9FEB QUANTILE=0.8 READBYTES=1048644 REQUEST=1491926070.58 RESPONSE=1491926071.80 SOCKET=1491926070.57 SOURCE=op-nl SOURCEADDRESS=unknown START=1491926070.57 TIMEOUT=1500 USED_AT= 1491926078.4 USED_BY=2002 WRITEBYTES=54 +@type torperf 1.1 +BUILDTIMES=0.210000038147,0.299999952316,0.55999994278 CIRC_ID=888 CONNECT=1491916770.56 DATACOMPLETE=1491916773.22 DATAPERC0=1491916771.17 DATAPERC10=1491916771.44 DATAPERC100=1491916773.22 DATAPERC20=1491916771.71 DATAPERC30=1491916771.88 DATAPERC40=1491916772.05 DATAPERC50=1491916772.22 DATAPERC60=1491916772.44 DATAPERC70=1491916772.61 DATAPERC80=1491916772.78 DATAPERC90=1491916773.02 DATAREQUEST=1491916770.85 DATARESPONSE=1491916771.17 DIDTIMEOUT=0 ENDPOINTLOCAL=localhost:127.0.0.1:49868 ENDPOINTPROXY=localhost:127.0.0.1:43531 ENDPOINTREMOTE=37.218.247.40:37.218.247.40:80 FILESIZE=1048576 HOSTNAMELOCAL=op-nl HOSTNAMEREMOTE=op-nl LAUNCH=1491916530.04 NEGOTIATE=1491916770.56 PATH=$9E978553A45A53CDDCAEE81FCD5F6878CC920B2F,$0C9B3C686421C5A8C20BAA0D7369CD929F43A174,$DD808ECE4F2E24F377CBE11E335ECDA196FE3B78 QUANTILE=0.8 READBYTES=1048644 REQUEST=1491916770.56 RESPONSE=1491916770.85 SOCKET=1491916770.56 SOURCE=op-nl SOURCEADDRESS=unknown START=1491916770.56 TIMEOUT=1500 USED_AT=1491916 773.23 USED_BY=1854 WRITEBYTES=54

1 0

[collector/release] Keep annotations of given descriptors.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 68f5301bd1cf8b10fcae0a2d7e924cc87093a97c Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Aug 18 13:03:05 2017 +0000 Keep annotations of given descriptors. Makes test pass again and implements task-23215. Changes a test descriptor to contain a second annotation. --- CHANGELOG.md | 8 ++++++++ .../collector/persist/DescriptorPersistence.java | 14 ++++++++++++-- src/test/resources/bridge-2016-10-02-08-09-00-extra-infos | 8 ++++++++ src/test/resources/moria-1048576-2016-10-05.tpf | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a8bf6..ada02ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# Changes in version 1.?.? - 2017-??-?? + + * Medium changes + - When synchronizing descriptors from another CollecTor instance, + keep annotations provided by the descriptor and only add the + default annotation, when there was none. + + # Changes in version 1.2.1 - 2017-08-17 * Medium changes diff --git a/src/main/java/org/torproject/collector/persist/DescriptorPersistence.java b/src/main/java/org/torproject/collector/persist/DescriptorPersistence.java index d54392d..e7bce5b 100644 --- a/src/main/java/org/torproject/collector/persist/DescriptorPersistence.java +++ b/src/main/java/org/torproject/collector/persist/DescriptorPersistence.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.List; public abstract class DescriptorPersistence<T extends Descriptor> { @@ -32,9 +33,18 @@ public abstract class DescriptorPersistence<T extends Descriptor> { protected String recentPath; /** Initializes the paths for storing descriptors of type <code>T</code>. */ - protected DescriptorPersistence(T desc, byte[] annotation) { + protected DescriptorPersistence(T desc, byte[] defaultAnnotation) { this.desc = desc; - this.annotation = annotation; + List<String> annotations = desc.getAnnotations(); + if (annotations.isEmpty()) { + this.annotation = defaultAnnotation; + } else { + StringBuilder sb = new StringBuilder(""); + for (String annotation : annotations) { + sb.append(annotation).append("\n"); + } + this.annotation = sb.toString().getBytes(); + } } /** Stores the descriptor to all locations. diff --git a/src/test/resources/bridge-2016-10-02-08-09-00-extra-infos b/src/test/resources/bridge-2016-10-02-08-09-00-extra-infos index f485178..7eea3f8 100644 --- a/src/test/resources/bridge-2016-10-02-08-09-00-extra-infos +++ b/src/test/resources/bridge-2016-10-02-08-09-00-extra-infos @@ -1,4 +1,5 @@ @type bridge-extra-info 1.3 +@some-future annotation extra-info DragonBridge A8D07F92DF78FA2C208FA6675B56E0A7BB750794 master-key-ed25519 jBoYHwHIP9vplYgonlMUFJxND0QMEBqCnL2cEe2NbYc published 2016-10-02 06:09:10 @@ -144,6 +145,13 @@ bridge-ip-transports <OR>=8,obfs3=8,obfs4=48 router-digest-sha256 b/PO7nPdxo+we9Tr/dQt2hAyu1TbtAWdjzac3CWYzpg router-digest BCFCBB38B15E9B500B1A6E9B0BCBBCE858660F17 @type bridge-extra-info 1.3 +@many more annotations +@one +@two +@three +@four +@five and +@last and final extra-info Unnamed 1283EDA3E80CA2D485CCFEDB9715392396DFAD29 master-key-ed25519 5GUbxrSHFtj6QjHHx5ovS2gAh8/ZjtbjU4yjULS0l0s published 2016-10-02 06:09:43 diff --git a/src/test/resources/moria-1048576-2016-10-05.tpf b/src/test/resources/moria-1048576-2016-10-05.tpf index ea05a13..780fa33 100644 --- a/src/test/resources/moria-1048576-2016-10-05.tpf +++ b/src/test/resources/moria-1048576-2016-10-05.tpf @@ -4,7 +4,7 @@ BUILDTIMES=0.658563137054,1.08981704712,1.52948904037 CIRC_ID=1166 CONNECT=14756 BUILDTIMES=0.696699857712,0.833825826645,0.981504917145 CIRC_ID=1172 CONNECT=1475627521.53 DATACOMPLETE=1475627523.30 DATAPERC10=1475627522.45 DATAPERC20=1475627522.57 DATAPERC30=1475627522.68 DATAPERC40=1475627522.73 DATAPERC50=1475627522.85 DATAPERC60=1475627522.93 DATAPERC70=1475627523.04 DATAPERC80=1475627523.05 DATAPERC90=1475627523.21 DATAREQUEST=1475627521.87 DATARESPONSE=1475627522.12 DIDTIMEOUT=0 FILESIZE=1048576 LAUNCH=1475626321.13 NEGOTIATE=1475627521.53 PATH=$B411027C926A9BFFCF7DA91E3CAF1856A321EFFD,$02928855BFAD787F8B363F21860D1E7D156E0655,$95DA61AEF23A6C851028C1AA88AD8593F659E60F QUANTILE=0.800000 READBYTES=1048873 REQUEST=1475627521.53 RESPONSE=1475627521.87 SOCKET=1475627521.53 SOURCE=moria START=1475627521.53 TIMEOUT=1500 USED_AT=1475627523.30 USED_BY=2771 WRITEBYTES=74 @type torperf 1.0 BUILDTIMES=0.574013948441,0.694298028946,0.844352960587 CIRC_ID=1173 CONNECT=1475629321.12 DATACOMPLETE=1475629323.34 DATAPERC10=1475629322.05 DATAPERC20=1475629322.18 DATAPERC30=1475629322.26 DATAPERC40=1475629322.44 DATAPERC50=1475629322.58 DATAPERC60=1475629322.76 DATAPERC70=1475629322.90 DATAPERC80=1475629323.13 DATAPERC90=1475629323.24 DATAREQUEST=1475629321.47 DATARESPONSE=1475629321.71 DIDTIMEOUT=0 FILESIZE=1048576 LAUNCH=1475628121.66 NEGOTIATE=1475629321.12 PATH=$20462CBA5DA4C2D963567D17D0B7249718114A68,$480CCC94CEA04D2DEABC0D7373868E245D4C2AE2,$E1E922A20AF608728824A620BADC6EFC8CB8C2B8 QUANTILE=0.800000 READBYTES=1048873 REQUEST=1475629321.12 RESPONSE=1475629321.47 SOCKET=1475629321.12 SOURCE=moria START=1475629321.12 TIMEOUT=1500 USED_AT=1475629323.35 USED_BY=2773 WRITEBYTES=74 -@type torperf 1.0 +@type torperf 0.9 BUILDTIMES=0.573640108109,0.690929174423,0.906491041183 CIRC_ID=1175 CONNECT=1475631121.93 DATACOMPLETE=1475631124.21 DATAPERC10=1475631122.88 DATAPERC20=1475631123.01 DATAPERC30=1475631123.11 DATAPERC40=1475631123.30 DATAPERC50=1475631123.40 DATAPERC60=1475631123.55 DATAPERC70=1475631123.69 DATAPERC80=1475631123.80 DATAPERC90=1475631123.97 DATAREQUEST=1475631122.27 DATARESPONSE=1475631122.54 DIDTIMEOUT=0 FILESIZE=1048576 LAUNCH=1475629921.18 NEGOTIATE=1475631121.93 PATH=$183C2E314DD677D28CDB6839A0204C4276F6E32D,$360911BEBE036BE7D0BA45B8FC07FF50119EB5B0,$47BAD834309368640EA066913DF741AF413EDE5E QUANTILE=0.800000 READBYTES=1048873 REQUEST=1475631121.93 RESPONSE=1475631122.27 SOCKET=1475631121.93 SOURCE=moria START=1475631121.93 TIMEOUT=1500 USED_AT=1475631124.22 USED_BY=2776 WRITEBYTES=74 @type torperf 1.0 BUILDTIMES=0.578256845474,0.696436882019,0.842072963715 CIRC_ID=1176 CONNECT=1475632921.76 DATACOMPLETE=1475632923.47 DATAPERC10=1475632922.60 DATAPERC20=1475632922.69 DATAPERC30=1475632922.78 DATAPERC40=1475632922.90 DATAPERC50=1475632922.94 DATAPERC60=1475632923.03 DATAPERC70=1475632923.11 DATAPERC80=1475632923.23 DATAPERC90=1475632923.32 DATAREQUEST=1475632922.05 DATARESPONSE=1475632922.29 DIDTIMEOUT=0 FILESIZE=1048576 LAUNCH=1475631721.73 NEGOTIATE=1475632921.76 PATH=$888933294902B310106CF68C012ECDD110114452,$7961C9991F022C8A363FD440CA395D47DB5D44D5,$F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 QUANTILE=0.800000 READBYTES=1048873 REQUEST=1475632921.76 RESPONSE=1475632922.05 SOCKET=1475632921.76 SOURCE=moria START=1475632921.76 TIMEOUT=1500 USED_AT=1475632923.48 USED_BY=2799 WRITEBYTES=74

1 0

[collector/release] Update to metrics-lib 2.1.0 and to Java 8.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit e6d14962867f02408c704edac078ca9c512a4d21 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 15 11:32:12 2017 +0200 Update to metrics-lib 2.1.0 and to Java 8. --- CHANGELOG.md | 3 +++ build.xml | 2 +- src/build | 2 +- src/main/java/org/torproject/collector/index/CreateIndexJson.java | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada02ae..7d70bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 1.?.? - 2017-??-?? + * Major changes + - Update to metrics-lib 2.1.0 and to Java 8. + * Medium changes - When synchronizing descriptors from another CollecTor instance, keep annotations provided by the descriptor and only add the diff --git a/build.xml b/build.xml index 279cd2f..3e162f0 100644 --- a/build.xml +++ b/build.xml @@ -11,7 +11,7 @@ <property name="release.version" value="1.2.1-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.0.0" /> + <property name="metricslibversion" value="2.1.0" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/build b/src/build index b491c84..a64b94a 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit b491c845ab81027d2f29fe541f8303ff151d3d0b +Subproject commit a64b94a635e5396266d5c5b5cf88850b9f4b6a24 diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 4b55d51..e10109b 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -9,8 +9,8 @@ import org.torproject.collector.conf.Key; import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.index.DirectoryNode; import org.torproject.descriptor.index.FileNode; -import org.torproject.descriptor.index.FileType; import org.torproject.descriptor.index.IndexNode; +import org.torproject.descriptor.internal.FileType; import com.google.gson.Gson; import com.google.gson.GsonBuilder;

1 0

[collector/release] Make OnionPerf adhere to the standard CollecTorMain.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit c4ab51e8dcdef8d8adb93d4cba7e35f82e1368ea Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 12 18:47:44 2017 +0000 Make OnionPerf adhere to the standard CollecTorMain. This includes adding property 'OnionPerfSources' and renaming some markers properly. In addition, all camel-case occurrences of 'OnionPerf' have a capitalized 'P' now. Part of task-21759. --- src/main/java/org/torproject/collector/Main.java | 4 ++-- src/main/java/org/torproject/collector/conf/Annotation.java | 2 +- .../java/org/torproject/collector/conf/Configuration.java | 2 +- src/main/java/org/torproject/collector/conf/Key.java | 7 ++++--- .../torproject/collector/exitlists/ExitListDownloader.java | 2 +- .../{OnionperfDownloader.java => OnionPerfDownloader.java} | 11 ++++++----- src/main/resources/collector.properties | 13 +++++++++---- src/test/java/org/torproject/collector/MainTest.java | 2 +- .../org/torproject/collector/conf/ConfigurationTest.java | 2 +- .../org/torproject/collector/cron/CollecTorMainTest.java | 1 + .../java/org/torproject/collector/cron/SchedulerTest.java | 10 +++++----- 11 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java index 6676eb8..1377fec 100644 --- a/src/main/java/org/torproject/collector/Main.java +++ b/src/main/java/org/torproject/collector/Main.java @@ -12,7 +12,7 @@ import org.torproject.collector.cron.Scheduler; import org.torproject.collector.cron.ShutdownHook; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; -import org.torproject.collector.onionperf.OnionperfDownloader; +import org.torproject.collector.onionperf.OnionPerfDownloader; import org.torproject.collector.relaydescs.ArchiveWriter; import org.slf4j.Logger; @@ -50,7 +50,7 @@ public class Main { collecTorMains.put(Key.ExitlistsActivated, ExitListDownloader.class); collecTorMains.put(Key.UpdateindexActivated, CreateIndexJson.class); collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class); - collecTorMains.put(Key.OnionperfActivated, OnionperfDownloader.class); + collecTorMains.put(Key.OnionPerfActivated, OnionPerfDownloader.class); } private static Configuration conf = new Configuration(); diff --git a/src/main/java/org/torproject/collector/conf/Annotation.java b/src/main/java/org/torproject/collector/conf/Annotation.java index e4e63d6..0dab546 100644 --- a/src/main/java/org/torproject/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/collector/conf/Annotation.java @@ -16,7 +16,7 @@ public enum Annotation { Microdescriptor("@type microdescriptor 1.0\n"), Server("@type server-descriptor 1.0\n"), Status("@type bridge-network-status 1.2\n"), - Onionperf("@type torperf 1.0\n"), + OnionPerf("@type torperf 1.0\n"), Vote("@type network-status-vote-3 1.0\n"); private final String annotation; diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java index 4e24a7b..1a62bc1 100644 --- a/src/main/java/org/torproject/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/collector/conf/Configuration.java @@ -92,7 +92,7 @@ public class Configuration extends Observable implements Cloneable { || this.getBool(Key.BridgedescsActivated) || this.getBool(Key.ExitlistsActivated) || this.getBool(Key.UpdateindexActivated) - || this.getBool(Key.OnionperfActivated))) { + || this.getBool(Key.OnionPerfActivated))) { throw new ConfigurationException("Nothing is activated!\n" + "Please edit collector.properties. Exiting."); } diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index e3bcff1..4341432 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -27,6 +27,7 @@ public enum Key { RelaySources(SourceType[].class), BridgeSources(SourceType[].class), ExitlistSources(SourceType[].class), + OnionPerfSources(SourceType[].class), RelayCacheOrigins(String[].class), RelayLocalOrigins(Path.class), RelaySyncOrigins(URL[].class), @@ -42,9 +43,9 @@ public enum Key { RelaydescsActivated(Boolean.class), RelaydescsOffsetMinutes(Integer.class), RelaydescsPeriodMinutes(Integer.class), - OnionperfActivated(Boolean.class), - OnionperfOffsetMinutes(Integer.class), - OnionperfPeriodMinutes(Integer.class), + OnionPerfActivated(Boolean.class), + OnionPerfOffsetMinutes(Integer.class), + OnionPerfPeriodMinutes(Integer.class), UpdateindexActivated(Boolean.class), UpdateindexOffsetMinutes(Integer.class), UpdateindexPeriodMinutes(Integer.class), diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index c27c970..b49189d 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -43,7 +43,7 @@ public class ExitListDownloader extends CollecTorMain { private String recentPathName; - /** Instanciate the exit-lists module using the given configuration. */ + /** Instantiate the exit-lists module using the given configuration. */ public ExitListDownloader(Configuration config) { super(config); this.mapPathDescriptors.put("recent/exit-lists", ExitList.class); diff --git a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java similarity index 97% rename from src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java rename to src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java index 577d16d..b2a27d4 100644 --- a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java +++ b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java @@ -41,14 +41,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; /** Download download .tpf files from OnionPerf hosts. */ -public class OnionperfDownloader extends CollecTorMain { +public class OnionPerfDownloader extends CollecTorMain { private static final Logger logger = LoggerFactory.getLogger( - OnionperfDownloader.class); + OnionPerfDownloader.class); private static final String TORPERF = "torperf"; - public OnionperfDownloader(Configuration config) { + /** Instantiate the OnionPerf module using the given configuration. */ + public OnionPerfDownloader(Configuration config) { super(config); } @@ -72,12 +73,12 @@ public class OnionperfDownloader extends CollecTorMain { @Override public String module() { - return TORPERF; + return "onionperf"; } @Override protected String syncMarker() { - return "TorperfFiles"; + return "OnionPerf"; } @Override diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 54caf5a..d49bd58 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -30,11 +30,11 @@ RelaydescsPeriodMinutes = 30 # offset in minutes since the epoch and RelaydescsOffsetMinutes = 5 ## the following defines, if this module is activated -OnionperfActivated = false +OnionPerfActivated = false # period in minutes -OnionperfPeriodMinutes = 360 +OnionPerfPeriodMinutes = 360 # offset in minutes since the epoch and -OnionperfOffsetMinutes = 1 +OnionPerfOffsetMinutes = 1 # the following defines, if this module is activated UpdateindexActivated = false # period in minutes @@ -139,7 +139,11 @@ ExitlistSyncOrigins = https://collector.torproject.org ## Where to download exit-lists from. ExitlistUrl = https://check.torproject.org/exit-addresses # -######## Onionperf downloader ######## +######## OnionPerf downloader ######## +# +## Define descriptor sources +# possible values: Remote +OnionPerfSources = Remote # ## OnionPerf base URLs ## Hosts must be configured to use the first subdomain part of the given URL as @@ -147,3 +151,4 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ## the second, etc.: ## OnionPerfHosts = http://first.torproject.org/, http://second.torproject.org/ OnionPerfHosts = https://op-us.onionperf.torproject.net/ + diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java index 2e9c0ce..b9cf5f8 100644 --- a/src/test/java/org/torproject/collector/MainTest.java +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -100,7 +100,7 @@ public class MainTest { Main.main(new String[]{conf.toString()}); assertTrue(4_000L <= conf.length()); changeFilePathsAndSetActivation(conf, - Key.OnionperfActivated.name()); + Key.OnionPerfActivated.name()); Main.main(new String[]{conf.toString()}); waitSec(2); } diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index a0c1dd1..bbe7832 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -40,7 +40,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 43, Key.values().length); + 44, Key.values().length); } @Test() diff --git a/src/test/java/org/torproject/collector/cron/CollecTorMainTest.java b/src/test/java/org/torproject/collector/cron/CollecTorMainTest.java index dc3e575..49f07ef 100644 --- a/src/test/java/org/torproject/collector/cron/CollecTorMainTest.java +++ b/src/test/java/org/torproject/collector/cron/CollecTorMainTest.java @@ -70,6 +70,7 @@ public class CollecTorMainTest { case "Relay": case "Bridge": case "Exitlist": + case "OnionPerf": assertNotNull("Property '" + key + "' not specified in " + Main.CONF_FILE + ".", props.getProperty(key)); diff --git a/src/test/java/org/torproject/collector/cron/SchedulerTest.java b/src/test/java/org/torproject/collector/cron/SchedulerTest.java index 63eb5af..062009f 100644 --- a/src/test/java/org/torproject/collector/cron/SchedulerTest.java +++ b/src/test/java/org/torproject/collector/cron/SchedulerTest.java @@ -22,8 +22,8 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; public class SchedulerTest { private static final String runConfigProperties = - "OnionperfActivated=true\nOnionperfPeriodMinutes=1\n" - + "OnionperfOffsetMinutes=0\n" + "OnionPerfActivated=true\nOnionPerfPeriodMinutes=1\n" + + "OnionPerfOffsetMinutes=0\n" + "RelaydescsActivated=true\nRelaydescsPeriodMinutes=1\n" + "RelaydescsOffsetMinutes=0\n" + "ExitlistsActivated=true\nExitlistsPeriodMinutes=1\n" @@ -38,7 +38,7 @@ public class SchedulerTest { Map<Key, Class<? extends CollecTorMain>> ctms = new HashMap<>(); Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream(runConfigProperties.getBytes())); - ctms.put(Key.OnionperfActivated, Dummy.class); + ctms.put(Key.OnionPerfActivated, Dummy.class); ctms.put(Key.BridgedescsActivated, Dummy.class); ctms.put(Key.RelaydescsActivated, Dummy.class); ctms.put(Key.ExitlistsActivated, Dummy.class); @@ -70,7 +70,7 @@ public class SchedulerTest { conf.load(new ByteArrayInputStream(("ShutdownGraceWaitMinutes=1\n" + runConfigProperties).getBytes())); conf.setProperty(Key.RunOnce.name(), "true"); - ctms.put(Key.OnionperfActivated, Counter.class); + ctms.put(Key.OnionPerfActivated, Counter.class); ctms.put(Key.BridgedescsActivated, Counter.class); ctms.put(Key.RelaydescsActivated, Counter.class); ctms.put(Key.ExitlistsActivated, Counter.class); @@ -93,7 +93,7 @@ public class SchedulerTest { Map<Key, Class<? extends CollecTorMain>> ctms = new HashMap<>(); Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream(runConfigProperties.getBytes())); - ctms.put(Key.OnionperfActivated, Broken.class); + ctms.put(Key.OnionPerfActivated, Broken.class); ctms.put(Key.BridgedescsActivated, Broken.class); ctms.put(Key.RelaydescsActivated, Broken.class); ctms.put(Key.ExitlistsActivated, Broken.class);

1 0

[collector/release] Add the build revision to index.json files.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 0a586c0be0a3d3f399f550ec3929a04ad4c71904 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Sep 20 10:15:32 2017 +0000 Add the build revision to index.json files. Implements the final part of task-21414 for CollecTor. --- CHANGELOG.md | 5 +++++ build.xml | 2 +- src/build | 2 +- .../torproject/collector/index/CreateIndexJson.java | 18 +++++++++++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0810fcf..4973f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ - Add "Sync" as OnionPerfSource to synchronize .tpf files from other CollecTor instances. + * Medium changes + - Add new optional "build_revision" field to index.json with the + Git revision of the CollecTor instance's software used to create + this file, which will be omitted if unknown. + * Minor changes - Remove all styling resources (fonts, CSS, etc.) from directory listings. diff --git a/build.xml b/build.xml index 943c717..d60350b 100644 --- a/build.xml +++ b/build.xml @@ -11,7 +11,7 @@ <property name="release.version" value="1.3.0-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.1.0" /> + <property name="metricslibversion" value="2.1.1" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/build b/src/build index a64b94a..78e533f 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit a64b94a635e5396266d5c5b5cf88850b9f4b6a24 +Subproject commit 78e533fdc84e20769a8a155d1426cbea59c6e7cc diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index e10109b..5c4daf9 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -21,10 +21,12 @@ import org.slf4j.LoggerFactory; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; +import java.io.InputStream; import java.io.OutputStreamWriter; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Locale; +import java.util.Properties; import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeSet; @@ -53,10 +55,23 @@ public class CreateIndexJson extends CollecTorMain { private static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC"); + private static String buildRevision = null; + /** Creates indexes of directories containing archived and recent * descriptors and write index files to disk. */ public CreateIndexJson(Configuration conf) { super(conf); + Properties buildProperties = new Properties(); + try (InputStream is = getClass().getClassLoader() + .getResourceAsStream("collector.buildrevision.properties")) { + buildProperties.load(is); + buildRevision = buildProperties.getProperty("collector.build.revision", + null); + } catch (Exception ex) { + // This doesn't hamper the index creation: only log a warning. + logger.warn("No build revision available.", ex); + buildRevision = null; + } } @Override @@ -107,7 +122,8 @@ public class CreateIndexJson extends CollecTorMain { } } return new IndexNode(dateTimeFormat.format( - System.currentTimeMillis()), basePath, null, directoryNodes); + System.currentTimeMillis()), buildRevision, basePath, null, + directoryNodes); } private DirectoryNode indexDirectory(File directory) {

1 0

[collector/release] Enable OnionPerf 'Sync' runs.
by karsten＠torproject.org 10 Oct '17

10 Oct '17

commit 8ffdfd6c2cd4a1d6a6f0660681e9a788b88dc342 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 12 18:47:45 2017 +0000 Enable OnionPerf 'Sync' runs. --- .../org/torproject/collector/conf/Annotation.java | 2 +- .../java/org/torproject/collector/conf/Key.java | 1 + .../collector/onionperf/OnionPerfDownloader.java | 1 + .../collector/persist/OnionPerfPersistence.java | 49 ++++++++++++++++++++++ .../torproject/collector/sync/SyncPersistence.java | 5 +++ src/main/resources/collector.properties | 5 ++- 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Annotation.java b/src/main/java/org/torproject/collector/conf/Annotation.java index 0dab546..23f485c 100644 --- a/src/main/java/org/torproject/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/collector/conf/Annotation.java @@ -16,7 +16,7 @@ public enum Annotation { Microdescriptor("@type microdescriptor 1.0\n"), Server("@type server-descriptor 1.0\n"), Status("@type bridge-network-status 1.2\n"), - OnionPerf("@type torperf 1.0\n"), + OnionPerf("@type torperf 1.1\n"), Vote("@type network-status-vote-3 1.0\n"); private final String annotation; diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 4341432..72af4bb 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -34,6 +34,7 @@ public enum Key { BridgeSyncOrigins(URL[].class), BridgeLocalOrigins(Path.class), ExitlistSyncOrigins(URL[].class), + OnionPerfSyncOrigins(URL[].class), BridgedescsActivated(Boolean.class), BridgedescsOffsetMinutes(Integer.class), BridgedescsPeriodMinutes(Integer.class), diff --git a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java index b2a27d4..36b006a 100644 --- a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java @@ -51,6 +51,7 @@ public class OnionPerfDownloader extends CollecTorMain { /** Instantiate the OnionPerf module using the given configuration. */ public OnionPerfDownloader(Configuration config) { super(config); + this.mapPathDescriptors.put("recent/torperf", TorperfResult.class); } /** File containing the download history, which is necessary, because diff --git a/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java new file mode 100644 index 0000000..22093eb --- /dev/null +++ b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java @@ -0,0 +1,49 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.persist; + +import org.torproject.collector.conf.Annotation; +import org.torproject.descriptor.TorperfResult; + +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +public class OnionPerfPersistence + extends DescriptorPersistence<TorperfResult> { + + private static final String ONIONPERF = "torperf"; + + public OnionPerfPersistence(TorperfResult desc) { + super(desc, Annotation.OnionPerf.bytes()); + calculatePaths(); + } + + private void calculatePaths() { + String[] parts = PersistenceUtils.dateTimeParts(desc.getStartMillis()); + String name = desc.getSource() + DASH + desc.getFileSize() + DASH + + parts[0] + DASH + parts[1] + DASH + parts[2] + ".tpf"; + this.recentPath = Paths.get(ONIONPERF, name).toString(); + this.storagePath = Paths.get( + ONIONPERF, + parts[0], // year + parts[1], // month + parts[2], // day + name).toString(); + } + + /** OnionPerf default storage appends. */ + @Override + public boolean storeOut(String outRoot) { + return super.storeOut(outRoot, StandardOpenOption.APPEND); + } + + /** OnionPerf default storage appends. */ + @Override + public boolean storeAll(String recentRoot, String outRoot) { + return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND, + StandardOpenOption.APPEND); + } + +} + diff --git a/src/main/java/org/torproject/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/collector/sync/SyncPersistence.java index 5fd6dc6..baca83d 100644 --- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java @@ -13,6 +13,7 @@ import org.torproject.collector.persist.DescriptorPersistence; import org.torproject.collector.persist.ExitlistPersistence; import org.torproject.collector.persist.ExtraInfoPersistence; import org.torproject.collector.persist.MicroConsensusPersistence; +import org.torproject.collector.persist.OnionPerfPersistence; import org.torproject.collector.persist.PersistenceUtils; import org.torproject.collector.persist.ServerDescriptorPersistence; import org.torproject.collector.persist.StatusPersistence; @@ -26,6 +27,7 @@ import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.RelayNetworkStatusVote; import org.torproject.descriptor.RelayServerDescriptor; +import org.torproject.descriptor.TorperfResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +129,9 @@ public class SyncPersistence { case "ExitList": // downloaded is part of desc, which to use? descPersist = new ExitlistPersistence((ExitList) desc, received); break; + case "TorperfResult": + descPersist = new OnionPerfPersistence((TorperfResult) desc); + break; default: log.trace("Invalid descriptor type {} for sync-merge.", clazz.getName()); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index d49bd58..0a9f932 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -142,8 +142,11 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ######## OnionPerf downloader ######## # ## Define descriptor sources -# possible values: Remote +# possible values: Remote,Sync OnionPerfSources = Remote +# Retrieve files from the following CollecTor instances. +# List of URLs separated by comma. +OnionPerfSyncOrigins = https://collector.torproject.org # ## OnionPerf base URLs ## Hosts must be configured to use the first subdomain part of the given URL as

1 0