November 2016 - tor-commits - lists.torproject.org

[tor/maint-0.2.9] Changes file for bug 20634
by nickm＠torproject.org 18 Nov '16

18 Nov '16

commit 44f3563d872b56593c11a6e557b130714fa8916b Author: teor <teor2345(a)gmail.com> Date: Wed Nov 16 13:08:39 2016 +1100 Changes file for bug 20634 --- changes/bug20634 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/bug20634 b/changes/bug20634 new file mode 100644 index 0000000..62fc9f4 --- /dev/null +++ b/changes/bug20634 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Stop spurious failures in the local interface address discovery unit + tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by Neel Chauhan.

1 0

[tor-messenger-build/master] Include mar files in sha256sums-unsigned-build.txt
by boklm＠torproject.org 17 Nov '16

17 Nov '16

commit 5b10e2bc56ba6afb46c801c5f3cb34fb731b190c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Nov 17 23:50:49 2016 +0100 Include mar files in sha256sums-unsigned-build.txt --- projects/tor-messenger-release/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tor-messenger-release/config b/projects/tor-messenger-release/config index 885eb35..2258d3c 100644 --- a/projects/tor-messenger-release/config +++ b/projects/tor-messenger-release/config @@ -39,5 +39,5 @@ build: | mv [% c('input_files_by_name/linux-i686') %]/* "$destdir"/ mv [% c('input_files_by_name/linux-x86_64') %]/* "$destdir"/ cd "$destdir" - sha256sum $(ls -1 *.exe *.tar.xz *.dmg | sort) > sha256sums-unsigned-build.txt + sha256sum $(ls -1 *.exe *.tar.xz *.dmg *.mar | grep -v '\.incremental\.mar$' | sort) > sha256sums-unsigned-build.txt cat sha256sums-unsigned-build.txt

1 0

[tor-messenger-build/master] Rename sha256sums.txt to sha256sums-unsigned-build.txt
by boklm＠torproject.org 17 Nov '16

17 Nov '16

commit e3b8b432ca16af389665693d350c15497dd5c1c7 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Nov 17 23:48:44 2016 +0100 Rename sha256sums.txt to sha256sums-unsigned-build.txt --- projects/tor-messenger-release/config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/tor-messenger-release/config b/projects/tor-messenger-release/config index aa19275..885eb35 100644 --- a/projects/tor-messenger-release/config +++ b/projects/tor-messenger-release/config @@ -39,5 +39,5 @@ build: | mv [% c('input_files_by_name/linux-i686') %]/* "$destdir"/ mv [% c('input_files_by_name/linux-x86_64') %]/* "$destdir"/ cd "$destdir" - sha256sum $(ls -1 *.exe *.tar.xz *.dmg | sort) > sha256sums.txt - cat sha256sums.txt + sha256sum $(ls -1 *.exe *.tar.xz *.dmg | sort) > sha256sums-unsigned-build.txt + cat sha256sums-unsigned-build.txt

1 0

[collector/master] Added tests for ReferenceChecker; prepares task-20695.
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit aa87253196c7134762a2b2f4d047ad6d2b58326d Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Nov 17 18:22:07 2016 +0100 Added tests for ReferenceChecker; prepares task-20695. --- .../collector/relaydescs/ReferenceCheckerTest.java | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/test/java/org/torproject/collector/relaydescs/ReferenceCheckerTest.java b/src/test/java/org/torproject/collector/relaydescs/ReferenceCheckerTest.java new file mode 100644 index 0000000..6ed3501 --- /dev/null +++ b/src/test/java/org/torproject/collector/relaydescs/ReferenceCheckerTest.java @@ -0,0 +1,78 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.relaydescs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.nio.charset.Charset; +import java.nio.file.Files; + +public class ReferenceCheckerTest { + + @Rule + public TemporaryFolder tmpf = new TemporaryFolder(); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private static final String validReferenceJson + = "[{\"referencing\":\"C-2016-09-17T03:00:00Z\"," + + "\"referenced\":\"S-D8736FB5916322CB4B0FC34FA9DA3D8ACBBCE352\"," + + "\"weight\":0.028224668360146768," + + "\"expiresAfterMillis\":33455631600000}]"; + + @Test() + public void testValidJson() throws Exception { + File descDir = tmpf.newFolder(); + File refsFile = tmpf.newFile(); + File histFile = tmpf.newFile(); + Files.write(refsFile.toPath(), validReferenceJson.getBytes()); + assertEquals(validReferenceJson, Files.readAllLines(refsFile.toPath(), + Charset.forName("US-ASCII")).get(0)); + ReferenceChecker rc = new ReferenceChecker(descDir, refsFile, histFile); + rc.check(); + assertTrue(refsFile.exists()); + assertEquals(validReferenceJson, Files.readAllLines(refsFile.toPath(), + Charset.forName("US-ASCII")).get(0)); + } + + @Test() + public void testInvalidJson() throws Exception { + File descDir = tmpf.newFolder(); + File refsFile = tmpf.newFile(); + File histFile = tmpf.newFile(); + String badJson = "[{\"xx\":7]}"; + Files.write(refsFile.toPath(), badJson.getBytes()); + assertEquals(badJson, Files.readAllLines(refsFile.toPath(), + Charset.forName("US-ASCII")).get(0)); + ReferenceChecker rc = new ReferenceChecker(descDir, refsFile, histFile); + rc.check(); + assertTrue(refsFile.exists()); + assertEquals("[]", Files.readAllLines(refsFile.toPath(), + Charset.forName("US-ASCII")).get(0)); + } + + @Test() + public void testMinimalValidJson() throws Exception { + File descDir = tmpf.newFolder(); + File refsFile = tmpf.newFile(); + File histFile = tmpf.newFile(); + Files.write(refsFile.toPath(), "[]".getBytes()); + ReferenceChecker rc = new ReferenceChecker(descDir, refsFile, histFile); + rc.check(); + assertTrue(refsFile.exists()); + assertEquals("The file's contents shouldn't have changed, but did.", "[]", + Files.readAllLines(refsFile.toPath(), Charset.forName("US-ASCII")) + .get(0)); + } + +} +

1 0

[collector/master] Makes all tests pass. Implements task-20695.
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit 74537ca0f92f31fc1d0053a31537820f9bc603d9 Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Nov 17 18:23:41 2016 +0100 Makes all tests pass. Implements task-20695. --- .../collector/relaydescs/ReferenceChecker.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java index 9c76216..dead416 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java +++ b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -102,6 +103,8 @@ public class ReferenceChecker { private long expiresAfterMillis; + public Reference() { /* empty */ } + public Reference(String referencing, String referenced, double weight, long expiresAfterMillis) { this.referencing = referencing; @@ -140,14 +143,24 @@ public class ReferenceChecker { return; } Gson gson = new Gson(); - try { - FileReader fr = new FileReader(this.referencesFile); + try (FileReader fr = new FileReader(this.referencesFile)) { this.references.addAll(Arrays.asList(gson.fromJson(fr, Reference[].class))); - fr.close(); } catch (IOException e) { logger.warn("Cannot read existing references file " + "from previous run.", e); + } catch (RuntimeException jpe) { + logger.warn("Content of {} cannot be parsed. " + + "File will be erased and rewritten. In general, {} " + + "shouldn't be edited manually. Error reason: {}", + this.referencesFile.toString(), + this.referencesFile.toString(), jpe.getMessage()); + try { + Files.deleteIfExists(this.referencesFile.toPath()); + } catch (IOException ioe) { + logger.warn("Cannot delete '{}', reason: {}", + this.referencesFile.toString(), ioe.getMessage(), ioe); + } } }

1 0

[stem/master] Stem release 1.5.1
by atagar＠torproject.org 17 Nov '16

17 Nov '16

commit 753b196c23710fbac99225ad768212ab5a3f93a0 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Nov 15 09:42:05 2016 -0800 Stem release 1.5.1 Damnit PyPI, you cannot legitimately claim this isn't a bug. Long story short if a file is improperly uploaded that version is left in an unfixable state, forcing us to do a version bump. https://github.com/pypa/packaging-problems/issues/74#issuecomment-260716129 No changes to Stem here. Just bumping the version to get around their bug. --- docs/index.rst | 2 +- stem/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 79e106d..eb58df2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Welcome to Stem! ================ -Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.0** (released November 14th, 2016). +Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.1** (released November 15th, 2016). .. Main Stem Logo Source: http://www.wpclipart.com/plants/assorted/P/plant_stem.png.html diff --git a/stem/__init__.py b/stem/__init__.py index 6ce1b4c..47b9a8f 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -470,7 +470,7 @@ Library for working with the tor process. import stem.util.enum -__version__ = '1.5.0' +__version__ = '1.5.1' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[collector/master] Addendum to task-20562: File 'nginx-collector' moved to src/main/resources
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit c32cb0bdbc8a49df73d0b6fed6ebcb2d88b40154 Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Nov 17 14:35:08 2016 +0100 Addendum to task-20562: File 'nginx-collector' moved to src/main/resources as it is not part of the webapp files. Adapted path in INSTALL.md to match the new location. --- INSTALL.md | 2 +- src/main/resources/nginx-collector | 33 +++++++++++++++++++++++++++++++++ src/main/webapp/nginx-collector | 33 --------------------------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9e3cf1a..7e6c4d4 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -167,7 +167,7 @@ refer to settings in `collector.properties`): You can also configure nginx as the web server of your choice. If you use nginx you will need to use the FancyIndex module to be able to include the provided footer and header of the webapp. Copy -`collector-<version>/src/main/webapp/nginx-collector` to +`collector-<version>/src/main/resources/nginx-collector` to `/etc/nginx/sites-available/` and make a symbolic link in `/etc/nginx/sites-enabled/` to enable it. diff --git a/src/main/resources/nginx-collector b/src/main/resources/nginx-collector new file mode 100644 index 0000000..fda655a --- /dev/null +++ b/src/main/resources/nginx-collector @@ -0,0 +1,33 @@ +## +# The following is a simple nginx configuration to run CollecTor. +## +server { + + # We assume the you have copied the content of webapp to + # /var/www/collector/html + + root /var/www/collector/html; + + # This option make sure that nginx will follow symlinks to the appropriate + # CollecTor folders + autoindex on; + + index index.html; + + # You can change the specified servername. If you leave it as it is you will + # need to add an entry on /etc/hosts for collector + server_name collector; + + location / { + fancyindex on; + fancyindex_exact_size off; + fancyindex_header /header.html; + fancyindex_footer /footer.html; + try_files $uri $uri/ =404; + } + + location ~/\.ht { + deny all; + } +} + diff --git a/src/main/webapp/nginx-collector b/src/main/webapp/nginx-collector deleted file mode 100644 index fda655a..0000000 --- a/src/main/webapp/nginx-collector +++ /dev/null @@ -1,33 +0,0 @@ -## -# The following is a simple nginx configuration to run CollecTor. -## -server { - - # We assume the you have copied the content of webapp to - # /var/www/collector/html - - root /var/www/collector/html; - - # This option make sure that nginx will follow symlinks to the appropriate - # CollecTor folders - autoindex on; - - index index.html; - - # You can change the specified servername. If you leave it as it is you will - # need to add an entry on /etc/hosts for collector - server_name collector; - - location / { - fancyindex on; - fancyindex_exact_size off; - fancyindex_header /header.html; - fancyindex_footer /footer.html; - try_files $uri $uri/ =404; - } - - location ~/\.ht { - deny all; - } -} -

1 0

[onionoo/master] Cleanup .gitignore.
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit c8d884254a46505ddc7b065f2c7defff029f78ea Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Nov 15 11:22:09 2016 +0100 Cleanup .gitignore. --- .gitignore | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 80c5ae5..ac1e6d0 100755 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,10 @@ +/generated/ +/lib/ +/geoip/ +/in/ +/out/ +/status/ +*~ .classpath .project -classes/ -etc/context.xml -etc/web.xml -geoip/ -in/ -dist/ -out/ -status/

1 0

[onionoo/master] Make tests pass. Implements task-19259.
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit 73856c4b07db31c35c23309760016352644f4b75 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Nov 15 11:26:29 2016 +0100 Make tests pass. Implements task-19259. --- .../org/torproject/onionoo/docs/WeightsStatus.java | 67 ++++++++++++++-------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index acd44a1..ce8d915 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -29,16 +29,19 @@ public class WeightsStatus extends Document { this.isDirty = false; } - private SortedMap<long[], double[]> history = - new TreeMap<long[], double[]>( - new Comparator<long[]>() { - public int compare(long[] first, long[] second) { - return first[0] < second[0] ? -1 - : first[0] > second[0] ? 1 - : 0; + private Comparator<long[]> histComparator = new Comparator<long[]>() { + public int compare(long[] first, long[] second) { + int relation = Long.compare(first[0], second[0]); + if (0 != relation) { + return relation; + } else { + return Long.compare(first[1], second[1]); } } - ); + }; + + private SortedMap<long[], double[]> history = + new TreeMap<long[], double[]>(histComparator); public void setHistory(SortedMap<long[], double[]> history) { this.history = history; @@ -53,7 +56,7 @@ public class WeightsStatus extends Document { try (Scanner s = new Scanner(documentString)) { while (s.hasNextLine()) { String line = s.nextLine(); - String[] parts = line.split(" "); + String[] parts = line.split(" ", 11); if (parts.length == 2) { /* Skip lines containing descriptor digest and advertised * bandwidth. */ @@ -69,30 +72,43 @@ public class WeightsStatus extends Document { * after. */ continue; } - long validAfterMillis = DateTimeHelper.parse(parts[0] + " " - + parts[1]); - long freshUntilMillis = DateTimeHelper.parse(parts[2] + " " - + parts[3]); + long validAfterMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]); + long freshUntilMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]); if (validAfterMillis < 0L || freshUntilMillis < 0L) { log.error("Could not parse timestamp while reading " + "weights status file. Skipping."); break; } - long[] interval = new long[] { validAfterMillis, - freshUntilMillis }; - double[] weights = new double[] { -1.0, - Double.parseDouble(parts[5]), - Double.parseDouble(parts[6]), - Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]), -1.0, -1.0 }; - if (parts.length == 11) { - weights[6] = Double.parseDouble(parts[10]); + if (validAfterMillis > freshUntilMillis) { + log.error("Illegal dates in '" + line + "' of weights " + + "status file. Skipping."); + break; + } + long[] interval = new long[] { validAfterMillis, freshUntilMillis }; + double[] weights; + try { + weights = new double[] { -1.0, + parseWeightDouble(parts[5]), + parseWeightDouble(parts[6]), + parseWeightDouble(parts[7]), + parseWeightDouble(parts[8]), -1.0, -1.0 }; + if (parts.length == 11) { + weights[6] = parseWeightDouble(parts[10]); + } + } catch (NumberFormatException e) { + log.error("Could not parse weights values in line '" + line + + "' while reading weights status file. Skipping."); + break; } this.history.put(interval, weights); } } } + private double parseWeightDouble(String in) throws NumberFormatException { + return in.isEmpty() ? Double.NaN : Double.parseDouble(in); + } + /** Adds all given weights history objects that don't overlap with * existing weights history objects. */ public void addToHistory(long validAfterMillis, long freshUntilMillis, @@ -113,7 +129,8 @@ public class WeightsStatus extends Document { * intervals, depending on how far back in the past they lie. */ public void compressHistory() { SortedMap<long[], double[]> uncompressedHistory = - new TreeMap<long[], double[]>(this.history); + new TreeMap<>(histComparator); + uncompressedHistory.putAll(this.history); history.clear(); long lastStartMillis = 0L; long lastEndMillis = 0L; @@ -144,7 +161,7 @@ public class WeightsStatus extends Document { DateTimeHelper.ISO_YEARMONTH_FORMAT); int missingValues = 0; for (int i = 0; i < weights.length; i++) { - if (weights[i] < -0.5) { + if (Double.valueOf(weights[i]).isNaN()) { missingValues += 1 << i; } } @@ -193,7 +210,7 @@ public class WeightsStatus extends Document { + DateTimeHelper.format(fresh[1])); for (int i = 0; i < weights.length; i++) { sb.append(" "); - if (i != 0 && i != 5) { + if (i != 0 && i != 5 && !Double.valueOf(weights[i]).isNaN()) { sb.append(String.format("%.12f", weights[i])); } }

1 0

[onionoo/master] Part of task-19259 implementation.
by karsten＠torproject.org 17 Nov '16

17 Nov '16

commit c54a3f7f3f603a7f11b46b606405e8c411c8f299 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Nov 16 15:03:48 2016 +0100 Part of task-19259 implementation. Make test for old-style weight status docs pass. --- src/main/java/org/torproject/onionoo/docs/WeightsStatus.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index ce8d915..50b8b35 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -106,7 +106,8 @@ public class WeightsStatus extends Document { } private double parseWeightDouble(String in) throws NumberFormatException { - return in.isEmpty() ? Double.NaN : Double.parseDouble(in); + return in.isEmpty() || in.startsWith("-") + ? Double.NaN : Double.parseDouble(in); } /** Adds all given weights history objects that don't overlap with

1 0