August 2017 - tor-commits - lists.torproject.org

[nyx/master] Run static checks in the background
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit c03d05abdec29eebba2d3ed0f15c7e5f46aa658b Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 16 07:46:03 2017 -0700 Run static checks in the background Stem got quite a substantial speedup in its tests by running long-running tasks in the background. Nyx is a much smaller test suite so not much to gain, but might as well improve things a tad here too. Running pyflakes and pycodestyle in the background. On my laptop this drops the test runtime from 3.8s to 2.4s (37% faster). --- run_tests.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/run_tests.py b/run_tests.py index 2b76561..e115552 100755 --- a/run_tests.py +++ b/run_tests.py @@ -9,6 +9,7 @@ render to the screen. """ import os +import multiprocessing import unittest import stem.util.conf @@ -27,6 +28,12 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in ( )] +def _run_wrapper(conn, runner, args): + os.nice(15) + conn.send(runner(*args) if args else runner()) + conn.close() + + @nyx.uses_settings def main(): nyx.TESTING = True @@ -38,7 +45,20 @@ def main(): for path in orphaned_pyc: print('Deleted orphaned pyc file: %s' % path) - tests = unittest.defaultTestLoader.discover('test', pattern='*.py') + pyflakes_task, pyflakes_pipe = None, None + pycodestyle_task, pycodestyle_pipe = None, None + + if stem.util.test_tools.is_pyflakes_available(): + pyflakes_pipe, child_pipe = multiprocessing.Pipe() + pyflakes_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.pyflakes_issues, (SRC_PATHS,))) + pyflakes_task.start() + + if stem.util.test_tools.is_pep8_available(): + pycodestyle_pipe, child_pipe = multiprocessing.Pipe() + pycodestyle_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.stylistic_issues, (SRC_PATHS, True, True, True))) + pycodestyle_task.start() + + tests = unittest.defaultTestLoader.discover('test', pattern = '*.py') test_runner = unittest.TextTestRunner() test_runner.run(tests) @@ -46,22 +66,19 @@ def main(): static_check_issues = {} - if stem.util.test_tools.is_pyflakes_available(): - pyflakes_issues = stem.util.test_tools.pyflakes_issues(SRC_PATHS) + if pyflakes_task: + pyflakes_issues = pyflakes_pipe.recv() + pyflakes_task.join() for path, issues in pyflakes_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) - if stem.util.test_tools.is_pep8_available(): - pep8_issues = stem.util.test_tools.stylistic_issues( - SRC_PATHS, - check_newlines = True, - check_exception_keyword = True, - prefer_single_quotes = True, - ) - - for path, issues in pep8_issues.items(): + if pycodestyle_task: + pycodestyle_issues = pycodestyle_pipe.recv() + pycodestyle_task.join() + + for path, issues in pycodestyle_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue)

1 0

[nyx/master] Move testing to DaemonTask
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit f29efbcb828ad7a783cbf42546730abbfdf7b9b7 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 18 08:10:15 2017 -0700 Move testing to DaemonTask Much nicer than using multiprocessing directly. --- run_tests.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/run_tests.py b/run_tests.py index e115552..d923178 100755 --- a/run_tests.py +++ b/run_tests.py @@ -9,10 +9,10 @@ render to the screen. """ import os -import multiprocessing import unittest import stem.util.conf +import stem.util.system import stem.util.test_tools import nyx @@ -28,12 +28,6 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in ( )] -def _run_wrapper(conn, runner, args): - os.nice(15) - conn.send(runner(*args) if args else runner()) - conn.close() - - @nyx.uses_settings def main(): nyx.TESTING = True @@ -45,18 +39,13 @@ def main(): for path in orphaned_pyc: print('Deleted orphaned pyc file: %s' % path) - pyflakes_task, pyflakes_pipe = None, None - pycodestyle_task, pycodestyle_pipe = None, None + pyflakes_task, pycodestyle_task = None, None if stem.util.test_tools.is_pyflakes_available(): - pyflakes_pipe, child_pipe = multiprocessing.Pipe() - pyflakes_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.pyflakes_issues, (SRC_PATHS,))) - pyflakes_task.start() + pyflakes_task = stem.util.system.DaemonTask(stem.util.test_tools.pyflakes_issues, (SRC_PATHS,), start = True) if stem.util.test_tools.is_pep8_available(): - pycodestyle_pipe, child_pipe = multiprocessing.Pipe() - pycodestyle_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.stylistic_issues, (SRC_PATHS, True, True, True))) - pycodestyle_task.start() + pycodestyle_task = stem.util.system.DaemonTask(stem.util.test_tools.stylistic_issues, (SRC_PATHS,), start = True) tests = unittest.defaultTestLoader.discover('test', pattern = '*.py') test_runner = unittest.TextTestRunner() @@ -67,18 +56,12 @@ def main(): static_check_issues = {} if pyflakes_task: - pyflakes_issues = pyflakes_pipe.recv() - pyflakes_task.join() - - for path, issues in pyflakes_issues.items(): + for path, issues in pyflakes_task.join().items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) if pycodestyle_task: - pycodestyle_issues = pycodestyle_pipe.recv() - pycodestyle_task.join() - - for path, issues in pycodestyle_issues.items(): + for path, issues in pycodestyle_task.join().items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue)

1 0

[nyx/master] Don't clear screen in main draw loop
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit 737c26c2fbc8bcfbb21c02db17a87dd082552ba9 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 19 04:43:13 2017 -0700 Don't clear screen in main draw loop To avoid screen hiccups from persisting we cleared the screen on every redraw... https://trac.torproject.org/projects/tor/ticket/2830 This seemed fine on my laptop since I wasn't running a real relay, but under actual load it causes nyx to flicker annoyingly every few seconds. This was a hack in case of anomalies. Cure was worse than the disease. We might need to revisit this (... again), but for now this makes the experience of running nyx on my relay much nicer. --- nyx/__init__.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index bb1d074..5cd4a22 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -432,23 +432,15 @@ class Interface(object): for panel in self.page_panels(): panel.redraw() - def redraw(self, force = True): + def redraw(self): """ Renders our displayed content. - - :param bool force: if **False** only redraws content if resized """ - # Curses may overly cache content without clearing here... - # https://trac.torproject.org/projects/tor/ticket/2830#comment:9 - - if force: - nyx.curses.clear() - occupied = 0 for panel in self.page_panels(): - panel.redraw(force = force, top = occupied) + panel.redraw(force = False, top = occupied) occupied += panel.get_height() def quit(self):

1 0

[collector/master] Bump version to 1.2.1-dev.
by karsten＠torproject.org 18 Aug '17

18 Aug '17

commit d162d90299a1d27fc8750832b56e60f3a5c97b66 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Aug 18 15:03:03 2017 +0200 Bump version to 1.2.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index ec0eb41..279cd2f 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.2.1" /> + <property name="release.version" value="1.2.1-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.0.0" />

1 0

[collector/master] Prepare for 1.2.1 release.
by karsten＠torproject.org 18 Aug '17

18 Aug '17

commit faaa1fae3ef0fdb4568be5361e20fa7a314727bb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 17 20:43:51 2017 +0200 Prepare for 1.2.1 release. --- CERT | 18 +++++++++--------- CHANGELOG.md | 2 +- build.xml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CERT b/CERT index 6cc5a93..d94c0c3 100644 --- a/CERT +++ b/CERT @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIDaTCCAlGgAwIBAgIEYQnM/zANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV +MIIDaTCCAlGgAwIBAgIEZTniETANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV UzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBU b3IgUHJvamVjdCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwHhcNMTcw -MTAyMTYzNTQ2WhcNMTcwNDAyMTYzNTQ2WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE +ODE3MTg1MDQ0WhcNMTcxMTE1MTg1MDQ0WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE CBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxHTAbBgNVBAoTFFRoZSBUb3IgUHJvamVj dCwgSW5jMRgwFgYDVQQDEw9LYXJzdGVuIExvZXNpbmcwggEiMA0GCSqGSIb3DQEB AQUAA4IBDwAwggEKAoIBAQChXn+IUp+o6G+k4ffxk3TkxZb3iXfiG7byNsG63olU @@ -11,11 +11,11 @@ Qw+VAhKTcEIv4yiR0BWapQyR07pgmKirYVjN6s6ef8NJzUptpxLlaYJ3ZfQfc4aE MXzScgaccwDFIWQ661lzLGCfeSxxa3Xy4wWsGwzNzLITYrrABcbg7yogLo2btNvD oEwGL3/baQdhl0dra6biVCZr9ydn3Hg57S55pUU0rBY25id78zUO8xrfNHw54wwX lOblGt75OOkahP/ZZSBxxoiknJ6y5VQV8y+noA4vigXFAgMBAAGjITAfMB0GA1Ud -DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAi2K5 -o3UnRBLdk9zkBEDbqPWxq8eiiIacbSoorspVlv4vqNO2UsTLc2sA4zZpY1L7Mkc6 -y7pcd1jFc6e9Wzt0agoIU9i3pyqkUx11ktE8a8CiVjDOns05GHe1DsLYcSLG8aJR -2pNEFjZEndHTXGRKwxtTmHimsIIWiy8tYFkC7SkBQj+BnXBqzCbJPhRDpMdhyJmQ -CLUhTTgU30G32x3vOhddWHAH8UDsXNlBASQOFs+A/1Mx5DI5AXLtwjp/rweTWoBk -11SfkVS/mfMZNRGOA9DEfoRYHPT0Rn4URAQrBr73CKQaVVYyeuDOoEs69KRH2JSO -IRcgAfpjtg8cry/eiw== +DgQWBBSeh60M+/wMYyYhlxtuff2Hk9n7bzANBgkqhkiG9w0BAQsFAAOCAQEAbsAc +gwl5KJH3pVKw4b+ACCOMgW+27MisCFbT1Izq2Wx+JcLMt3N//MoIpYOZWhsIeazW +/NE0fNbkLi0IYA0F1nUC9pHl44Hd8Gjfqa/YQUi9ALtgsY7l6W0sceW8WnZ8bu8J +DfrqnmB0bD2xc9ZjOn58al8dVjVWs95M87D9WCRU6LiaKFj5c45wciABQsTmC0qD +pyHYOaSGtXxXKDw5pAntdtHkCbowV5tDi/QQ8Tg7i5O7xwSh71Q7TZiNFMpLomBL +QllHfTZryFmoHyGn5MfngBUVCVHig5nXmk0dUMGuLiK4789dkgiPRz0vpB5Yf8Yy +CCE2jB6VBi2g5fMx0w== -----END CERTIFICATE----- diff --git a/CHANGELOG.md b/CHANGELOG.md index 59690ca..e6a8bf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.2.1 - 2017-08-?? +# Changes in version 1.2.1 - 2017-08-17 * Medium changes - Fix a bug while sanitizing bridge network statuses without entries. diff --git a/build.xml b/build.xml index ae25f41..ec0eb41 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.2.0-dev" /> + <property name="release.version" value="1.2.1" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.0.0" />

1 0

[sandboxed-tor-browser/master] Unset the addon autoupdater URL prefs.
by yawning＠torproject.org 17 Aug '17

17 Aug '17

commit 65ef5ea681e6836a95ec161d5de4b831fc8c21f9 Author: Yawning Angel <yawning(a)schwanenlied.me> Date: Thu Aug 17 19:02:45 2017 +0000 Unset the addon autoupdater URL prefs. Updating addons is disabled by default and will not work, even though it may appear to (#23265). Instead of checking and then failing to apply the update, by removing the URLs from the prefs, the check will fail immediately. This applies to everything but HTTPSE, because HTTPSE provides it's own updateURL in the install.rdf file. --- data/installer/mozilla.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/installer/mozilla.cfg b/data/installer/mozilla.cfg index 040d80e..ed832c7 100644 --- a/data/installer/mozilla.cfg +++ b/data/installer/mozilla.cfg @@ -16,6 +16,8 @@ lockPref("extensions.torbutton.versioncheck_enabled", false); // Disable extension auto update. lockPref("extensions.update.enabled", false); lockPref("extensions.update.autoUpdateDefault", false); +lockPref("extensions.update.url", ""); +lockPref("extensions.update.background.url", ""); // Disable the safe mode prompt, because we sometimes violently kill Tor // Browser and "safe mode" when we don't integrate addons and prefs properly

1 0

[onionoo/master] Reset IPv6 exit-policy summary in details status.
by karsten＠torproject.org 16 Aug '17

16 Aug '17

commit 6d23a7ce43e5d88cff132cacd38206ebb5d7e62c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon May 29 09:32:48 2017 +0200 Reset IPv6 exit-policy summary in details status. Fixes #22030. --- CHANGELOG.md | 7 +++++++ .../org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java | 2 ++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2922d9..8412f87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 4.0-1.?.? - 2017-0?-?? + + * Medium changes + - Reset IPv6 exit-policy summary in details status if a newer + server descriptor doesn't contain such a summary anymore. + + # Changes in version 4.0-1.3.0 - 2017-08-04 * Medium changes diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 8172cb7..cc50f4b 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -196,6 +196,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), portsOrPortRanges); detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); + } else { + detailsStatus.setExitPolicyV6Summary(null); } detailsStatus.setHibernating(descriptor.isHibernating() ? true : null);

1 0

[onionoo/master] Add test for exit policy reset.
by karsten＠torproject.org 16 Aug '17

16 Aug '17

commit 6473fdc8dcee245370cc5efa411c177bd1d8cfb2 Author: iwakeh <iwakeh(a)torproject.org> Date: Tue Aug 8 08:50:44 2017 +0000 Add test for exit policy reset. Test part of task-22030. --- .../onionoo/docs/DummyDocumentStore.java | 2 +- .../updater/NodeDetailsStatusUpdaterTest.java | 142 +++++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java b/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java index 4904044..2f8063e 100644 --- a/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java +++ b/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java @@ -12,7 +12,7 @@ import java.util.TreeSet; public class DummyDocumentStore extends DocumentStore { - private Map<Class<? extends Document>, SortedMap<String, Document>> + public Map<Class<? extends Document>, SortedMap<String, Document>> storedDocuments = new HashMap<>(); private static final String FINGERPRINT_NULL = ""; diff --git a/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java new file mode 100644 index 0000000..84c1fa6 --- /dev/null +++ b/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java @@ -0,0 +1,142 @@ +/* Copyright 2014--2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.updater; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorParser; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.ServerDescriptor; +import org.torproject.onionoo.docs.DetailsStatus; +import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.DummyDocumentStore; + +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.List; + +public class NodeDetailsStatusUpdaterTest { + + private DummyDocumentStore docStore; + + @Before + public void createDummyDocumentStore() { + this.docStore = new DummyDocumentStore(); + DocumentStoreFactory.setDocumentStore(this.docStore); + } + + @Test + public void testExitReset() throws Exception { + NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(null, null); + DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser(); + String descString = RELAY1 + PUB1 + RELAY2 + POLICY1 + RELAY3; + for (Descriptor desc : dp.parseDescriptors(descString.getBytes(), + new File("dummy"), "dummy")) { + assertTrue(desc.getClass().getName(), desc instanceof ServerDescriptor); + ndsu.processDescriptor(desc, true); + } + assertEquals(1, this.docStore.getPerformedStoreOperations()); + assertEquals(1, this.docStore.storedDocuments.size()); + DetailsStatus dd = this.docStore.getDocument(DetailsStatus.class, FP); + assertNotNull("docs: " + this.docStore.storedDocuments, dd); + assertEquals("Found: " + dd.getExitPolicy(), 1, dd.getExitPolicy().size()); + assertEquals("Found: " + dd.getExitPolicy().get(0), + "reject *:25", dd.getExitPolicy().get(0)); + List<String> acc = dd.getExitPolicyV6Summary().get("accept"); + List<String> rej = dd.getExitPolicyV6Summary().get("reject"); + assertNull(rej); + assertEquals("Found: " + dd.getExitPolicyV6Summary(), "23", acc.get(0)); + assertEquals("Found: " + dd.getExitPolicyV6Summary(), "42", acc.get(1)); + + descString = RELAY1 + PUB2 + RELAY2 + POLICY2 + RELAY3; + for (Descriptor desc : dp.parseDescriptors(descString.getBytes(), + new File("dummy"), "dummy")) { + assertTrue(desc.getClass().getName(), desc instanceof ServerDescriptor); + ndsu.processDescriptor(desc, true); + } + assertEquals(2, this.docStore.getPerformedStoreOperations()); + assertEquals(1, this.docStore.storedDocuments.size()); + dd = this.docStore.getDocument(DetailsStatus.class, FP); + assertNotNull("docs: " + this.docStore.storedDocuments, dd); + assertEquals("Found: " + dd.getExitPolicy(), 1, dd.getExitPolicy().size()); + assertEquals("Found: " + dd.getExitPolicy().get(0), + "reject *:*", dd.getExitPolicy().get(0)); + assertNull(dd.getExitPolicyV6Summary()); + } + + private static final String FP = "42CAF9C0588BBADDD338025E8F2D3CCF35CEEC25"; + + private static final String RELAY1 = "@type server-descriptor 1.0\n" + + "router impedance 84.201.150.89 443 0 0\n" + + "identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQQABkIAAS889JQwk/Acb0kWwkX1lolsL72YAGUvNgrLjz8fxczJAQAgBACw4otT\n" + + "Ajr+/0iRTRaCbdHs7BHSZ5qA3c4DG6a71zMzM0ismH/bVLbTuIsRrOuBfpOux+Cs\n" + + "ONO0fGuk5uazrY0iGcKJtZckyW4W7RYLkeGdNnYJ6BpshTEP0Rd61roeBQs=\n" + + "-----END ED25519 CERT-----\n" + + "master-key-ed25519 sOKLUwI6/v9IkU0Wgm3R7OwR0meagN3OAxumu9czMzM\n" + + "platform Tor 0.2.7.6 on Linux\n" + + "protocols Link 1 2 Circuit 1\n"; + + private static final String PUB1 = "published 2016-10-02 15:01:09\n"; + + private static final String PUB2 = "published 2016-10-03 15:01:09\n"; + + private static final String RELAY2 + = "fingerprint 42CA F9C0 588B BADD D338 025E 8F2D 3CCF 35CE EC25\n" + + "uptime 329888\n" + + "bandwidth 10240000 10240000 167976\n" + + "extra-info-digest 43A1C5D7EBA550CEC7465E20E3027772D5C552DD " + + "O3s96zE6cCXRrv3+KKpHm20GATS1s/9jyxbEv91v9mk\n" + + "onion-key\n" + + "-----BEGIN RSA PUBLIC KEY-----\n" + + "MIGJAoGBAMyCYbuR3p6mQDGSjkeO37/BUsGEPiFtKsd7e3m59cXB5oXmEqz0n2dp\n" + + "6qQAfyjPLP7Q1VZvvHxhD9n/qVXOmJRJGdlzhNBx1kIfvFhqKPop68cbwkcOlJ8r\n" + + "c3RjwDiv24q2kVAAq+NsYEhVECeshodf1A0off3J/cNsP6zXHn2HAgMBAAE=\n" + + "-----END RSA PUBLIC KEY-----\n" + + "signing-key\n" + + "-----BEGIN RSA PUBLIC KEY-----\n" + + "MIGJAoGBAMl++uLczhgkVvFLWuPON5Ynk820A/5W4Sm/tAg5klhRj1fvn5SxnSIZ\n" + + "l4VvAHSCZvojqEDCQvWwJUDEfEgT7tSlkq0Xjfsr7IhipnOkqhJFceLVCCdPHkkn\n" + + "L4maXPKcJ+JjhRQABEUM/+HOhBJHWyJ2V1Dc2C/8VISH+bG6BE/1AgMBAAE=\n" + + "-----END RSA PUBLIC KEY-----\n" + + "onion-key-crosscert\n" + + "-----BEGIN CROSSCERT-----\n" + + "AY2c8r2+VIHDo5ga+vSCfOt327496XcMSMJp/9DNQrWiXk+Xwr9z4cTTvcHun7Tc\n" + + "A17/Gjyo2yXDJABOQyZFbbKlCNUaoddOJc/s9hLCnkB//pxkO3sz7BwF+eOxqr9z\n" + + "u4TV8c2yOPqPybsm5ctmNmmP6eecUcNFmBA4uX7QSMs=\n" + + "-----END CROSSCERT-----\n" + + "ntor-onion-key-crosscert 0\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQoABkGYAbDii1MCOv7/SJFNFoJt0ezsEdJnmoDdzgMbprvXMzMzAAN9vMECBiJ2\n" + + "4Gbu9L3oZ983R5LwaE92TObml1anqe7W5uQDSskkIpEauL8tt029rS+xzQ7pCpdy\n" + + "EgyktjqXGQM=\n" + + "-----END ED25519 CERT-----\n" + + "hidden-service-dir\n" + + "contact dimanne dimanne(a)ya.ru\n" + + "ntor-onion-key DpZv/Br0X2JgxtohD5Kr7A7NgAK4HjBgTbTB+uVSolE=\n"; + + private static final String POLICY1 + = "reject *:25\nipv6-policy accept 23,42\n"; + + private static final String POLICY2 = "reject *:*\n"; + + private static final String RELAY3 + = "router-sig-ed25519 8pfPgYjlpwDoyESOZJHQwMwpmoyWCFg9dcswb8RTra4FT5jgol" + + "HTgkX51h/yUXBx7jUibs2EVaRTOPm9TuiVDA\n" + + "router-signature\n" + + "-----BEGIN SIGNATURE-----\n" + + "el2XtigqzbiSbUW11POx+l4kThP7c12JMyLlMPJXDncEYEa4F0+M0dQQF0BOeEIT\n" + + "D0lj1h4iWT4R+uO5/7umXyAhZidfbhLoQsWa/dGs5BfO4ROgvVLc8o4Za6PYcPE9\n" + + "DXd4yh+SZ86zaAWLUbr1VhRvSLWbFJwNn/aAQdAu70M=\n" + + "-----END SIGNATURE-----\n"; + +} +

1 0

[collector/master] Add test for sanitizing status without entries.
by karsten＠torproject.org 16 Aug '17

16 Aug '17

commit 44bcd9f57b33a5e76ba43bf242bd787fa8375a77 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 16 18:00:18 2017 +0200 Add test for sanitizing status without entries. Related to #23255. --- .../collector/bridgedescs/DescriptorBuilder.java | 15 +++++++++++++++ .../collector/bridgedescs/SanitizedBridgesWriterTest.java | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java index 08f9820..429a0ea 100644 --- a/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java +++ b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java @@ -60,6 +60,21 @@ abstract class DescriptorBuilder extends ArrayList<String> { fail("Line start not found: " + lineStart); } + /** Finds the first line that starts with the given line start and truncates + * that line and possibly subsequent lines, or fails if no line can be found + * with that line start. */ + void truncateAtLineStartingWith(String lineStart) { + for (int i = 0; i < this.size(); i++) { + if (this.get(i).startsWith(lineStart)) { + while (this.size() > i) { + this.remove(i); + } + return; + } + } + fail("Line start not found: " + lineStart); + } + /** Writes all descriptor lines with newlines to the given output stream. */ void build(OutputStream outputStream) throws IOException { for (String line : this) { diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java index 40e3a47..0e62573 100644 --- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -659,6 +659,19 @@ public class SanitizedBridgesWriterTest { } @Test + public void testNetworkStatusNoEntry() throws Exception { + this.defaultNetworkStatusBuilder.truncateAtLineStartingWith("r "); + this.runTest(); + assertFalse("Skipped network status without entries.", + this.parsedNetworkStatuses.isEmpty()); + for (String line : this.parsedNetworkStatuses.get(0)) { + if (line.startsWith("r ")) { + fail("Sanitized non-existent r line: " + line); + } + } + } + + @Test public void testNetworkStatusFromBifroest() throws Exception { this.defaultTarballBuilder.setTarballFileName( this.defaultTarballBuilder.getTarballFileName()

1 0

[collector/master] Handle bridge network statuses without entries.
by karsten＠torproject.org 16 Aug '17

16 Aug '17

commit f6e9d647a3af851b2bf624f25bfb410c07cba254 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 16 18:00:30 2017 +0200 Handle bridge network statuses without entries. Fixes #23255. --- CHANGELOG.md | 6 ++++++ .../torproject/collector/bridgedescs/SanitizedBridgesWriter.java | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d8f1e..59690ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 1.2.1 - 2017-08-?? + + * Medium changes + - Fix a bug while sanitizing bridge network statuses without entries. + + # Changes in version 1.2.0 - 2017-07-12 * Major changes diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index e3abf16..f53a3d1 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -600,7 +600,11 @@ public class SanitizedBridgesWriter extends CollecTorMain { SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - if (formatter.parse(publicationTime).getTime() + if (null == mostRecentDescPublished) { + logger.warn("The bridge network status published at " + publicationTime + + " does not contain a single entry. Please ask the bridge " + + "authority operator to check!"); + } else if (formatter.parse(publicationTime).getTime() - formatter.parse(mostRecentDescPublished).getTime() > 60L * 60L * 1000L) { logger.warn("The most recent descriptor in the bridge "

1 0