July 2020 - tor-commits - lists.torproject.org

[stem/master] Fix test.network tests
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit d799036fdf476a6a011b1abe16a70a9afa1ea630 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 15:36:17 2020 -0700 Fix test.network tests Turns out there was just one hanging test (test_attachstream). The problem was that the test socket's connect() method blocks until the connection is established, which in turn won't happen until we receive its STREAM event - producing deadlock. Solution is to simply connect from another thread so we don't disrupt our controller's event handling. --- test/integ/control/controller.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 0a6d908c..6ed89d2b 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -7,6 +7,7 @@ import os import shutil import socket import tempfile +import threading import time import unittest @@ -1127,8 +1128,6 @@ class TestController(unittest.TestCase): Tests Controller.get_streams(). """ - self.skipTest('asyncio unsupported by test.network') - host = socket.gethostbyname('www.torproject.org') port = 443 @@ -1157,8 +1156,6 @@ class TestController(unittest.TestCase): Tests Controller.close_stream with valid and invalid input. """ - self.skipTest('asyncio unsupported by test.network') - runner = test.runner.get_runner() async with await runner.get_tor_controller() as controller: @@ -1172,7 +1169,7 @@ class TestController(unittest.TestCase): # There's only one stream right now. Right? - built_stream = await controller.get_streams()[0] + built_stream = (await controller.get_streams())[0] # Make sure we have the stream for which we asked, otherwise # the next assertion would be a false positive. @@ -1196,7 +1193,6 @@ class TestController(unittest.TestCase): @test.require.online @async_test async def test_mapaddress(self): - self.skipTest('asyncio unsupported by test.network') self.skipTest('(https://trac.torproject.org/projects/tor/ticket/25611)') runner = test.runner.get_runner() @@ -1523,16 +1519,16 @@ class TestController(unittest.TestCase): @test.require.online @async_test async def test_attachstream(self): - self.skipTest('asyncio unsupported by test.network') - host = socket.gethostbyname('www.torproject.org') port = 80 circuit_id, streams = None, [] + stream_attached = asyncio.Event() async def handle_streamcreated(stream): if stream.status == 'NEW' and circuit_id: await controller.attach_stream(stream.id, circuit_id) + stream_attached.set() async with await test.runner.get_runner().get_tor_controller() as controller: # try 10 times to build a circuit we can connect through @@ -1546,9 +1542,16 @@ class TestController(unittest.TestCase): socks_listener = (await controller.get_listeners(Listener.SOCKS))[0] with test.network.Socks(socks_listener) as s: - s.settimeout(30) - s.connect((host, port)) + s.settimeout(5) + + t = threading.Thread(target = s.connect, args = ((host, port),)) + t.start() + + await asyncio.wait_for(stream_attached.wait(), timeout = 6) streams = await controller.get_streams() + + t.join() + break except (stem.CircuitExtensionFailed, socket.timeout): continue

1 0

[stem/master] Cache AddressDisableIPv6 option
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit 452c96a1b1be09e38a50c25be29ca25f69abdf3d Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 15:53:41 2020 -0700 Cache AddressDisableIPv6 option We just re-cached manual data a couple days ago but tor added a new option... https://gitweb.torproject.org/tor.git/commit/?id=c3d113a Since I'm in the middle of fixing our ONLINE test target might as well include this addition. --- stem/cached_manual.sqlite | Bin 253952 -> 253952 bytes stem/settings.cfg | 1 + 2 files changed, 1 insertion(+) diff --git a/stem/cached_manual.sqlite b/stem/cached_manual.sqlite index 809512fe..944c6722 100644 Binary files a/stem/cached_manual.sqlite and b/stem/cached_manual.sqlite differ diff --git a/stem/settings.cfg b/stem/settings.cfg index b82abed8..3c739394 100644 --- a/stem/settings.cfg +++ b/stem/settings.cfg @@ -224,6 +224,7 @@ manual.summary.DormantCanceledByStartup Disable dormant mode when tor's restarte # Server Config Options manual.summary.Address Overwrites address others will use to reach this relay +manual.summary.AddressDisableIPv6 Skip attempt to discover an IPv6 ORPort manual.summary.AssumeReachable Skips reachability test at startup manual.summary.AssumeReachableIPv6 Skip IPv6 reachability test at startup manual.summary.BridgeRelay Act as a bridge

1 0

[stem/master] Fix stem.descriptor.remote deadlock
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit 02f74f44f09efa9f854f830ae2a2f28514040e1d Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 17:52:44 2020 -0700 Fix stem.descriptor.remote deadlock Oops! While writing our Synchronous class I made our Query directly stop its loop, ending its loop thread. Our class internally expects our thread attribute to be None when its terminated, and as such wouldn't resume. To reproduce the deadlock I used the following script... import stem.descriptor.remote downloader = stem.descriptor.remote.DescriptorDownloader(validate = True) consensus_query = downloader.get_consensus() consensus_query.run() consensus = list(consensus_query) print('count: %s' % len(consensus)) --- stem/descriptor/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 2c97c361..96ffceed 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -475,7 +475,7 @@ class Query(Synchronous): return [desc async for desc in self._run(suppress)] finally: if stop: - self._loop.call_soon_threadsafe(self._loop.stop) + self.stop() async def _run(self, suppress: bool) -> AsyncIterator[stem.descriptor.Descriptor]: with self._downloader_lock:

1 0

[stem/master] Drop query 'stop' argument
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit 278e7ee8abf83a79f107bff20cd0e654c0e4977e Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 18:06:59 2020 -0700 Drop query 'stop' argument I added a 'stop' argument to our Query's run method because our Synchronous class couldn't be used after it was discontinued. However, it now resumes itself upon further async method calls so we do not need to avoid stoppages. --- stem/descriptor/remote.py | 11 ++--------- test/unit/descriptor/remote.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 96ffceed..2912134c 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -447,14 +447,12 @@ class Query(Synchronous): loop = asyncio.get_event_loop() self._downloader_task = loop.create_task(self._download_descriptors(self.retries, self.timeout)) - async def run(self, suppress: bool = False, stop: bool = True) -> List['stem.descriptor.Descriptor']: + async def run(self, suppress: bool = False) -> List['stem.descriptor.Descriptor']: """ Blocks until our request is complete then provides the descriptors. If we haven't yet started our request then this does so. :param suppress: avoids raising exceptions if **True** - :param stop: terminates the resources backing this query if **True**, - further method calls will raise a RuntimeError :returns: list for the requested :class:`~stem.descriptor.__init__.Descriptor` instances @@ -467,15 +465,10 @@ class Query(Synchronous): * :class:`~stem.DownloadFailed` if our request fails """ - # TODO: We should replace our 'stop' argument with a new API design prior - # to release. Self-destructing this object by default for synchronous users - # is quite a step backward, but is acceptable as we iterate on this. - try: return [desc async for desc in self._run(suppress)] finally: - if stop: - self.stop() + self.stop() async def _run(self, suppress: bool) -> AsyncIterator[stem.descriptor.Descriptor]: with self._downloader_lock: diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 3facd6a5..6b9ca3ad 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -135,7 +135,7 @@ class TestDescriptorDownloader(unittest.TestCase): def test_reply_header_data(self): query = stem.descriptor.remote.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31', start = False) self.assertEqual(None, query.reply_headers) # initially we don't have a reply - query.run(stop = False) + query.run() self.assertEqual('Fri, 13 Apr 2018 16:35:50 GMT', query.reply_headers.get('Date')) self.assertEqual('application/octet-stream', query.reply_headers.get('Content-Type'))

1 0

[stem/master] Bad endpoint type within test
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit a65a4777727c9941ab533c219d854d80b5149d1c Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 18:12:33 2020 -0700 Bad endpoint type within test Stem 2.x dropped support for tuple endpoints. Guess I haven't run our ONLINE test target since then... ====================================================================== ERROR: test_using_authorities ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/require.py", line 60, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/require.py", line 75, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/integ/descriptor/remote.py", line 113, in test_using_authorities validate = True, File "/home/atagar/Desktop/stem/stem/descriptor/remote.py", line 408, in __init__ raise ValueError("Endpoints must be an stem.ORPort or stem.DirPort. '%s' is a %s." % (endpoint, type(endpoint).__name__)) ValueError: Endpoints must be an stem.ORPort or stem.DirPort. '('128.31.0.39', 9131)' is a tuple. --- test/integ/descriptor/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 5788f322..319800f5 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -108,7 +108,7 @@ class TestDescriptorDownloader(unittest.TestCase): queries.append((stem.descriptor.remote.Query( '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', - endpoints = [(authority.address, authority.dir_port)], + endpoints = [stem.DirPort(authority.address, authority.dir_port)], timeout = 30, validate = True, ), authority))

1 0

[stem/master] Trailing newline broke descriptor downloads
by atagar＠torproject.org 21 Jul '20

21 Jul '20

commit 1e1f3665aebb521acbcf332dbf752f1a815c62cf Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 21 18:38:51 2020 -0700 Trailing newline broke descriptor downloads I rewrote stem.descriptor.remote's download code for our asyncio migration. When retrieving server descriptors we ended with a double newline ('\n\n') which caused our parser to think that there are two descriptors, the second of which is '\n'. This in turned broke validation with... ====================================================================== FAIL: test_using_authorities ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/integ/descriptor/remote.py", line 118, in test_using_authorities descriptors = list(query.run()) File "/home/atagar/Desktop/stem/stem/util/__init__.py", line 363, in _run_async_method return future.result() File "/home/atagar/Python-3.7.0/Lib/concurrent/futures/_base.py", line 432, in result return self.__get_result() ValueError: Descriptor must have a 'router' entry During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/require.py", line 60, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/require.py", line 75, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/integ/descriptor/remote.py", line 120, in test_using_authorities self.fail('Unable to use %s (%s:%s, %s): %s' % (authority.nickname, authority.address, authority.dir_port, type(exc), exc)) AssertionError: Unable to use moria1 (128.31.0.39:9131, <class 'ValueError'>): Descriptor must have a 'router' entry --- stem/descriptor/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 2912134c..ff4b1403 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -1003,11 +1003,11 @@ def _http_body_and_headers(data: bytes) -> Tuple[bytes, Dict[str, str]]: encoding = headers.get('Content-Encoding') if encoding == 'deflate': - return stem.descriptor.Compression.GZIP.decompress(body_data), headers + return stem.descriptor.Compression.GZIP.decompress(body_data).rstrip(), headers for compression in stem.descriptor.Compression: if encoding == compression.encoding: - return compression.decompress(body_data), headers + return compression.decompress(body_data).rstrip(), headers raise ValueError("'%s' is an unrecognized encoding" % encoding)

1 0

[tor-browser-build/maint-9.5] Release preparations for 9.5.3
by sysrqb＠torproject.org 21 Jul '20

21 Jul '20

commit 8aa469dc35d9003ca38006f15c809581f698abfa Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Tue Jul 21 16:39:51 2020 +0000 Release preparations for 9.5.3 Changelog update and versions bump --- projects/firefox/config | 2 +- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 6 ++++++ projects/tor-browser/config | 4 ++-- projects/tor/build | 3 --- projects/tor/config | 3 +-- rbm.conf | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/projects/firefox/config b/projects/firefox/config index 414097f..022eea9 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -8,7 +8,7 @@ git_submodule: 1 gpg_keyring: torbutton.gpg var: - firefox_platform_version: 68.10.1 + firefox_platform_version: 68.11.0 firefox_version: '[% c("var/firefox_platform_version") %]esr' torbrowser_branch: 9.5 branding_directory: '[% IF c("var/android") %]mobile/android[% ELSE %]browser[% END %]/branding/alpha' diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index bd49a23..0e14dae 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,9 @@ +Tor Browser 9.5.3 -- July 28 2020 + * All Platforms + * Update Firefox to 68.11.0esr + * Update NoScript to 11.0.34 + * Update Tor to 0.4.3.6 + Tor Browser 9.5.2 -- July 7 2020 * Android * Update Firefox to 68.10.1esr diff --git a/projects/tor-browser/config b/projects/tor-browser/config index 6bd09de..9254cf6 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -73,9 +73,9 @@ input_files: enable: '[% c("var/snowflake") %]' - filename: Bundle-Data enable: '[% ! c("var/android") %]' - - URL: https://addons.cdn.mozilla.net/user-media/addons/722/noscript_security_suit… + - URL: https://addons.cdn.mozilla.net/user-media/addons/722/noscript_security_suit… name: noscript - sha256sum: 088c0b0f08529d88aacee3b1c09cfafe304b98cb2847c3fc255e05d1bf10bd6d + sha256sum: 759a38df3ef121804bf8933ab5289ec8452d5663fcad2f484b73095d8b488578 - filename: 'RelativeLink/start-tor-browser.desktop' enable: '[% c("var/linux") %]' - filename: 'RelativeLink/execdesktop' diff --git a/projects/tor/build b/projects/tor/build index 012ec98..6dd22f7 100644 --- a/projects/tor/build +++ b/projects/tor/build @@ -92,9 +92,6 @@ openssldir=/var/tmp/dist/openssl/openssl cd /var/tmp/build/[% project %]-[% c('version') %] -# Cherry-pick patch until 0.4.3.6 is released. -patch -p1 < $rootdir/0001-hs-v3-Improve-accessor-semantic-of-client-cached-obj.patch - # add git hash to micro-revision.i for #24995 echo '"[% c("abbrev", { abbrev_length => 16 }) %]"' > micro-revision.i ./autogen.sh diff --git a/projects/tor/config b/projects/tor/config index 1f57465..999f387 100644 --- a/projects/tor/config +++ b/projects/tor/config @@ -1,6 +1,6 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %]' -version: 0.4.3.5 +version: 0.4.3.6 git_hash: 'tor-[% c("version") %]' git_url: https://git.torproject.org/tor.git git_submodule: 1 @@ -68,4 +68,3 @@ input_files: - name: zstd project: zstd enable: '[% c("var/android") %]' - - filename: 0001-hs-v3-Improve-accessor-semantic-of-client-cached-obj.patch diff --git a/rbm.conf b/rbm.conf index 05ffcf2..111ea70 100644 --- a/rbm.conf +++ b/rbm.conf @@ -24,10 +24,10 @@ buildconf: git_signtag_opt: '-s' var: - torbrowser_version: '9.5.2' + torbrowser_version: '9.5.3' torbrowser_build: 'build1' torbrowser_incremental_from: - - 9.5 + - 9.5.1 project_name: tor-browser multi_lingual: 0 build_mar: 1

1 0

[chutney/master] Make sure IPv6 ORPorts are configured
by nickm＠torproject.org 21 Jul '20

21 Jul '20

commit 6ea3e64b9944a65956f1730c1117459af8969649 Author: David Goulet <dgoulet(a)torproject.org> Date: Tue Jul 21 14:49:58 2020 -0400 Make sure IPv6 ORPorts are configured Our exit-v6.tmpl should always have a v6 ORPort. Furthermore, bridges+ipv6-min network configuration now uses v6 authorities and a v6 Exit. Without this, the IPv6 bridge self rechability test would never work. Closes #40007 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- networks/bridges+ipv6-min | 6 ++++-- torrc_templates/exit-v6.i | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/networks/bridges+ipv6-min b/networks/bridges+ipv6-min index a305b89..060a847 100644 --- a/networks/bridges+ipv6-min +++ b/networks/bridges+ipv6-min @@ -1,7 +1,9 @@ Require("IPV6") # By default, Authorities are not configured as exits -Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl") -ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl") +Authority = Node(tag="a", authority=1, relay=1, ipv6_addr="[::1]", + torrc="authority-orport-v6.tmpl") +ExitRelay = Node(tag="r", relay=1, exit=1, ipv6_addr="[::1]", + torrc="relay.tmpl") BridgeAuthority = Node(tag="ba", authority=1, bridgeauthority=1, relay=1, torrc="bridgeauthority.tmpl") diff --git a/torrc_templates/exit-v6.i b/torrc_templates/exit-v6.i index bac1ba6..4bf5be8 100644 --- a/torrc_templates/exit-v6.i +++ b/torrc_templates/exit-v6.i @@ -1,3 +1,5 @@ +${include:orport-v6.i} + # Must be included after relay-non-exit.tmpl ExitRelay 1

1 0

[torspec/master] Fix wording in a clarification.
by nickm＠torproject.org 21 Jul '20

21 Jul '20

commit fd47b40097d3a1482bafec813078660307d60baa Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Tue Jul 21 13:06:05 2020 -0500 Fix wording in a clarification. --- proposals/324-rtt-congestion-control.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proposals/324-rtt-congestion-control.txt b/proposals/324-rtt-congestion-control.txt index be81d0e..e288563 100644 --- a/proposals/324-rtt-congestion-control.txt +++ b/proposals/324-rtt-congestion-control.txt @@ -485,11 +485,11 @@ double or triple this), then the congestion window is decreased. elif queue_use > vegas_beta: cwnd = cwnd - circwindow_inc/cwnd # linear backoff -Notice that we only decrease the window size by a single packet per +Notice that we only change the window size by a single packet per congestion window, rather than by the full delta between current queue_use and the target value. This is done because if more than one connection jumps to use the available bandwidth at once, excess -congestion will result. +congestion will result (or underutilization). 3.2.3. Tor Vegas: Complete SENDME Update Algorithm

1 0

[torspec/master] Resolve a question in a TODO about Vegas, and add refs to other TODO.
by nickm＠torproject.org 21 Jul '20

21 Jul '20

commit a24d66be15b61e61b01fda535eac6c072bf29977 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Tue Jul 21 12:59:52 2020 -0500 Resolve a question in a TODO about Vegas, and add refs to other TODO. Thanks goes to Toke Høiland-Jørgensen. --- proposals/324-rtt-congestion-control.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/proposals/324-rtt-congestion-control.txt b/proposals/324-rtt-congestion-control.txt index eede20b..be81d0e 100644 --- a/proposals/324-rtt-congestion-control.txt +++ b/proposals/324-rtt-congestion-control.txt @@ -409,6 +409,7 @@ This will run each time we get a SENDME (aka sendme_process_circuit_level()): 3.2. Tor Vegas: TCP Vegas with Aggressive Slow Start [TOR_VEGAS] http://intronetworks.cs.luc.edu/1/html/newtcps.html#tcp-vegas http://pages.cs.wisc.edu/~akella/CS740/F08/740-Papers/BOP94.pdf + http://www.mathcs.richmond.edu/~lbarnett/cs332/assignments/brakmo_peterson_… ftp://ftp.cs.princeton.edu/techreports/2000/628.pdf TCP Vegas control algorithm makes use of two RTT measurements: @@ -437,7 +438,11 @@ queue_use estimate. TODO: This simplification might not hold for some versions of BWE and BDP estimation. See also the [TOR_WESTWOOD] section's TODO - and paper citations for both TCP Westwood and TCP Vegas. + and paper citations for both TCP Westwood and TCP Vegas. In + particular, see Section 3.5.2 of: + http://pages.cs.wisc.edu/~akella/CS740/F08/740-Papers/BOP94.pdf + or Section 3.2 of: + http://www.mathcs.richmond.edu/~lbarnett/cs332/assignments/brakmo_peterson_… 3.2.1. Tor Vegas Slow Start @@ -480,9 +485,11 @@ double or triple this), then the congestion window is decreased. elif queue_use > vegas_beta: cwnd = cwnd - circwindow_inc/cwnd # linear backoff - TODO: Why not reduce the window by the number of packets that - queue_use is over or under the target value by, rather than - just 1 cell per cwnd? Need to ask some TCP folks, or experiment. +Notice that we only decrease the window size by a single packet per +congestion window, rather than by the full delta between current +queue_use and the target value. This is done because if more than one +connection jumps to use the available bandwidth at once, excess +congestion will result. 3.2.3. Tor Vegas: Complete SENDME Update Algorithm

1 0