February 2021 - tor-commits - lists.torproject.org

[sbws/master] fix: scanner: remove relay to measure as helper
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 50377680448d66bc95a09fc5333da9465bd1b791 Author: juga0 <juga(a)riseup.net> Date: Mon Feb 8 16:24:11 2021 +0000 fix: scanner: remove relay to measure as helper --- sbws/core/scanner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index c7ee1ee..40e3093 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -219,6 +219,12 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): if is_exit else rl.non_exits if not len(candidates): return None + # In the case the helper is an exit, the entry could be an exit too + # (#40041), so ensure the helper is not the same as the entry, likely to + # happen in a test network. + if is_exit: + candidates = [c for c in candidates + if c.fingerprint != relay.fingerprint] min_relay_bw = rl.exit_min_bw() if is_exit else rl.non_exit_min_bw() log.debug('Picking a 2nd hop to measure %s from %d choices. is_exit=%s', relay.nickname, len(candidates), is_exit)

1 0

[sbws/master] fix: scanner: log exit policy when stream fails
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit a570a707ec898cf59bdd12d3a8b20d25ce9459fd Author: juga0 <juga(a)riseup.net> Date: Mon Feb 8 16:21:53 2021 +0000 fix: scanner: log exit policy when stream fails --- sbws/core/scanner.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 903f09f..c7ee1ee 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -267,7 +267,7 @@ def create_path_relay_as_entry(relay, dest, rl, cb): return error_no_helper(relay, dest) circ_fps = [relay.fingerprint, helper.fingerprint] nicknames = [relay.nickname, helper.nickname] - return helper, circ_fps, nicknames + return circ_fps, nicknames, helper.exit_policy def create_path_relay_as_exit(relay, dest, rl, cb): @@ -278,7 +278,7 @@ def create_path_relay_as_exit(relay, dest, rl, cb): circ_fps = [helper.fingerprint, relay.fingerprint] # stored for debugging nicknames = [helper.nickname, relay.nickname] - return circ_fps, nicknames + return circ_fps, nicknames, relay.exit_policy def error_no_circuit(circ_fps, nicknames, reason, relay, dest, our_nick): @@ -341,9 +341,11 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # Instead of ensuring that the relay can exit to all IPs, try first with # the relay as an exit, if it can exit to some IPs. if relay.is_exit_not_bad_allowing_port(dest.port): - circ_fps, nicknames = create_path_relay_as_exit(relay, dest, rl, cb) + circ_fps, nicknames, exit_policy = \ + create_path_relay_as_exit(relay, dest, rl, cb) else: - circ_fps, nicknames = create_path_relay_as_entry(relay, dest, rl, cb) + circ_fps, nicknames, exit_policy = \ + create_path_relay_as_entry(relay, dest, rl, cb) # Build the circuit circ_id, reason = cb.build_circuit(circ_fps) @@ -367,10 +369,12 @@ def measure_relay(args, conf, destinations, cb, rl, relay): if not is_usable and \ relay.is_exit_not_bad_allowing_port(dest.port): log.info( - "Exit %s (%s) that can't exit all ips failed to connect to " - " %s via circuit %s (%s). Trying again with it as entry.", - relay.fingerprint, relay.nickname, dest, circ_fps, nicknames) - circ_fps, nicknames = create_path_relay_as_entry(relay, dest, rl, cb) + "Exit %s (%s) that can't exit all ips, with exit policy %s, failed" + " to connect to %s via circuit %s (%s). Reason: %s. Trying again " + "with it as entry.", relay.fingerprint, relay.nickname, + exit_policy, dest.url, circ_fps, nicknames, usable_data) + circ_fps, nicknames, exit_policy = \ + create_path_relay_as_entry(relay, dest, rl, cb) circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: log.warning( @@ -385,8 +389,10 @@ def measure_relay(args, conf, destinations, cb, rl, relay): is_usable, usable_data = connect_to_destination_over_circuit( dest, circ_id, s, cb.controller, dest._max_dl) if not is_usable: - log.debug('Destination %s unusable via circuit %s (%s), %s', - dest.url, circ_fps, nicknames, usable_data) + log.debug('Failed to connect to %s to measure %s (%s) via circuit ' + '%s (%s). Exit policy: %s. Reason: %s.', dest.url, + relay.fingerprint, relay.nickname, circ_fps, nicknames, + exit_policy, usable_data) cb.close_circuit(circ_id) return [ ResultErrorStream(relay, circ_fps, dest.url, our_nick, @@ -410,9 +416,10 @@ def measure_relay(args, conf, destinations, cb, rl, relay): bw_results, reason = measure_bandwidth_to_server( s, conf, dest, usable_data['content_length']) if bw_results is None: - log.debug('Unable to measure bandwidth for %s (%s) to %s via circuit ' - '%s (%s): %s', relay.fingerprint, relay.nickname, - dest.url, circ_fps, nicknames, reason) + log.debug('Failed to measure %s (%s) via circuit %s (%s) to %s. Exit' + ' policy: %s. Reason: %s.', relay.fingerprint, + relay.nickname, circ_fps, nicknames, dest.url, exit_policy, + reason) cb.close_circuit(circ_id) return [ ResultErrorStream(relay, circ_fps, dest.url, our_nick,

1 0

[sbws/master] fix:scanner: Move as_entry/as_exit into one function
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 8846c32a500515fc3d5d094a6ff5b0777f2c30fb Author: juga0 <juga(a)riseup.net> Date: Tue Feb 9 12:57:36 2021 +0000 fix:scanner: Move as_entry/as_exit into one function since they're similar code --- sbws/core/scanner.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 40e3093..97829f4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -266,25 +266,24 @@ def error_no_helper(relay, dest, our_nick=""): ] -def create_path_relay_as_entry(relay, dest, rl, cb): - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=True) - if not helper: - return error_no_helper(relay, dest) - circ_fps = [relay.fingerprint, helper.fingerprint] - nicknames = [relay.nickname, helper.nickname] - return circ_fps, nicknames, helper.exit_policy - - -def create_path_relay_as_exit(relay, dest, rl, cb): - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=False) +def create_path_relay(relay, dest, rl, cb, relay_as_entry=True): + if relay_as_entry: + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=True) + else: + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=False) if not helper: return error_no_helper(relay, dest) - circ_fps = [helper.fingerprint, relay.fingerprint] - # stored for debugging - nicknames = [helper.nickname, relay.nickname] - return circ_fps, nicknames, relay.exit_policy + if relay_as_entry: + circ_fps = [relay.fingerprint, helper.fingerprint] + nicknames = [relay.nickname, helper.nickname] + exit_policy = helper.exit_policy + else: + circ_fps = [helper.fingerprint, relay.fingerprint] + nicknames = [helper.nickname, relay.nickname] + exit_policy = relay.exit_policy + return circ_fps, nicknames, exit_policy def error_no_circuit(circ_fps, nicknames, reason, relay, dest, our_nick): @@ -348,10 +347,10 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # the relay as an exit, if it can exit to some IPs. if relay.is_exit_not_bad_allowing_port(dest.port): circ_fps, nicknames, exit_policy = \ - create_path_relay_as_exit(relay, dest, rl, cb) + create_path_relay(relay, dest, rl, cb, relay_as_entry=False) else: circ_fps, nicknames, exit_policy = \ - create_path_relay_as_entry(relay, dest, rl, cb) + create_path_relay(relay, dest, rl, cb) # Build the circuit circ_id, reason = cb.build_circuit(circ_fps) @@ -380,7 +379,7 @@ def measure_relay(args, conf, destinations, cb, rl, relay): "with it as entry.", relay.fingerprint, relay.nickname, exit_policy, dest.url, circ_fps, nicknames, usable_data) circ_fps, nicknames, exit_policy = \ - create_path_relay_as_entry(relay, dest, rl, cb) + create_path_relay(relay, dest, rl, cb) circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: log.warning(

1 0

[sbws/master] fix:scanner: Rm condition assigning helper
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 72b43570589d10a668039341895948ded0f1bfd7 Author: juga0 <juga(a)riseup.net> Date: Tue Feb 9 13:08:44 2021 +0000 fix:scanner: Rm condition assigning helper --- sbws/core/scanner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 97829f4..5fd917a 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -267,12 +267,12 @@ def error_no_helper(relay, dest, our_nick=""): def create_path_relay(relay, dest, rl, cb, relay_as_entry=True): - if relay_as_entry: - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=True) - else: - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=False) + # the helper `is_exit` arg (should be better called `helper_as_exit`), + # is True when the relay is the entry (helper has to be exit) + # and False when the relay is not the entry, ie. is the exit (helper does + # not have to be an exit) + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=relay_as_entry) if not helper: return error_no_helper(relay, dest) if relay_as_entry:

1 0

[sbws/master] Merge remote-tracking branch 'gitlab/merge-requests/61' into maint-1.1
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 3e4ccb90ea64b391d0003bb72f17d513845680ae Merge: e375cd2 d0a0202 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Feb 22 17:12:18 2021 +0000 Merge remote-tracking branch 'gitlab/merge-requests/61' into maint-1.1 docs/source/activity_measure.puml | 30 +++++++ docs/source/how_works.rst | 12 ++- docs/source/images/activity_measure.svg | 42 +++++++++ sbws/core/scanner.py | 145 ++++++++++++++++++++++---------- sbws/lib/relaylist.py | 35 +++++--- 5 files changed, 204 insertions(+), 60 deletions(-)

1 0

[sbws/master] Merge branch 'maint-1.1'
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 0e37b0e98ba19d243985ff48dc670faee21fe94b Merge: 7495401 3e4ccb9 Author: juga0 <juga(a)riseup.net> Date: Tue Feb 23 07:23:55 2021 +0000 Merge branch 'maint-1.1' docs/source/activity_measure.puml | 30 +++++++ docs/source/how_works.rst | 12 ++- docs/source/images/activity_measure.svg | 42 +++++++++ sbws/core/scanner.py | 145 ++++++++++++++++++++++---------- sbws/lib/relaylist.py | 35 +++++--- 5 files changed, 204 insertions(+), 60 deletions(-)

1 0

[sbws/master] fix: CI: Make wget quiet
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit d0a0202f47b5e019f08c0b4b285b4f2cb6a87352 Author: juga0 <juga(a)riseup.net> Date: Wed Feb 10 09:30:39 2021 +0000 fix: CI: Make wget quiet to avoid many lines of non useful text the CI. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f0aadaf..51daeba 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ commands = bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 &>/dev/null &" sleep 1 - wget -O/dev/null http://127.0.0.1:28888/sbws.bin + wget -qO/dev/null http://127.0.0.1:28888/sbws.bin ; Run actually the scanner mkdir -p /tmp/.sbws ; This add around 3min more to the tests

1 0

[sbws/master] fix: relaylist: rename can_exit_to_port
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 1014bac294d03790c15bec1bff21e9ca9802acdb Author: juga0 <juga(a)riseup.net> Date: Tue Feb 2 11:15:02 2021 +0000 fix: relaylist: rename can_exit_to_port to can_exit_to_port_all_ips, because it's using `strict`, which means that it allows to exit to all IPs. It seems more convenient to try first with exits that allow to exit to some IPs and only try a second time if that fails, because there are more. --- sbws/lib/relaylist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index ba0b176..5eef4aa 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -178,7 +178,7 @@ class Relay: """Number of times the relay was in a conensus.""" return len(self.relay_in_recent_consensus) - def can_exit_to_port(self, port): + def can_exit_to_port_all_ips(self, port): """ Returns True if the relay has an exit policy and the policy accepts exiting to the given portself or False otherwise. @@ -192,7 +192,7 @@ class Relay: # When the attribute is gotten in getattr(self._desc, "exit_policy"), # is possible that stem's _input_rules is None and raises an exception # (#29899): - # File "/usr/lib/python3/dist-packages/sbws/lib/relaylist.py", line 117, in can_exit_to_port # noqa + # File "/usr/lib/python3/dist-packages/sbws/lib/relaylist.py", line 117, in can_exit_to_port_all_ips # noqa # if not self.exit_policy: # File "/usr/lib/python3/dist-packages/stem/exit_policy.py", line 512, in __len__ # noqa # return len(self._get_rules()) @@ -222,7 +222,7 @@ class Relay: def is_exit_not_bad_allowing_port(self, port): return (Flag.BADEXIT not in self.flags and Flag.EXIT in self.flags and - self.can_exit_to_port(port)) + self.can_exit_to_port_all_ips(port)) def increment_relay_recent_measurement_attempt(self): """

1 0

[sbws/master] fix: scanner: extract method to create paths
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit fa9314123b702feee230f7974bfbad2d783d8365 Author: juga0 <juga(a)riseup.net> Date: Tue Feb 2 11:56:05 2021 +0000 fix: scanner: extract method to create paths because `measure_relay` method is too long, confusing and we have had several bugs in this part of the code. --- sbws/core/scanner.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 2736059..228ad74 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -244,6 +244,27 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): return chosen +def create_path_relay_as_entry(relay, dest, rl, cb): + circ_fps = nicknames = [] + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=True) + if helper: + circ_fps = [relay.fingerprint, helper.fingerprint] + nicknames = [relay.nickname, helper.nickname] + return helper, circ_fps, nicknames + + +def create_path_relay_as_exit(relay, dest, rl, cb): + circ_fps = nicknames = [] + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=False) + if helper: + circ_fps = [helper.fingerprint, relay.fingerprint] + # stored for debugging + nicknames = [helper.nickname, relay.nickname] + return helper, circ_fps, nicknames + + def measure_relay(args, conf, destinations, cb, rl, relay): """ Select a Web server, a relay to build the circuit, @@ -295,18 +316,11 @@ def measure_relay(args, conf, destinations, cb, rl, relay): helper = None circ_fps = None if relay.is_exit_not_bad_allowing_port_all_ips(dest.port): - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=False) - if helper: - circ_fps = [helper.fingerprint, relay.fingerprint] - # stored for debugging - nicknames = [helper.nickname, relay.nickname] + helper, circ_fps, nicknames = create_path_relay_as_exit( + relay, dest, rl, cb) else: - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=True) - if helper: - circ_fps = [relay.fingerprint, helper.fingerprint] - nicknames = [relay.nickname, helper.nickname] + helper, circ_fps, nicknames = create_path_relay_as_entry( + relay, dest, rl, cb) if not helper: reason = 'Unable to select a second relay' log.debug(reason + ' to help measure %s (%s)', @@ -323,11 +337,7 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # If that's the case, try again using them as 2nd hop. # We could reuse the helper, but it does not need to be an exit now, # so choose other again. - helper = _pick_ideal_second_hop( - relay, dest, rl, cb.controller, is_exit=False) - if helper: - circ_fps = [helper.fingerprint, relay.fingerprint] - nicknames = [helper.nickname, relay.nickname] + create_path_relay_as_exit(relay, dest, rl, cb) circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: log.debug('Could not build circuit with path %s (%s): %s ',

1 0

[sbws/master] fix: relaylist: Add methods to obtain exits that
by juga＠torproject.org 23 Feb '21

23 Feb '21

commit 55b76a84b9da3d13f2d2616cb83a29e4726b8a67 Author: juga0 <juga(a)riseup.net> Date: Tue Feb 2 11:46:21 2021 +0000 fix: relaylist: Add methods to obtain exits that can exit to some IPs. To use them in the cases it will be more convenient. --- sbws/lib/relaylist.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 863689e..3ff1f73 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -219,11 +219,34 @@ class Relay: return False return False + def can_exit_to_port_some_ips(self, port): + """ + Returns True if the relay has an exit policy and the policy accepts + exiting to the given port and some public IPs or False otherwise. + """ + assert isinstance(port, int) + try: + if self.exit_policy: + # Not using argument `strict`, to know whether it can exit + # some public IPs, though not all. + return ( + self.exit_policy.strip_private() + .can_exit_to(port=port) + ) + except TypeError: + return False + return False + def is_exit_not_bad_allowing_port_all_ips(self, port): return (Flag.BADEXIT not in self.flags and Flag.EXIT in self.flags and self.can_exit_to_port_all_ips(port)) + def is_exit_not_bad_allowing_port_some_ips(self, port): + return (Flag.BADEXIT not in self.flags and + Flag.EXIT in self.flags and + self.can_exit_to_port_some_ips(port)) + def increment_relay_recent_measurement_attempt(self): """ Increment The number of times that a relay has been queued @@ -457,6 +480,10 @@ class RelayList: return [r for r in self.exits if r.is_exit_not_bad_allowing_port_all_ips(port)] + def exits_not_bad_allowing_port_some_ips(self, port): + return [r for r in self.exits + if r.is_exit_not_bad_allowing_port_some_ips(port)] + def increment_recent_measurement_attempt(self): """ Increment the number of times that any relay has been queued to be

1 0