December 2020 - tor-commits - lists.torproject.org

[sbws/maint-1.1] chg: scanner: Stop storing recent_measurement_attempt
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit f014084108eafd6478fbe5e2e0afc6855435c87f Author: juga0 <juga(a)riseup.net> Date: Mon Nov 23 20:02:26 2020 +0000 chg: scanner: Stop storing recent_measurement_attempt because it stores a timestamp for each attempt, which makes state.dat grow thousand of lines (json). Closes #40023, #40020 --- sbws/core/scanner.py | 3 ++- tests/integration/test_files.py | 7 +++++-- tests/unit/lib/test_heartbeat.py | 2 ++ tests/unit/lib/test_v3bwfile.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index e5d5e90..8b6a6a4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -523,7 +523,8 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump, # Don't start measuring a relay if sbws is stopping. if settings.end_event.is_set(): break - relay_list.increment_recent_measurement_attempt() + # 40023, disable to decrease state.dat json lines + # relay_list.increment_recent_measurement_attempt() target.increment_relay_recent_measurement_attempt() num_relays += 1 # callback and callback_err must be non-blocking diff --git a/tests/integration/test_files.py b/tests/integration/test_files.py index aa7973d..c44340c 100644 --- a/tests/integration/test_files.py +++ b/tests/integration/test_files.py @@ -32,7 +32,9 @@ def test_state(conf): assert 1 == state.count("recent_consensus") assert 1 == state.count("recent_priority_list") assert 15 == state.count("recent_priority_relay") - assert 15 == state.count("recent_measurement_attempt") + # Because of 40023, it's 0. Change to 15 if we store it again at some point + # assert 15 == state.count("recent_measurement_attempt") + assert not state.count("recent_measurement_attempt") def test_v3bwfile(conf): @@ -42,7 +44,8 @@ def test_v3bwfile(conf): assert "1" == bwfile.header.recent_consensus_count assert "1" == bwfile.header.recent_priority_list_count assert "15" == bwfile.header.recent_priority_relay_count - assert "15" == bwfile.header.recent_measurement_attempt_count + # Because of 40023, there is not this header + # assert "15" == bwfile.header.recent_measurement_attempt_count for bwline in bwfile.bw_lines: assert 1 == bwline.relay_in_recent_consensus_count assert 1 == bwline.relay_recent_priority_list_count diff --git a/tests/unit/lib/test_heartbeat.py b/tests/unit/lib/test_heartbeat.py index b2ad634..8a2b8ea 100644 --- a/tests/unit/lib/test_heartbeat.py +++ b/tests/unit/lib/test_heartbeat.py @@ -1,10 +1,12 @@ """Unit tests for heartbeat""" import logging +import pytest from sbws.lib import heartbeat from sbws.util.state import State +(a)pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled") def test_total_measured_percent(conf, caplog): state = State(conf["paths"]["state_fname"]) state["recent_priority_list"] = [1, 2, 3] diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 07f59d7..d1448d1 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -4,6 +4,7 @@ import json import logging import math import os.path +import pytest from unittest import mock from sbws import __version__ as version @@ -588,6 +589,7 @@ def test_recent_consensus_count(root_data_path, datadir): assert "1" == header.recent_consensus_count +(a)pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled") def test_recent_measurement_attempt_count(root_data_path, datadir): state_fpath = os.path.join(root_data_path, '.sbws/state.dat') assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file(

1 0

[sbws/maint-1.1] Merge remote-tracking branch 'gitlab/merge-requests/34' into maint-1.1
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 8fdd85b73315a5dd72de68c8a876ca9fe823fe7d Merge: 81d2bea f014084 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Nov 30 10:02:29 2020 +0000 Merge remote-tracking branch 'gitlab/merge-requests/34' into maint-1.1 sbws/core/scanner.py | 3 ++- tests/integration/test_files.py | 7 +++++-- tests/unit/lib/test_heartbeat.py | 2 ++ tests/unit/lib/test_v3bwfile.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-)

1 0

[sbws/maint-1.1] chg: circuitbuilder: Simplify building circuit
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit b3ab8ebf9209f4e396f42e254afcca58fefc1e58 Author: juga0 <juga(a)riseup.net> Date: Tue Dec 8 15:43:23 2020 +0000 chg: circuitbuilder: Simplify building circuit Since sbws is only building 2 hop paths, there is no need to add random relays to the path, or convert back and forth between fingerprint and ``Relay`` objects. This will eliminate the circuit errors: - Tor seems to no longer think XXX is a relay - Can not build a circuit, no path. - Can not build a circuit with the current relays. If a relay is not longer running when attempting to build the circuit, it will probably fail with one of the other circuit errors: TIMEOUT, DESTROYED or CHANNEL_CLOSED. Closes: #40027 --- sbws/lib/circuitbuilder.py | 62 ++++++---------------------------------------- 1 file changed, 8 insertions(+), 54 deletions(-) diff --git a/sbws/lib/circuitbuilder.py b/sbws/lib/circuitbuilder.py index 45b807f..fb23737 100644 --- a/sbws/lib/circuitbuilder.py +++ b/sbws/lib/circuitbuilder.py @@ -91,62 +91,16 @@ class CircuitBuilder: # build the circuit, the relays are not just choosen as random as this class # does. class GapsCircuitBuilder(CircuitBuilder): - ''' The build_circuit member function takes a list. Falsey values in the - list will be replaced with relays chosen uniformally at random; Truthy - values will be assumed to be relays. ''' + """Same as ``CircuitBuilder`` but implements build_circuit.""" def __init__(self, *a, **kw): super().__init__(*a, **kw) - def _normalize_path(self, path): - ''' Change fingerprints/nicks to relay descriptor and change Falsey - values to None. Return the new path, or None if error ''' - new_path = [] - for fp in path: - if not fp: - new_path.append(None) - continue - relay = Relay(fp, self.controller) - if not relay.fingerprint: - log.debug('Tor seems to no longer think %s is a relay', fp) - return None - new_path.append(relay) - return new_path - - def _random_sample_relays(self, number, blacklist): - ''' Get <number> random relays from self.relays that are not in the - blacklist. Return None if it cannot be done because too many are - blacklisted. Otherwise return a list of relays. ''' - all_fps = [r.fingerprint for r in self.relays] - black_fps = [r.fingerprint for r in blacklist] - if len(black_fps) + number > len(all_fps): - return None - chosen_fps = [] - while len(chosen_fps) < number: - choice = self.rng.choice(all_fps) - if choice in black_fps: - continue - chosen_fps.append(choice) - black_fps.append(choice) - return [Relay(fp, self.controller) for fp in chosen_fps] - def build_circuit(self, path): - ''' <path> is a list of relays and Falsey values. Relays can be - specified by fingerprint or nickname, and fingerprint is highly - recommended. Falsey values (like None) will be replaced with relays - chosen uniformally at random. A relay will not be in a circuit twice. - ''' - if not valid_circuit_length(path): - return None, "Can not build a circuit, invalid path." - path = self._normalize_path(path) - if path is None: - return None, "Can not build a circuit, no path." - num_missing = len(['foo' for r in path if not r]) - insert_relays = self._random_sample_relays( - num_missing, [r for r in path if r is not None]) - if insert_relays is None: - path = ','.join([r.nickname if r else str(None) for r in path]) - return None, "Can not build a circuit with the current relays." - assert len(insert_relays) == num_missing - path = [r.fingerprint if r else insert_relays.pop().fingerprint - for r in path] + """Return parent class build circuit method. + + Since sbws is only building 2 hop paths, there is no need to add random + relays to the path, or convert back and forth between fingerprint and + ``Relay`` objects. + + """ return self._build_circuit_impl(path)

1 0

[sbws/maint-1.1] fix: v3bwfile: Tor version added in bandwidth v1.4.0
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 50821110c908e5da9f1aab3b14883e39bb226614 Author: juga0 <juga(a)riseup.net> Date: Mon Dec 7 11:28:42 2020 +0000 fix: v3bwfile: Tor version added in bandwidth v1.4.0 since, by mistake, the bandwidth file version here was never updated to v1.5.0. This patch only changes the constants names, but logic remains the same. Related to torspec#35. --- sbws/lib/v3bwfile.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 45563b6..553265b 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -125,13 +125,9 @@ HEADER_KEYS_V1_4 = [ 'time_to_report_half_network', ] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS -# KeyValues added in the Bandwidth File v1.5.0 -# XXX: Change SPEC_VERSION when all the v1.5.0 keys are added, before a new -# sbws release. # Tor version will be obtained from the state file, so it won't be pass as an # argument, but will be self-initialized. -HEADER_KEYS_V1_5_TO_INIT = ['tor_version'] -HEADER_KEYS_V1_5 = HEADER_KEYS_V1_5_TO_INIT +HEADER_KEYS_V1_4_TO_INIT = ['tor_version'] # KeyValues that are initialized from arguments, not self-initialized. HEADER_INIT_KEYS = ( @@ -139,7 +135,7 @@ HEADER_INIT_KEYS = ( + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 - + HEADER_KEYS_V1_5_TO_INIT + + HEADER_KEYS_V1_4_TO_INIT ) HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 @@ -150,7 +146,7 @@ HEADER_UNORDERED_KEYS = ( + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 - + HEADER_KEYS_V1_5 + + HEADER_KEYS_V1_4_TO_INIT ) # List of all the KeyValues currently being used to generate the file HEADER_ALL_KEYS = HEADER_KEYS_V1_1_ORDERED + HEADER_UNORDERED_KEYS

1 0

[sbws/maint-1.1] fix: Bump bandwidth file version to 1.5.0
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 378f1792cb9e5af1c9b46088f1dc30b3ec9bc89f Author: juga0 <juga(a)riseup.net> Date: Mon Dec 7 11:31:52 2020 +0000 fix: Bump bandwidth file version to 1.5.0 after removing KeyValue recent_measurement_attempt_count in #40023. Changed also torspec, issue #20. Closes: #40026. --- sbws/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/globals.py b/sbws/globals.py index 575a19f..253e2aa 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) RESULT_VERSION = 4 WIRE_VERSION = 1 -SPEC_VERSION = '1.4.0' +SPEC_VERSION = '1.5.0' # This is a dictionary of torrc options we always want to set when launching # Tor and that do not depend on any runtime configuration

1 0

[sbws/maint-1.1] chg: circuitbuilder: Remove not used attributes
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit cb97fe8c553eec48edef32becc4acecf877d18a9 Author: juga0 <juga(a)riseup.net> Date: Tue Dec 8 15:48:45 2020 +0000 chg: circuitbuilder: Remove not used attributes and make argument optional. --- sbws/lib/circuitbuilder.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/sbws/lib/circuitbuilder.py b/sbws/lib/circuitbuilder.py index fb23737..81742f1 100644 --- a/sbws/lib/circuitbuilder.py +++ b/sbws/lib/circuitbuilder.py @@ -1,7 +1,5 @@ from stem import CircuitExtensionFailed, InvalidRequest, ProtocolError, Timeout from stem import InvalidArguments, ControllerError, SocketClosed -import random -from .relaylist import Relay import logging log = logging.getLogger(__name__) @@ -25,24 +23,14 @@ class CircuitBuilder: them, but CircuitBuilder will keep track of existing circuits and close them when it is deleted. ''' - def __init__(self, args, conf, controller, relay_list, + # XXX: In new major version, remove args and conf, they are not used. + def __init__(self, args, conf, controller, relay_list=None, close_circuits_on_exit=True): self.controller = controller - self.rng = random.SystemRandom() - self.relay_list = relay_list self.built_circuits = set() self.close_circuits_on_exit = close_circuits_on_exit self.circuit_timeout = conf.getint('general', 'circuit_timeout') - @property - def relays(self): - return self.relay_list.relays - - def build_circuit(self, *a, **kw): - ''' Implementations of this method should build the circuit and return - its (str) ID. If it cannot be built, it should return None. ''' - raise NotImplementedError() - def close_circuit(self, circ_id): try: self.controller.close_circuit(circ_id)

1 0

[sbws/maint-1.1] fix: config: Increment circuit build timeout
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 396ccefbd5814ed00526d7385fa9562edaede1ba Author: juga0 <juga(a)riseup.net> Date: Fri Dec 11 14:34:18 2020 +0000 fix: config: Increment circuit build timeout setting it to the default, 60secs. Since many relays fail to be measured cause of circuit timeout. Maybe closes #40029. --- sbws/config.default.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index f159b68..cb943d0 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -25,7 +25,7 @@ http_timeout = 10 # to give to stem for waiting on a circuit to be built since # CircuitBuildTimeout doesn't handle the case of a TLS connection to a relay # taking forever, and probably other not-yet-discovered cases. -circuit_timeout = 10 +circuit_timeout = 60 # Whether or not to reset the bandwidth measurements when the relay's IP # address changes. If it changes, we only consider results for the relay that # we obtained while the relay was located at its most recent IP address.

1 0

[sbws/maint-1.1] fix: scanner: Retry to measure exit as exit
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 033603a607a8aede9ba4496b029d2545808e560e Author: juga0 <juga(a)riseup.net> Date: Sun Dec 13 13:04:27 2020 +0000 fix: scanner: Retry to measure exit as exit if it fails to be measured as entry. Mayb closes: #40029. --- sbws/core/scanner.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 8b6a6a4..5e6cac3 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -318,6 +318,17 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # Build the circuit circ_id, reason = cb.build_circuit(circ_fps) + if not circ_id and relay.fingerprint == circ_fps[0]: + # We detected that some exits fail to build circuits as 1st hop. + # If that's the case, try again using them as 2nd hop. + # We could reuse the helper, but it does not need to be an exit now, + # so choose other again. + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=False) + if helper: + circ_fps = [helper.fingerprint, relay.fingerprint] + nicknames = [helper.nickname, relay.nickname] + circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: log.debug('Could not build circuit with path %s (%s): %s ', circ_fps, nicknames, reason)

1 0

[sbws/maint-1.1] Merge remote-tracking branch 'gitlab/merge-requests/39' into maint-1.1
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit a66516f8b5dc90b7caa039e442c020c6a8e04d9d Merge: 378f179 cb97fe8 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Dec 11 16:37:06 2020 +0000 Merge remote-tracking branch 'gitlab/merge-requests/39' into maint-1.1 sbws/lib/circuitbuilder.py | 78 ++++++---------------------------------------- 1 file changed, 10 insertions(+), 68 deletions(-)

1 0

[sbws/maint-1.1] fix: relaylist: Comment on IPv6 exit policy
by juga＠torproject.org 15 Dec '20

15 Dec '20

commit 3a4a47d302d5b23b9452a073c1b75f3daaf88608 Author: juga0 <juga(a)riseup.net> Date: Sun Dec 13 13:12:54 2020 +0000 fix: relaylist: Comment on IPv6 exit policy that could be also checked, increasing the chances that the exit can exit to our Web servers. But if it could not, then we need to retry to measure it as 1st hop. --- sbws/lib/relaylist.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index dfd8b76..8a07ce7 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -205,6 +205,11 @@ class Relay: # and ips and that port. See #40006. # Using `strip_private` to ignore reject rules to private # networks. + # We could increase the chances that the exit can exit + # checking IPv6 with: + # ``or self.exit_policy_v6.can_exit_to(port=443, strict=True)`` + # But if it can still not exit to our Web server, then we + # should retry to measure it as entry. return ( self.exit_policy.strip_private() .can_exit_to(port=port, strict=True)

1 0