February 2021 - tor-commits - lists.torproject.org

[sbws/master] Merge remote-tracking branch 'gitlab/merge-requests/50' into maint-1.1
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit c40e6e11f3d064f7c5fc8ec1895081ae673781fc Merge: e37c472 cb6a8a7 Author: Georg Koppen <gk(a)torproject.org> Date: Thu Jan 7 22:55:33 2021 +0000 Merge remote-tracking branch 'gitlab/merge-requests/50' into maint-1.1 docs/source/man_sbws.ini.rst | 4 +++ sbws/config.default.ini | 1 + sbws/core/scanner.py | 18 ++---------- sbws/util/config.py | 4 ++- sbws/util/stem.py | 66 +++++++++++++++++++++---------------------- tests/integration/conftest.py | 4 +-- 6 files changed, 45 insertions(+), 52 deletions(-)

1 0

[sbws/master] fix: stem: Add possible exception cause
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit cb6a8a76da0d77ef2896c9f99003649b3d43c1b4 Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:55:13 2020 +0000 fix: stem: Add possible exception cause --- sbws/util/stem.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sbws/util/stem.py b/sbws/util/stem.py index 5605c29..ce5253c 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -181,9 +181,12 @@ def set_torrc_options_can_fail(controller): for k, v in TORRC_OPTIONS_CAN_FAIL.items(): try: controller.set_conf(k, v) - except InvalidArguments as error: + except (InvalidArguments, InvalidRequest) as error: log.debug('Ignoring option not supported by this Tor version. %s', error) + except ControllerError as e: + log.exception(e) + exit(1) def launch_tor(conf):

1 0

[sbws/master] Merge branch 'bug_40035' into 'maint-1.1'
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit e37c47238f3cbea71efa0ad4ef237f8e1423efd6 Merge: 713d400 17e6eca Author: juga <juga(a)riseup.net> Date: Thu Jan 7 15:51:35 2021 +0000 Merge branch 'bug_40035' into 'maint-1.1' fix: doc: Update values in config_tor.rst + clean-up Closes #40035 See merge request tpo/network-health/sbws!49 docs/source/config_tor.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)

1 0

[sbws/master] chg: stem, scanner: Change args initializing controller
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 80f379ac5f12078ca358d7645beee5f4278acd95 Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:39:31 2020 +0000 chg: stem, scanner: Change args initializing controller to check whether the external control port configuration is set. There is no need to assert all argument options nor to return the error. --- sbws/core/scanner.py | 3 +-- sbws/util/stem.py | 30 +++++++++++------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 5e6cac3..bcbeb47 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -679,8 +679,7 @@ def run_speedtest(args, conf): """ global rd, pool, controller - controller, _ = stem_utils.init_controller( - path=conf.getpath('tor', 'control_socket')) + controller = stem_utils.init_controller(conf) if not controller: controller = stem_utils.launch_tor(conf) else: diff --git a/sbws/util/stem.py b/sbws/util/stem.py index 746fa88..169acda 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -60,27 +60,19 @@ def remove_event_listener(controller, func): log.exception("Exception trying to remove event %s", e) -def init_controller(port=None, path=None, set_custom_stream_settings=True): - # NOTE: we do not currently support a control port even though the rest of - # this function will pretend like port could be set. - assert port is None - # make sure only one is set - assert port is not None or path is not None - assert not (port is not None and path is not None) - # and for the one that is set, make sure it is likely valid - assert port is None or isinstance(port, int) - assert path is None or isinstance(path, str) +def init_controller(conf): c = None - if port: - c = _init_controller_port(port) - if not c: - return None, 'Unable to reach tor on control port' + # If the external control port is set, use it to initialize the controller. + control_port = conf['tor']['external_control_port'] + if control_port: + control_port = int(control_port) + # If it can not connect, the program will exit here + c = _init_controller_port(control_port) else: - c = _init_controller_socket(path) - if not c: - return None, 'Unable to reach tor on control socket' - assert c is not None - return c, '' + c = _init_controller_socket( + socket=conf.getpath('tor', 'control_socket') + ) + return c def is_bootstrapped(c):

1 0

[sbws/master] fix: doc: Add forgotten image from consensus health
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 8805a1d2f68260a3f996e16a8938aff954af6172 Author: juga0 <juga(a)riseup.net> Date: Mon Jan 11 07:37:25 2021 +0000 fix: doc: Add forgotten image from consensus health It was referenced by 6e6a8f3ba534cbd93b830fe3ffd5ce40abe8e77d. Since that image was wrong, created a new screenshot from the current "past 90 days" at consensus-health.tpo. --- .../source/images/20210111_consensushealth_bwauths.png | Bin 0 -> 100029 bytes docs/source/monitoring_bandwidth.rst | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/images/20210111_consensushealth_bwauths.png b/docs/source/images/20210111_consensushealth_bwauths.png new file mode 100644 index 0000000..6898fd2 Binary files /dev/null and b/docs/source/images/20210111_consensushealth_bwauths.png differ diff --git a/docs/source/monitoring_bandwidth.rst b/docs/source/monitoring_bandwidth.rst index d9ebdf2..b3a6c65 100644 --- a/docs/source/monitoring_bandwidth.rst +++ b/docs/source/monitoring_bandwidth.rst @@ -15,7 +15,7 @@ Bwauths number of measured relays It should be approximately equal for all bwauths. -.. image:: images/20200927_consensushealth_bwauths.png +.. image:: images/20210111_consensushealth_bwauths.png :alt: bwauths measured relays https://consensus-health.torproject.org/graphs.html#votedaboutgraphs

1 0

[sbws/master] chg: stem: Add function to connect or start tor
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit f7847ed28cab55bd1260d307b42c4ef59772937a Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:51:55 2020 +0000 chg: stem: Add function to connect or start tor Move initialization via existing socket to this new function and start tor only when it fails. Closes: #33150. --- sbws/core/scanner.py | 17 ++--------------- sbws/util/stem.py | 22 ++++++++++++++++------ tests/integration/conftest.py | 4 ++-- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index bcbeb47..312993a 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -679,21 +679,8 @@ def run_speedtest(args, conf): """ global rd, pool, controller - controller = stem_utils.init_controller(conf) - if not controller: - controller = stem_utils.launch_tor(conf) - else: - log.warning( - 'Is sbws already running? ' - 'We found an existing Tor process at %s. We are not going to ' - 'launch Tor, nor are we going to try to configure it to behave ' - 'like we expect. This might work okay, but it also might not. ' - 'If you experience problems, you should try letting sbws launch ' - 'Tor for itself. The ability to use an already running Tor only ' - 'exists for sbws developers. It is expected to be broken and may ' - 'even lead to messed up results.', - conf.getpath('tor', 'control_socket')) - time.sleep(15) + + controller = stem_utils.launch_or_connect_to_tor(conf) # When there will be a refactor where conf is global, this can be removed # from here. diff --git a/sbws/util/stem.py b/sbws/util/stem.py index 169acda..5605c29 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -6,7 +6,6 @@ from stem import (SocketError, InvalidRequest, UnsatisfiableRequest, ProtocolError, SocketClosed) from stem.connection import IncorrectSocketType import stem.process -from configparser import ConfigParser from threading import RLock import copy import logging @@ -68,10 +67,8 @@ def init_controller(conf): control_port = int(control_port) # If it can not connect, the program will exit here c = _init_controller_port(control_port) - else: - c = _init_controller_socket( - socket=conf.getpath('tor', 'control_socket') - ) + # There is no configuration for external control socket, therefore do not + # attempt to connect to the control socket. return c @@ -190,7 +187,6 @@ def set_torrc_options_can_fail(controller): def launch_tor(conf): - assert isinstance(conf, ConfigParser) os.makedirs(conf.getpath('tor', 'datadir'), mode=0o700, exist_ok=True) os.makedirs(conf.getpath('tor', 'log'), exist_ok=True) os.makedirs(conf.getpath('tor', 'run_dpath'), mode=0o700, exist_ok=True) @@ -216,6 +212,8 @@ def launch_tor(conf): torrc = parse_user_torrc_config(torrc, conf['tor']['extra_lines']) # Finally launch Tor try: + # If there is already a tor process running with the same control + # socket, this will exit here. stem.process.launch_tor_with_config( torrc, init_msg_handler=log.debug, take_ownership=True) except Exception as e: @@ -223,6 +221,18 @@ def launch_tor(conf): log.info("Started own tor.") # And return a controller to it cont = _init_controller_socket(conf.getpath('tor', 'control_socket')) + # In the case it was not possible to connect to own tor socket. + if not cont: + fail_hard('Could not connect to own tor control socket.') + return cont + + +def launch_or_connect_to_tor(conf): + # If connecting to an existing controller, there is no need to configure + # own tor. + cont = init_controller(conf) + if not cont: + cont = launch_tor(conf) # Set options that can fail at runtime set_torrc_options_can_fail(cont) # Set runtime options diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index afd7256..2c0e675 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,7 @@ from sbws.lib.circuitbuilder import GapsCircuitBuilder as CB from sbws.lib.destination import DestinationList from sbws.lib.relaylist import RelayList from sbws.util.config import _get_default_config -from sbws.util.stem import launch_tor +from sbws.util.stem import launch_or_connect_to_tor class _PseudoArguments(argparse.Namespace): @@ -91,7 +91,7 @@ SafeLogging 0 @pytest.fixture(scope='session') def persistent_launch_tor(conf): - cont = launch_tor(conf) + cont = launch_or_connect_to_tor(conf) return cont

1 0

[sbws/master] fix: stem: Exit on failure connecting to control port
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 6cfd60ed6272afa15458d893ffbbe2fe3703f87c Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:21:44 2020 +0000 fix: stem: Exit on failure connecting to control port because when trying to connect to an external tor (chutney), it does not make sense to start own tor. Also log how the connection has been made. --- sbws/util/stem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sbws/util/stem.py b/sbws/util/stem.py index 05f8e90..d2c596f 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -110,8 +110,9 @@ def _init_controller_port(port): c = Controller.from_port(port=port) c.authenticate() except (IncorrectSocketType, SocketError): - return None + fail_hard("Unable to connect to control port %s.", port) # TODO: Allow for auth via more than just CookieAuthentication + log.info("Connected to tor via port %s", port) return c @@ -127,6 +128,7 @@ def _init_controller_socket(socket): log.exception("Error initting controller socket: %s", e) return None # TODO: Allow for auth via more than just CookieAuthentication + log.info("Connected to tor via socket %s", socket) return c @@ -233,6 +235,7 @@ def launch_tor(conf): torrc, init_msg_handler=log.debug, take_ownership=True) except Exception as e: fail_hard('Error trying to launch tor: %s', e) + log.info("Started own tor.") # And return a controller to it cont = _init_controller_socket(conf.getpath('tor', 'control_socket')) # Set options that can fail at runtime @@ -240,8 +243,7 @@ def launch_tor(conf): # Set runtime options set_torrc_runtime_options(cont) - log.info('Started and connected to Tor %s via %s', cont.get_version(), - conf.getpath('tor', 'control_socket')) + log.info('Started or connected to Tor %s.', cont.get_version()) return cont

1 0

[sbws/master] fix: relaylist: Stop measuring relays not in the consenus
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit a050110db59d136f6f46809c4b18fe487012e546 Author: juga0 <juga(a)riseup.net> Date: Mon Dec 21 16:24:50 2020 +0000 fix: relaylist: Stop measuring relays not in the consenus as this might cause many circuit errors. They're already added to the generator. Also adapt the number in test_init_relays. Closes: #40037. Happy solstice! :) --- sbws/lib/relaylist.py | 17 +++++------------ tests/unit/lib/test_relaylist.py | 8 ++++---- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 8a07ce7..640e3eb 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -14,7 +14,7 @@ from ..globals import ( MAX_RECENT_PRIORITY_LIST_COUNT, MEASUREMENTS_PERIOD ) -from ..util import timestamp, timestamps +from ..util import timestamps log = logging.getLogger(__name__) @@ -250,12 +250,6 @@ class Relay: def relay_recent_priority_list_count(self): return len(self.relay_recent_priority_list) - def is_old(self): - """Whether the last consensus seen for this relay is older than the - measurement period. - """ - return timestamp.is_old(self.last_consensus_timestamp) - # XXX: tech-debt: replace `_desc` attr by a a `dequee` of the last # descriptors seen for this relay and the timestamp. def update_server_descriptor(self, server_descriptor): @@ -417,11 +411,10 @@ class RelayList: # already added to the new list. new_relays_dict.pop(fp) - # If the relay is not in the current consensus but is not "old" - # yet, add it to the new list of relays too, though its timestamp, - # router status and descriptor can't be updated. - elif not r.is_old(): - new_relays.append(r) + # In #30727, the relay that is not in the current conensus but is + # not "old", was added to the new list of relays too. + # In #40037 we think it should not be measured, as it might cause + # many circuit errors. It's already added to the generator. # Otherwise, don't add it to the new list of relays. # For debugging, count the old relays that will be discarded. else: diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 31673ba..399d897 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -51,7 +51,7 @@ def test_init_relays( assert 6505 == 6433 + len(added_fps) # The calculated min bw for the second hop assert 2120000 == relay_list._exit_min_bw - assert 200000 == relay_list._non_exit_min_bw + assert 210000 == relay_list._non_exit_min_bw # Five days later plus 1 second. # The first consensus timestamp will get removed. @@ -69,10 +69,10 @@ def test_init_relays( removed_fps = fps.difference(fps_5days_later) # The number of relays will be the number of relays in the cosensus plus # the added ones minus the removed ones. - assert 6925 == 6505 + len(added_fps) - len(removed_fps) + assert 6596 == 6505 + len(added_fps) - len(removed_fps) # The calculated min bw for the second hop - assert 2790000 == relay_list._exit_min_bw - assert 110000 == relay_list._non_exit_min_bw + assert 2800000 == relay_list._exit_min_bw + assert 150000 == relay_list._non_exit_min_bw def test_increment_recent_measurement_attempt(args, conf, controller):

1 0

[sbws/master] fix: doc: Update values in config_tor.rst + clean-up
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 17e6eca63645271cbf2f5553b1a71aafd3f542cd Author: Georg Koppen <gk(a)torproject.org> Date: Thu Jan 7 14:38:00 2021 +0000 fix: doc: Update values in config_tor.rst + clean-up Closes #40035. --- docs/source/config_tor.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/source/config_tor.rst b/docs/source/config_tor.rst index 175a9ff..e609468 100644 --- a/docs/source/config_tor.rst +++ b/docs/source/config_tor.rst @@ -3,7 +3,7 @@ Internal Tor configuration for the scanner ------------------------------------------ -The scanner needs an specific Tor configuration. +The scanner needs a specific Tor configuration. The following options are either set when launching Tor or required when connection to an existing Tor daemon. @@ -16,6 +16,9 @@ Default configuration: - ``SafeLogging 0``: Useful for logging, since there's no need for anonymity. - ``LogTimeGranularity 1`` - ``ProtocolWarnings 1`` +- ``FetchDirInfoEarly 1`` +- ``FetchDirInfoExtraEarly 1``: Respond to `MaxAdvertisedBandwidth` as soon as possible. +- ``FetchUselessDescriptors 1``: Keep fetching descriptors, even when idle. - ``LearnCircuitBuildTimeout 0``: To keep circuit build timeouts static. Configuration that depends on the user configuration file: @@ -29,9 +32,13 @@ Configuration that depends on the user configuration file: Configuration that needs to be set on runtime: - ``__DisablePredictedCircuits 1``: To build custom circuits. -- ``__LeaveStreamsUnattached 1`` +- ``__LeaveStreamsUnattached 1``: The scanner is attaching the streams itself. + +Configuration that can be set on runtime and fail: + +- ``ConnectionPadding 0``: Useful for avoiding extra traffic, since scanner anonymity is not a goal. Currently most of the code that sets this configuration is in :func:`sbws.util.stem.launch_tor` and the default configuration is ``sbws/globals.py``. -.. note:: the location of these code is being refactored. \ No newline at end of file +.. note:: the location of this code is being refactored.

1 0

[sbws/master] chg: config: Add option to connect to external tor
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit f6e300c83508508a6f38abe76390f9c6c3de00a2 Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:14:48 2020 +0000 chg: config: Add option to connect to external tor via control port. --- docs/source/man_sbws.ini.rst | 4 ++++ sbws/config.default.ini | 1 + sbws/util/config.py | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/man_sbws.ini.rst b/docs/source/man_sbws.ini.rst index f878ce8..e2127ab 100644 --- a/docs/source/man_sbws.ini.rst +++ b/docs/source/man_sbws.ini.rst @@ -93,6 +93,10 @@ tor sbws's owned tor pid file. (Default: ~/.sbws/tor/sbws/tor.pid) log = STR sbws's owned tor directory log files. (Default: ~/.sbws/tor/log) + external_control_port = INT + tor control port to connect to. Useful to run integration tests with + chutney. + (Default: not set. If set, it takes preference over the control socket) extra_lines = sbws's tor extra configuration. (Default: None) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index cb943d0..0105527 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -66,6 +66,7 @@ control_socket = ${tor:run_dpath}/control pid = ${tor:run_dpath}/tor.pid # note this is a directory log = ${tor:datadir}/log +external_control_port = extra_lines = [cleanup] diff --git a/sbws/util/config.py b/sbws/util/config.py index e1bc429..5ba3fd5 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -323,7 +323,9 @@ def _validate_tor(conf): sec = 'tor' err_tmpl = Template('$sec/$key ($val): $e') unvalidated_keys = [ - 'datadir', 'run_dpath', 'control_socket', 'pid', 'log', 'extra_lines'] + 'datadir', 'run_dpath', 'control_socket', 'pid', 'log', + 'external_control_port', 'extra_lines', + ] all_valid_keys = unvalidated_keys errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) return errors

1 0