June 2018 - tor-commits - lists.torproject.org

[sbws/master] 'time' isn't always a program: sometimes it's a bash feature
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 8f5f590a4d1af216afd1fd2130c7465ed19e6b1f Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 6 20:48:07 2018 -0400 'time' isn't always a program: sometimes it's a bash feature --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8d0b52e..5134fb4 100644 --- a/tox.ini +++ b/tox.ini @@ -22,11 +22,10 @@ deps = .[test] whitelist_externals = tar bash - time commands = tar -C {envtmpdir} -vxf {toxinidir}/tests/integration/net.tar bash {envtmpdir}/net/start.sh - time bash -c "python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" + bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest -s {toxinidir}/tests/integration -vv bash {envtmpdir}/net/stop.sh

1 0

[sbws/master] Clarify: no, there doesn't seem to be a nice way to do this
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 59e4d88add83fc8014b419cde07b61809a0f2f39 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 6 21:40:05 2018 -0400 Clarify: no, there doesn't seem to be a nice way to do this --- sbws/config.log.default.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini index 6c51326..3d8d9bd 100644 --- a/sbws/config.log.default.ini +++ b/sbws/config.log.default.ini @@ -58,6 +58,8 @@ class = handlers.RotatingFileHandler formatter = sbwssys # when not using level, it seems to be using the one passed as cli arg, # which is the desired behaviour -# dunno if there is a way to have log file as a variable -# setting one as default +# There doesn't seem to be a way to put ${paths:log_filepath} here since the +# logging library eval()s this, which doesn't give the ini parsing library a +# chance to replace the text. So we do it in configure_logging() in +# sbws/util/config.py instead. args = ('/nonexist',)

1 0

[sbws/master] Move tests data files to new unit dir
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 75d19b8ef81e57e67362e06db712fde9142dd6ee Author: juga0 <juga(a)riseup.net> Date: Tue Jun 5 12:38:42 2018 +0000 Move tests data files to new unit dir --- tests/{ => unit}/lib/data/results.txt | 0 tests/{ => unit}/lib/data/v3bw.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/lib/data/results.txt b/tests/unit/lib/data/results.txt similarity index 100% rename from tests/lib/data/results.txt rename to tests/unit/lib/data/results.txt diff --git a/tests/lib/data/v3bw.txt b/tests/unit/lib/data/v3bw.txt similarity index 100% rename from tests/lib/data/v3bw.txt rename to tests/unit/lib/data/v3bw.txt

1 0

[sbws/master] Indeed, handlers don't necessarily need log levels
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 8674d780ee913609995cc2a546c5fb527029897a Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 6 21:41:30 2018 -0400 Indeed, handlers don't necessarily need log levels --- sbws/config.log.default.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini index 3d8d9bd..9bbe7aa 100644 --- a/sbws/config.log.default.ini +++ b/sbws/config.log.default.ini @@ -56,8 +56,6 @@ datefmt = %b %d %H:%M:%S [handler_sbwsfile] class = handlers.RotatingFileHandler formatter = sbwssys -# when not using level, it seems to be using the one passed as cli arg, -# which is the desired behaviour # There doesn't seem to be a way to put ${paths:log_filepath} here since the # logging library eval()s this, which doesn't give the ini parsing library a # chance to replace the text. So we do it in configure_logging() in

1 0

[sbws/master] Remove unneded print
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 3d9a73f25ef205d97b22b7fbada7677f5709fd26 Author: juga0 <juga(a)riseup.net> Date: Thu Jun 7 07:54:59 2018 +0000 Remove unneded print --- tests/unit/core/test_generate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/core/test_generate.py b/tests/unit/core/test_generate.py index a26ca5e..7777308 100644 --- a/tests/unit/core/test_generate.py +++ b/tests/unit/core/test_generate.py @@ -124,8 +124,6 @@ def test_generate_single_success_noscale(dotsbws_success_result, caplog, time=unixts_to_isodt_str(round(result.time)), success=1, error_circ=0, error_misc=0, error_stream=0) - # bw_line = V3BWLine.from_results(results) - print(stdout_lines) assert stdout_lines[NUM_LINES_HEADER_V110] + '\n' == str(bw_line)

1 0

[sbws/master] Merge pull request #176 from juga0/issue160
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 966354a448eb1978117e86aa8d9de90f5881bb92 Merge: f804d95 f9a8507 Author: Matt Traudt <pastly(a)users.noreply.github.com> Date: Thu Jun 7 08:30:00 2018 -0400 Merge pull request #176 from juga0/issue160 Thanks! docs/source/sbws.lib.rst | 7 ++ sbws/core/generate.py | 100 +---------------- sbws/globals.py | 3 + sbws/lib/v3bwfile.py | 232 +++++++++++++++++++++++++++++++++++++-- tests/unit/conftest.py | 34 ++++++ tests/unit/core/test_generate.py | 63 ++++++----- tests/unit/lib/data/results.txt | 2 + tests/unit/lib/data/v3bw.txt | 10 ++ tests/unit/lib/test_v3bwfile.py | 104 +++++++++++++++++- 9 files changed, 422 insertions(+), 133 deletions(-)

1 0

[sbws/master] Use keyserver pool to avoid single server timeout (#185)
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit f804d950b1dec58f3d40a3444390a667a1b29799 Author: juga0 <juga(a)riseup.net> Date: Thu Jun 7 12:28:00 2018 +0000 Use keyserver pool to avoid single server timeout (#185) * Use keyserver pool to avoid single server timeout travis test failed because keys.gnupg.net timeout. It should not be the case using the pool. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3beac5a..b4d9e9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,8 @@ install: pip install tox-travis script: tox before_install: - - gpg --keyserver keys.gnupg.net --recv A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 + - gpg --version + - gpg --keyserver hkp://pool.sks-keyservers.net --recv A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 - gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - echo "deb https://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list - echo "deb-src https://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list

1 0

[sbws/master] Restore for else clause
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit f9a8507ab282acd0d373e104ddd6ef8587780d07 Author: juga0 <juga(a)riseup.net> Date: Thu Jun 7 08:01:45 2018 +0000 Restore for else clause --- tests/unit/core/test_generate.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/unit/core/test_generate.py b/tests/unit/core/test_generate.py index 7777308..e9b341c 100644 --- a/tests/unit/core/test_generate.py +++ b/tests/unit/core/test_generate.py @@ -86,11 +86,9 @@ def test_generate_single_error(dotsbws_error_result, caplog, parser): for record in caplog.records: if 'Keeping 0/1 read lines from {}'.format(dd) in record.getMessage(): break - else: - # FIXME: what was intended to be here? - assert None is None - # assert None, 'Unable to find log line indicating 0 success ' \ - # 'results in data file' + else: + assert None, 'Unable to find log line indicating 0 success ' \ + 'results in data file' assert 'No recent results' in caplog.records[-1].getMessage()

1 0

[sbws/master] Catch a case where we might leak circuits
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit a9b3967d7557424b871b709acaa681e6427a650c Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 7 19:29:24 2018 -0400 Catch a case where we might leak circuits --- sbws/lib/destination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index e87f71c..4d0c1f3 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -203,6 +203,7 @@ class DestinationList: is_usable, data = dest.is_usable(circ_id, session, cont) if not is_usable: log.warning(data) + self._cb.close_circuit(circ_id) continue assert is_usable log.debug('%s seems usable so we will keep it', dest.url)

1 0

[sbws/master] Initial revision of RelayNS/Relay class
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit c56eeaa962a33079685bbb7c30c944a594197275 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 7 20:54:11 2018 -0400 Initial revision of RelayNS/Relay class With unused support for ed25519 keys --- sbws/globals.py | 3 + sbws/lib/relaylist.py | 120 ++++++++++++++++++-------------- sbws/lib/resultdump.py | 4 +- sbws/util/stem.py | 3 +- tests/integration/lib/test_relaylist.py | 17 +++++ tests/unit/conftest.py | 50 +------------ tests/unit/lib/test_relaylist.py | 13 ---- 7 files changed, 93 insertions(+), 117 deletions(-) diff --git a/sbws/globals.py b/sbws/globals.py index a6621ab..a16682c 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -22,6 +22,9 @@ TORRC_STARTING_POINT = { 'CookieAuthentication': '1', # To avoid path bias warnings 'UseEntryGuards': '0', + # Because we need things from full server descriptors (namely for now: the + # bandwidth line) + 'UseMicrodescriptors': '0', } SCALE_CONSTANT = 7500 diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 6d0743d..c0ca9ad 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -1,5 +1,5 @@ from stem.descriptor.router_status_entry import RouterStatusEntryV3 - +from stem.descriptor.server_descriptor import ServerDescriptor import sbws.util.stem as stem_utils from stem import Flag from stem import DescriptorUnavailable @@ -13,16 +13,73 @@ from sbws.globals import resolve log = logging.getLogger(__name__) -class RelayNS(RouterStatusEntryV3): - """Inherit from RouterStatusEntryV3 and add the attribute - master_key_ed25519. +class Relay: + def __init__(self, fp, cont, ns=None, desc=None): + ''' + Given a relay fingerprint, fetch all the information about a relay that + sbws currently needs and store it in this class. Acts as an abstraction + to hide the confusion that is Tor consensus/descriptor stuff. + + :param str fp: fingerprint of the relay. + :param cont: active and valid stem Tor controller connection + ''' + assert isinstance(fp, str) + assert len(fp) == 40 + assert stem_utils.is_controller_okay(cont) + if ns is not None: + assert isinstance(ns, RouterStatusEntryV3) + self._ns = ns + else: + self._ns = cont.get_network_status(fp, default=None) + if desc is not None: + assert isinstance(desc, ServerDescriptor) + self._desc = desc + else: + self._desc = cont.get_server_descriptor(fp, default=None) + + def _from_desc(self, attr): + if not self._desc: + return None + assert hasattr(self._desc, attr) + return getattr(self._desc, attr) + + def _from_ns(self, attr): + if not self._ns: + return None + assert hasattr(self._ns, attr) + return getattr(self._ns, attr) + + @property + def nickname(self): + return self._from_ns('nickname') + + @property + def fingerprint(self): + return self._from_ns('fingerprint') + + @property + def flags(self): + return self._from_ns('flags') + + @property + def exit_policy(self): + return self._from_desc('exit_policy') + + @property + def average_bandwidth(self): + return self._from_desc('average_bandwidth') + + @property + def bandwidth(self): + return self._from_ns('bandwidth') + + @property + def address(self): + return self._from_ns('address') - :param str ed25519: the ed25519 master key base 64 encoded. - """ - def __init__(self, ed25519=None, *args, **kwargs): - super().__init__(*args, **kwargs) - if ed25519 is not None: - self.master_key_ed25519 = ed25519 + @property + def ed25519_master_key(self): + return self._from_desc('ed25519_master_key').rstrip('=') class RelayList: @@ -82,40 +139,6 @@ class RelayList: # return [r for r in relays if r.measured is not None] return [r for r in relays if not r.is_unmeasured] - def relay_ed25519_master_key(self, ns): - """Obtain ed25519 master key of the relay represented by - the network status relay line. - - :param RouterStatusEntryV3 ns: the network status relay - :returns: str, the ed25519 master key base 64 encoded without - trailing '='s. - """ - # In theory this is never going to be the case? - if ns.identifier is None or ns.identifier_type != 'ed25519': - log.debug('Getting microdescriptor to obtain ed25519 identity.') - mdesc = self._controller.get_microdescriptor(ns.fingerprint, None) - if mdesc is not None: - if 'ed25519' in mdesc.identifiers.keys(): - ed25519 = mdesc.identifiers['ed25519'].rstrip('=') - log.debug('Found ed25519 master key.') - return ed25519 - log.debug('No ed25519 master-key found') - log.debug('Could not get microdescriptor') - # In case Tor can not retrive microdescriptors, - # try with server descriptors. - log.debug('Getting server descriptor to obtain ' - 'ed25519 master key.') - sdesc = self._controller.get_server_descriptor(ns.fingerprint, - None) - if sdesc is not None: - ed25519 = sdesc.ed25519_master_key().rstrip('=') - log.debug('Found ed25519 master key.') - return ed25519 - log.debug('Could not get server descriptor') - return None - log.debug('Relay has already ed25519 master key') - return ns.identifier - def exits_can_exit_to(self, host, port): ''' Return exits that can MOST LIKELY exit to the given host:port. **host** @@ -179,13 +202,8 @@ class RelayList: def _init_relays(self): c = self._controller assert stem_utils.is_controller_okay(c) - relays = [] - # for each network status relay, obtain the ed25519 master key - # and generate a new list of RelayNS objects - for ns in c.get_network_statuses(): - ed25519 = self.relay_ed25519_master_key(ns) - rns = RelayNS(ed25519=ed25519, content=ns._raw_contents) - relays.append(rns) + relays = [Relay(ns.fingerprint, c, ns=ns) + for ns in c.get_network_statuses()] return relays def _refresh(self): diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 7cff36f..ffa493d 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -11,9 +11,9 @@ from queue import Empty from datetime import datetime from datetime import timedelta from enum import Enum -from stem.descriptor.router_status_entry import RouterStatusEntryV3 from sbws.globals import RESULT_VERSION from sbws.util.filelock import DirectoryLock +from sbws.lib.relaylist import Relay log = logging.getLogger(__name__) @@ -481,7 +481,7 @@ class ResultDump: 'Ignoring %s', type(data)) def results_for_relay(self, relay): - assert isinstance(relay, RouterStatusEntryV3) + assert isinstance(relay, Relay) fp = relay.fingerprint with self.data_lock: if fp not in self.data: diff --git a/sbws/util/stem.py b/sbws/util/stem.py index 5291bba..0341d74 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -2,7 +2,6 @@ from stem.control import (Controller, Listener) from stem import (SocketError, InvalidRequest, UnsatisfiableRequest) from stem.connection import IncorrectSocketType import stem.process -from stem.descriptor.router_status_entry import RouterStatusEntryV3 from configparser import ConfigParser from threading import RLock import copy @@ -223,7 +222,7 @@ def only_relays_with_bandwidth(controller, relays, min_bw=None, max_bw=None): assert max_bw is None or max_bw >= 0 ret = [] for relay in relays: - assert isinstance(relay, RouterStatusEntryV3) + assert hasattr(relay, 'bandwidth') if min_bw is not None and relay.bandwidth < min_bw: continue if max_bw is not None and relay.bandwidth > max_bw: diff --git a/tests/integration/lib/test_relaylist.py b/tests/integration/lib/test_relaylist.py new file mode 100644 index 0000000..8f16ef8 --- /dev/null +++ b/tests/integration/lib/test_relaylist.py @@ -0,0 +1,17 @@ +from sbws.lib.relaylist import Relay + + +def test_relay_properties(persistent_launch_tor): + cont = persistent_launch_tor + # AA45C13025C037F056E734169891878ED0880231 is auth1 + fp = 'AA45C13025C037F056E734169891878ED0880231' + relay = Relay(fp, cont) + assert relay.nickname == 'auth1' + assert relay.fingerprint == 'AA45C13025C037F056E734169891878ED0880231' + assert 'Authority' in relay.flags + assert not relay.exit_policy or not relay.exit_policy.is_exiting_allowed() + assert relay.average_bandwidth == 1073741824 + assert relay.bandwidth == 0 + assert relay.address == '127.10.0.1' + assert relay.ed25519_master_key == \ + 'wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q' diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2847fe1..0701646 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,9 +2,8 @@ from sbws.lib.resultdump import ResultError from sbws.lib.resultdump import ResultSuccess from sbws.lib.resultdump import Result from sbws.lib.resultdump import write_result_to_datadir -from sbws.util.config import get_config, _get_default_config +from sbws.util.config import get_config from sbws.util.parser import create_parser -import sbws.util.stem as stem_utils import sbws.core.init from tempfile import TemporaryDirectory import pytest @@ -28,53 +27,6 @@ class _PseudoArguments(argparse.Namespace): setattr(self, key, kw[key]) -(a)pytest.fixture -def tmpdir(tmpdir_factory, request): - base = str(hash(request.node.nodeid))[:3] - bn = tmpdir_factory.mktemp(base) - return bn - - -(a)pytest.fixture() -def datadir(request): - """ get, read, open test files from the "data" directory. """ - class D: - def __init__(self, basepath): - self.basepath = basepath - - def open(self, name, mode="r"): - return self.basepath.join(name).open(mode) - - def join(self, name): - return self.basepath.join(name).strpath - - def read_bytes(self, name): - with self.open(name, "rb") as f: - return f.read() - - def read(self, name): - with self.open(name, "r") as f: - return f.read() - - def readlines(self, name): - with self.open(name, "r") as f: - return f.readlines() - return D(request.fspath.dirpath("data")) - - -(a)pytest.fixture() -def start_tor(request, tmpdir): - """Star Tor or connect to existing socket in a temporal directory.""" - conf = _get_default_config() - home = tmpdir.join('.sbws') - conf['paths']['sbws_home'] = home.strpath - controller, _ = stem_utils.init_controller( - path=conf['tor']['control_socket']) - if not controller: - controller = stem_utils.launch_tor(conf) - return controller - - @pytest.fixture(scope='session') def parser(): return create_parser() diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py deleted file mode 100644 index cdb71c2..0000000 --- a/tests/unit/lib/test_relaylist.py +++ /dev/null @@ -1,13 +0,0 @@ -from sbws.lib.relaylist import RelayList - - -def test_relaylist_master_key_ed25519(start_tor): - # This test starts tor, so it is slow. And it will fail whenever there are - # network problems - controller = start_tor - rl = RelayList(None, None, controller) - relay = [r for r in rl.relays if r.nickname == 'moria1'][0] - assert relay.fingerprint == '9695DFC35FFEB861329B9F1AB04C46397020CE31' - assert relay.identifier is None - assert relay.master_key_ed25519 == \ - 'yp0fwtp4aa/VMyZJGz8vN7Km3zYet1YBZwqZEk1CwHI'

1 0