July 2018 - tor-commits - lists.torproject.org

[sbws/master] Add changelog entry for contributing additions
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 0f82e0b41446ca6b40fe7dc78139349543f8a447 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 15:16:29 2018 -0400 Add changelog entry for contributing additions --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8bbd36..77fb16a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Summary of changes: - Maintenance script to help us find functions that are (probably) no longer being called. - Integration test(s) for RelayPrioritizer (GHPR#206) +- Git/GitHub usage guidelines to CONTRIBUTING document (GH#208 GHPR#215) ### Fixed

1 0

[sbws/master] Measure Exit+BadExit relays as non-exits
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 305e75d40b12054e23ac19eba2e4828f48bd7252 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 29 09:17:59 2018 -0400 Measure Exit+BadExit relays as non-exits --- CHANGELOG.md | 4 ++++ sbws/core/scanner.py | 3 ++- sbws/lib/relaylist.py | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77fb16a..1c3e372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Fixed + +- Measure relays that have both Exit and BadExit as non-exits, which is how + clients would use them. (GH#217) ## [0.5.0] - 2018-06-26 diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 25c1dd3..249d8d4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -178,7 +178,8 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # exit, then pick a non-exit. Otherwise pick an exit. helper = None circ_fps = None - if relay.can_exit_to(dest.hostname, dest.port): + if relay.can_exit_to(dest.hostname, dest.port) and \ + relay not in rl.bad_exits: helper = _pick_ideal_second_hop( relay, dest, rl, cb.controller, is_exit=False) if helper: diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index ddd982a..c3bb745 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -164,6 +164,10 @@ class RelayList: return self._relays_with_flag(Flag.EXIT) @property + def bad_exits(self): + return self._relays_without_flag(Flag.BADEXIT) + + @property def non_exits(self): return self._relays_without_flag(Flag.EXIT)

1 0

[sbws/master] Fix dumb logic error in RelayList.bad_exits(); Thanks @teor2345
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit d1537ac69df8d744053f8334c99f7126ad317dab Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jul 2 08:39:41 2018 -0400 Fix dumb logic error in RelayList.bad_exits(); Thanks @teor2345 GH: closes #218 --- sbws/lib/relaylist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index c3bb745..11390de 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -165,7 +165,7 @@ class RelayList: @property def bad_exits(self): - return self._relays_without_flag(Flag.BADEXIT) + return self._relays_with_flag(Flag.BADEXIT) @property def non_exits(self):

1 0

[sbws/master] Check that there are exits
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit a00625b036dbff58d2038deea91782e0ba664549 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 10:55:53 2018 +0000 Check that there are exits to perform usability tests. This will avoid raising IndexError when choosing an exit. --- sbws/lib/destination.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index 84dc580..ed1bc9b 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -186,6 +186,9 @@ class DestinationList: possible_exits = sorted( possible_exits, key=lambda e: e.bandwidth, reverse=True) exits = possible_exits[0:num_keep] + if len(exits) < 1: + log.warning("There are no exits to perform usability tests.") + continue # Try three times to build a circuit to test this destination circ_id = None for _ in range(0, 3):

1 0

[sbws/master] Add funcs for validating enums
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 11fac0c6581158586f26e94a1ac2dfe6e4ca140d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 09:57:49 2018 -0400 Add funcs for validating enums --- sbws/util/config.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sbws/util/config.py b/sbws/util/config.py index faedbd5..0bfa072 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -368,6 +368,27 @@ def _validate_section_urls(conf, sec, urls, tmpl): return errors +def _validate_section_enums(conf, sec, enums, tmpl): + errors = [] + section = conf[sec] + for key in enums: + choices = enums[key]['choices'] + valid, error = _validate_enum(section, key, choices) + if not valid: + errors.append(tmpl.substitute( + sec=sec, key=key, val=section[key], + e='Not a valid enum choice ({})'.format(', '.join(choices)))) + return errors + + +def _validate_enum(section, key, choices): + value = section[key] + if value not in choices: + return False, '{} not in allowed choices: {}'.format( + value, ', '.join(choices)) + return True, '' + + def _validate_url(section, key): value = section[key] if not value.startswith(('http://', 'https://')):

1 0

[sbws/master] Move back write function to V3BWFile
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 666b9d7f57f2080146c9ee14aa8ee59e9228849b Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 15:52:16 2018 +0000 Move back write function to V3BWFile but not allowing /dev/stdout --- sbws/core/generate.py | 28 +--------------------------- sbws/lib/v3bwfile.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 827058e..6492423 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -4,7 +4,6 @@ from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter import os import logging -from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import now_fname log = logging.getLogger(__name__) @@ -33,31 +32,6 @@ def gen_parser(sub): 'out, and we do so proportionally') -def _write_v3bw_file(v3bwfile, output): - log.info('Writing v3bw file to %s', output) - out_dir = os.path.dirname(output) - out_link = os.path.join(out_dir, 'latest.v3bw') - if os.path.exists(out_link): - log.debug('Deleting existing symlink before creating a new one.') - os.remove(out_link) - # to keep test_generate.py working - if output != '/dev/stdout': - with DirectoryLock(out_dir): - with open(output, 'wt') as fd: - fd.write(str(v3bwfile.header)) - for line in v3bwfile.bw_lines: - fd.write(str(line)) - output_basename = os.path.basename(output) - log.debug('Creating symlink from {} to {}.' - .format(output_basename, out_link)) - os.symlink(output_basename, out_link) - else: - with open(output, 'wt') as fd: - fd.write(str(v3bwfile.header)) - for line in v3bwfile.bw_lines: - fd.write(str(line)) - - def main(args, conf): if not is_initted(args.directory): fail_hard('Sbws isn\'t initialized. Try sbws init') @@ -83,5 +57,5 @@ def main(args, conf): return bw_file = V3BWFile.from_arg_results(args, conf, results) output = args.output or conf['paths']['v3bw_fname'].format(now_fname()) - _write_v3bw_file(bw_file, output) + V3BWFile.write(output) log.info('Mean bandwidth per line: %f "KiB"', bw_file.avg_bw) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 64395ef..fb9e3d3 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -3,12 +3,13 @@ (v3bw) used by bandwidth authorities.""" import logging +import os from statistics import median from sbws import __version__ from sbws.globals import SPEC_VERSION, BW_LINE_SIZE from sbws.lib.resultdump import ResultSuccess, _ResultType -from sbws.util.filelock import FileLock +from sbws.util.filelock import FileLock, DirectoryLock from sbws.util.timestamp import now_isodt_str, unixts_to_isodt_str log = logging.getLogger(__name__) @@ -405,3 +406,23 @@ class V3BWFile(object): header = V3BWHeader.from_results(conf, results) f = cls(header, bw_lines) return f + + def write(self, output): + if output == '/dev/stdout': + log.info("Writing to stdout is not supported.") + return + log.info('Writing v3bw file to %s', output) + out_dir = os.path.dirname(output) + out_link = os.path.join(out_dir, 'latest.v3bw') + if os.path.exists(out_link): + log.debug('Deleting existing symlink before creating a new one.') + os.remove(out_link) + with DirectoryLock(out_dir): + with open(output, 'wt') as fd: + fd.write(str(self.header)) + for line in self.bw_lines: + fd.write(str(line)) + output_basename = os.path.basename(output) + log.debug('Creating symlink from {} to {}.' + .format(output_basename, out_link)) + os.symlink(output_basename, out_link)

1 0

[sbws/master] Add unit tests for writing v3bw file
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 6912a1fb2aa42cdd221b189fb0c1bf8e44043630 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 16:12:29 2018 +0000 Add unit tests for writing v3bw file --- tests/unit/lib/test_v3bwfile.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index bfe9d9b..bede4ff 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Test generation of bandwidth measurements document (v3bw)""" import json +import os.path from sbws import __version__ as version from sbws.globals import SPEC_VERSION, RESULT_VERSION @@ -8,6 +9,7 @@ from sbws.lib.resultdump import Result, load_result_file from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V110, num_results_of_type, V3BWFile) +from sbws.util.timestamp import now_fname timestamp = 1523974147 timestamp_l = str(timestamp) @@ -168,3 +170,25 @@ def test_v3bwfile(datadir, tmpdir): bwls = [V3BWLine.from_results(results[fp]) for fp in results] f = V3BWFile(header, bwls) assert v3bw == str(f) + + +def test_from_arg_results(datadir, tmpdir, unittest_conf, unittest_args): + results = load_result_file(str(datadir.join("results.txt"))) + expected_header = V3BWHeader(timestamp_l, + earliest_bandwidth=earliest_bandwidth, + latest_bandwidth=latest_bandwidth) + expected_bwls = [V3BWLine.from_results(results[fp]) for fp in results] + expected_f = V3BWFile(expected_header, expected_bwls) + v3bwfile = V3BWFile.from_arg_results(unittest_args, unittest_conf, results) + assert str(expected_f)[1:] == str(v3bwfile)[1:] + output = os.path.join(unittest_args.output, now_fname()) + print(output) + v3bwfile.write(output) + + +def test_from_arg_results_write(datadir, tmpdir, unittest_conf, unittest_args): + results = load_result_file(str(datadir.join("results.txt"))) + v3bwfile = V3BWFile.from_arg_results(unittest_args, unittest_conf, results) + output = os.path.join(unittest_args.output, now_fname()) + v3bwfile.write(output) + assert os.path.isfile(output)

1 0

[sbws/master] Add fixtures for unit tests
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 89ed8ace6ef2d6e3d0dd68ee5d671fcdf96005fd Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 16:09:41 2018 +0000 Add fixtures for unit tests --- tests/unit/conftest.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c9002eb..002987c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,7 +2,7 @@ from sbws.lib.resultdump import ResultError from sbws.lib.resultdump import ResultSuccess from sbws.lib.resultdump import Result from sbws.lib.resultdump import write_result_to_datadir -from sbws.util.config import get_config +from sbws.util.config import get_config, _get_default_config from sbws.util.parser import create_parser import sbws.core.init from tempfile import TemporaryDirectory @@ -50,6 +50,39 @@ def datadir(request): return D(request.fspath.dirpath("data")) +(a)pytest.fixture +def tmpdir(tmpdir_factory, request): + """Create a tmp dir for the tests""" + base = str(hash(request.node.nodeid))[:3] + bn = tmpdir_factory.mktemp(base) + return bn + + +(a)pytest.fixture +def tmpdir_sbwshome(tmpdir): + """Create .sbws inside of the tests tmp dir""" + home = tmpdir.join('.sbws') + os.makedirs(home.strpath, exist_ok=True) + return home + + +(a)pytest.fixture() +def unittest_args(tmpdir_sbwshome, parser): + """Args with sbws home in the tests tmp dir""" + return _PseudoArguments(directory=tmpdir_sbwshome.strpath, + output=tmpdir_sbwshome.strpath, + scale=False) + + +(a)pytest.fixture() +def unittest_conf(tmpdir_sbwshome): + """Default configuration with sbws home in the tmp test dir""" + conf = _get_default_config() + conf['paths']['sbwshome'] = tmpdir_sbwshome.strpath + conf['paths']['started_filepath'] = "" + return conf + + @pytest.fixture(scope='session') def parser(): return create_parser()

1 0

[sbws/master] Create generator_started line only when it is known
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit de0a46cbf1dba509d68a429f51ac7763d4658bbc Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 16:13:25 2018 +0000 Create generator_started line only when it is known --- sbws/lib/v3bwfile.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index fb9e3d3..1d7ec61 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -76,14 +76,14 @@ def read_started_ts(conf): try: filepath = conf['paths']['started_filepath'] except TypeError: - return '' + return None try: with FileLock(filepath): with open(filepath, 'r') as fd: generator_started = fd.read() except FileNotFoundError as e: log.warn('File %s not found.%s', filepath, e) - return '' + return None return generator_started @@ -229,7 +229,8 @@ class V3BWHeader(object): timestamp = str(latest_bandwidth) kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) - kwargs['generator_started'] = generator_started + if generator_started is not None: + kwargs['generator_started'] = generator_started h = cls(timestamp, **kwargs) return h

1 0

[sbws/master] Remove config.log.ini
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 10c7b9e0034a26c7f65948573724ea7723acf588 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 10:01:30 2018 -0400 Remove config.log.ini --- sbws/core/init.py | 3 --- sbws/sbws.py | 2 +- sbws/util/config.py | 13 ------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/sbws/core/init.py b/sbws/core/init.py index e7e839e..ea415a3 100644 --- a/sbws/core/init.py +++ b/sbws/core/init.py @@ -24,9 +24,6 @@ def main(args, conf): log.info('Creating %s', args.directory) os.makedirs(args.directory, exist_ok=False) - # Create config.log.ini #### - touch_file(os.path.join(args.directory, 'config.log.ini')) - # Create config.ini #### fname = os.path.join(args.directory, 'config.ini') if os.path.exists(fname) and not os.path.isfile(fname): diff --git a/sbws/sbws.py b/sbws/sbws.py index b95f738..584ce02 100644 --- a/sbws/sbws.py +++ b/sbws/sbws.py @@ -28,7 +28,7 @@ def main(): for e in conf_errors: log.critical(e) exit(1) - configure_logging(conf) + # configure_logging(conf) def_args = [args, conf] def_kwargs = {} known_commands = { diff --git a/sbws/util/config.py b/sbws/util/config.py index c24ac23..25df375 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -46,18 +46,6 @@ def _get_user_config(args, conf=None): return conf -def _get_user_logging_config(args, conf=None): - if not conf: - conf = ConfigParser(interpolation=ExtendedInterpolation()) - else: - assert isinstance(conf, ConfigParser) - fname = os.path.join(args.directory, 'config.log.ini') - if not os.path.isfile(fname): - return conf - conf = _read_config_file(conf, fname) - return conf - - def _get_default_logging_config(args, conf=None): if not conf: conf = ConfigParser(interpolation=ExtendedInterpolation()) @@ -73,7 +61,6 @@ def get_config(args): conf = _get_default_config() conf = _get_default_logging_config(args, conf=conf) conf = _get_user_config(args, conf=conf) - conf = _get_user_logging_config(args, conf=conf) return conf

1 0