June 2018 - tor-commits - lists.torproject.org

[sbws/master] Standardize capitalization of v3bw class names
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit ee80724056521a4aa3b0741ab53f68b0448189bb Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 18:37:25 2018 -0400 Standardize capitalization of v3bw class names --- sbws/core/generate.py | 4 ++-- sbws/lib/v3bwfile.py | 10 +++++----- tests/unit/lib/test_v3bwfile.py | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 3f0df18..25653b1 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,5 +1,5 @@ from sbws.globals import (fail_hard, is_initted, SCALE_CONSTANT) -from sbws.lib.v3bwfile import V3BwFile +from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter import os @@ -77,7 +77,7 @@ def main(args, conf): log.warning('No recent results, so not generating anything. (Have you ' 'ran sbws scanner recently?)') return - bw_file = V3BwFile.from_arg_results(args, conf, results) + bw_file = V3BWFile.from_arg_results(args, conf, results) output = args.output or conf['paths']['v3bw_fname'].format(now_fname()) _write_v3bw_file(bw_file, output) log.info('Mean bandwidth per line: %f "KiB"', bw_file.avg_bw) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index e09392e..5b07f80 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -95,7 +95,7 @@ def result_type_to_key(type_str): return type_str.replace('-', '_') -class V3BwHeader(object): +class V3BWHeader(object): """ Create a bandwidth measurements (V3bw) header following bandwidth measurements document spec version 1.1.0. @@ -178,7 +178,7 @@ class V3BwHeader(object): def from_lines_v110(cls, lines): """ :param list lines: list of lines to parse - :returns: tuple of V3BwHeader object and non-header lines + :returns: tuple of V3BWHeader object and non-header lines """ assert isinstance(lines, list) try: @@ -198,7 +198,7 @@ class V3BwHeader(object): def from_text_v110(self, text): """ :param str text: text to parse - :returns: tuple of V3BwHeader object and non-header lines + :returns: tuple of V3BWHeader object and non-header lines """ assert isinstance(text, str) return self.from_lines_v110(text.split(LINE_SEP)) @@ -351,7 +351,7 @@ class V3BWLine(object): return cls.from_results(data[fingerprint]) -class V3BwFile(object): +class V3BWFile(object): """ Create a Bandwidth List file following spec version 1.1.0 @@ -384,6 +384,6 @@ class V3BwFile(object): bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) if args.scale: bw_lines = scale_lines(bw_lines, args.scale_constant) - header = V3BwHeader.from_results(conf, results) + header = V3BWHeader.from_results(conf, results) f = cls(header, bw_lines) return f diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index e18278d..2f4a631 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -5,9 +5,9 @@ import json from sbws import __version__ as version from sbws.globals import SPEC_VERSION, RESULT_VERSION from sbws.lib.resultdump import Result, load_result_file -from sbws.lib.v3bwfile import (V3BwHeader, V3BWLine, TERMINATOR, LINE_SEP, +from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V110, num_results_of_type, - V3BwFile) + V3BWFile) timestamp = 1523974147 timestamp_l = str(timestamp) @@ -86,13 +86,13 @@ RESULT_ERROR_STREAM_STR = str(RESULT_ERROR_STREAM_DICT) def test_v3bwheader_str(): """Test header str""" - header = V3BwHeader(timestamp_l, file_created=file_created) + header = V3BWHeader(timestamp_l, file_created=file_created) assert header_str == str(header) def test_v3bwheader_extra_str(): """Test header str with additional headers""" - header = V3BwHeader(timestamp_l, + header = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, earliest_bandwidth=earliest_bandwidth) @@ -101,33 +101,33 @@ def test_v3bwheader_extra_str(): def test_v3bwheader_from_lines(): """""" - header_obj = V3BwHeader(timestamp_l, + header_obj = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, earliest_bandwidth=earliest_bandwidth) - header, _ = V3BwHeader.from_lines_v110(header_extra_ls) + header, _ = V3BWHeader.from_lines_v110(header_extra_ls) assert str(header_obj) == str(header) def test_v3bwheader_from_text(): """""" - header_obj = V3BwHeader(timestamp_l, + header_obj = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, earliest_bandwidth=earliest_bandwidth) - header, _ = V3BwHeader.from_text_v110(header_extra_str) + header, _ = V3BWHeader.from_text_v110(header_extra_str) assert str(header_obj) == str(header) def test_v3bwheader_from_file(datadir): """Test header str with additional headers""" - header = V3BwHeader(timestamp_l, + header = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, earliest_bandwidth=earliest_bandwidth) # at some point this should be read from conftest text = datadir.read('v3bw/20180425_131057.v3bw') - h, _ = V3BwHeader.from_text_v110(text) + h, _ = V3BWHeader.from_text_v110(text) assert str(h) == str(header) @@ -160,10 +160,10 @@ def test_v3bwfile(datadir, tmpdir): # at some point this should be obtained from conftest v3bw = datadir.read('v3bw/20180425_131057.v3bw') results = load_result_file(str(datadir.join("results.txt"))) - header = V3BwHeader(timestamp_l, + header = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, earliest_bandwidth=earliest_bandwidth) bwls = [V3BWLine.from_results(results[fp]) for fp in results] - f = V3BwFile(header, bwls) + f = V3BWFile(header, bwls) assert v3bw == str(f)

1 0

[sbws/master] Remove v3bw file link if it exist
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 75c37b818138d74ce5174dad4112619b797375ed Author: juga0 <juga(a)riseup.net> Date: Thu Jun 14 16:30:54 2018 +0000 Remove v3bw file link if it exist also fix link path --- sbws/lib/v3bwfile.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 1294d99..40a8fd5 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -402,14 +402,21 @@ class V3BwFile(object): def write(self, output): log.info('Writing v3bw file to %s', output) + out_dir = os.path.dirname(output) + out_link = os.path.join(out_dir, 'latest.v3bw') + if os.path.exists(out_link): + log.debug('Deleting existing symlink before creating a new one.') + os.remove(out_link) # to keep test_generate.py working if output != '/dev/stdout': - with DirectoryLock(os.path.dirname(output)): + with DirectoryLock(out_dir): with open(output, 'wt') as fd: fd.write(str(self.header)) for line in self.bw_lines: fd.write(str(line)) - os.symlink(output, 'latest.v3bw') + log.debug('Creating symlink from {} to {}.' + .format(output, out_link)) + os.symlink(output, out_link) else: with open(output, 'wt') as fd: fd.write(str(self.header))

1 0

[sbws/master] Implement the "MaxAdvertisedBandwidth" hack ...
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 068d521002e0858d91c2fc65d4c16b1c226d8a1b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 19:49:19 2018 -0400 Implement the "MaxAdvertisedBandwidth" hack ... Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound on the measurements we make for it. --- CHANGELOG.md | 5 +++++ sbws/core/scanner.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a79c02..6ce42bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound + on the measurements we make for it. (GH#155) + ### Changed - Change the path where the Bandwidth List files are generated: now they are diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 6af1b0c..2a3aadc 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -165,6 +165,41 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): return chosen +def _clamp_bw_results(relay, bw_results): + ''' + If a relay has MaxAdvertisedBandwidth set, they may be capable of some + large amount of bandwidth but prefer if they didn't receive it. We also + could have managed to measure them faster than their {,Relay}BandwidthRate + somehow. + + See https://github.com/pastly/simple-bw-scanner/issues/155 and + https://trac.torproject.org/projects/tor/ticket/8494 + ''' + upper_bound = relay.average_bandwidth + if upper_bound is None: + log.warning( + 'Could not get average bandwidth from %s\'s descriptor. Not ' + 'capping the results for it to some upper bound.', relay.nickname) + return bw_results + capped_count = 0 + new_results = [] + for result in bw_results: + rate = result['amount'] / result['duration'] + if rate > upper_bound: + capped_count += 1 + new_results.append({ + 'amount': int(upper_bound * result['duration']), + 'duration': result['duration']}) + else: + new_results.append(result) + assert len(new_results) == len(bw_results) + if capped_count > 0: + log.debug( + 'Capped %d results to %d for relay %s', capped_count, upper_bound, + relay.nickname) + return new_results + + def measure_relay(args, conf, destinations, cb, rl, relay): s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) @@ -246,6 +281,7 @@ def measure_relay(args, conf, destinations, cb, rl, relay): ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), ] cb.close_circuit(circ_id) + bw_results = _clamp_bw_results(relay, bw_results) # Finally: store result return [ ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),

1 0

[sbws/master] Add simple HTTP server script
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit f55b84aca876a9c72ce4d364695457f8a0fd124d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:16:43 2018 -0400 Add simple HTTP server script --- scripts/tools/sbws-http-server.py | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/scripts/tools/sbws-http-server.py b/scripts/tools/sbws-http-server.py new file mode 100755 index 0000000..d8575d0 --- /dev/null +++ b/scripts/tools/sbws-http-server.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# File: sbws-http-server.py +# Author: Matt Traudt +# License: CC0 +# +# This script implements just enough of the HTTP protocol to work with Simple +# Bandwidth Scanner. +# +# All requested URLs exist. All return 1 GiB of garbage data. We always speak +# HTTP/1.1 because that's necessary for Keep-Alive request headers +# (used by sbws scanners) to work. +# +# HEAD and GET requests are supported to the minimum extent necessary. +# This essentially means that if the client sends Range request headers just +# like sbws does, then we'll only send back the number of bytes they requested. +# Indeed, this was the motivating reason for the complexity of this script; +# normally I would have used SimpleHTTPRequestHandler unmodified. +# +# Don't breathe too hard or this script might break. +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +import http.server +from http import HTTPStatus +# import time + +FILE_SIZE = 1*1024*1024*1024 # 1 GiB + + +def _get_resp_size_from_range(range_str): + assert range_str.startswith('bytes=') + range_str = range_str[len('bytes='):] + start_byte, end_byte = range_str.split('-') + return int(end_byte) - int(start_byte) + 1 + + +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + + def send_head(self, length): + self.send_response(HTTPStatus.OK) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', length) + # self.send_header('Last-Modified', self.date_time_string(time.time())) + self.end_headers() + + def do_GET(self): + range_hdr = self.headers['Range'] + if not range_hdr: + num_bytes = FILE_SIZE + else: + assert range_hdr.startswith('bytes=') + num_bytes = _get_resp_size_from_range(range_hdr) + self.send_head(num_bytes) + self.wfile.write(b'A' * num_bytes) + + def do_HEAD(self): + self.send_head(FILE_SIZE) + + +def main(args): + addr = ('', args.port) + print('Listening on', addr) + httpd = http.server.HTTPServer(addr, MyHTTPRequestHandler) + httpd.serve_forever() + + +if __name__ == '__main__': + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument( + '-p', '--port', default=8000, type=int, help='Port on which to listen') + args = parser.parse_args() + try: + exit(main(args)) + except KeyboardInterrupt: + pass

1 0

[sbws/master] Remove unused member function
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 7f085e23137b0420e2ec485639411fd7b0bbbea5 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 18:35:11 2018 -0400 Remove unused member function --- sbws/lib/v3bwfile.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 3cff29d..e09392e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -379,15 +379,6 @@ class V3BwFile(object): return self.total_bw / self.num_lines @classmethod - def from_results(cls, conf, output, results): - bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] - bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) - header = V3BwHeader.from_results(conf, results) - f = cls(header, bw_lines) - f.write(output) - return f - - @classmethod def from_arg_results(cls, args, conf, results): bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True)

1 0

[sbws/master] Add changelog entry for v3bw refactoring
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 6440da403c36c7eba2e3d736ff3ef7874a2d76bc Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 18:40:47 2018 -0400 Add changelog entry for v3bw refactoring --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e838ac..3a79c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Change the path where the Bandwidth List files are generated: now they are stored in `v3bw` directory, named `YYmmdd_HHMMSS.v3bw`, and previously generated ones are kept. A `latest.v3bw` symlink is updated. (GH#179 GHPR#190) +- Code refactoring in the v3bw classes and generation area ## [0.4.1]

1 0

[sbws/master] Replace v3bw-into-xy bash script with python script
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 09691a0fe7b3809f2cdacd7713c2b37668c6c93b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 22:23:22 2018 -0400 Replace v3bw-into-xy bash script with python script GH: ref #182 --- CHANGELOG.md | 2 ++ scripts/tools/v3bw-into-xy.py | 53 +++++++++++++++++++++++++++++++++++++++++++ scripts/tools/v3bw-into-xy.sh | 26 --------------------- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a79c02..6e40965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. stored in `v3bw` directory, named `YYmmdd_HHMMSS.v3bw`, and previously generated ones are kept. A `latest.v3bw` symlink is updated. (GH#179 GHPR#190) - Code refactoring in the v3bw classes and generation area +- Replace v3bw-into-xy bash script with python script to handle a more complex + v3bw file format (GH#182) ## [0.4.1] diff --git a/scripts/tools/v3bw-into-xy.py b/scripts/tools/v3bw-into-xy.py new file mode 100755 index 0000000..fda5d74 --- /dev/null +++ b/scripts/tools/v3bw-into-xy.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import sys +import re +# File: v3bw-into-xy.py +# Author: Matt Traudt +# License: CC0 +# +# Takes one or more v3bw files as arguments. +# +# Looks for lines that contain actual data. That means most of them, since most +# of them contain "node_id=" and those are the ones that are interesting. +# +# Extract the fingerprint and bandwidth values for each of those lines and put +# them on stdout, one per line. Effectively, after ignoring other lines, this: +# node_id=$AAAA...AAAA bw=12345 +# becomes this: +# AAAA...AAAA 12345 +# +# NOTE: If you specify more than v3bw file, this will do NOTHING to tell you +# when the output from one file stops and the next begins +# +# With v1.1.0 of the v3bw file format, we no longer know if node_id or bw is +# first in the line. Hence two regular expresions and searching for the matched +# item that has 40 chars (the fingerprint) + + +def main(): + re1 = re.compile('.*node_id=\$?([\w]+).* bw=([\d]+).*') # noqa + re2 = re.compile('.*bw=([\d]+).* node_id=\$?([\w]+)') # noqa + for fname in sys.argv[1:]: + with open(fname, 'rt') as fd: + for line in fd: + if 'node_id' not in line: + continue + match = re1.match(line) or re2.match(line) + if not match: + continue + items = match.groups() + assert len(items) == 2 + s = '{} {}\n' + if len(items[0]) == 40: + s = s.format(*items) + else: + s = s.format(*items[::-1]) + sys.stdout.write(s) + return 0 + + +if __name__ == '__main__': + try: + exit(main()) + except (KeyboardInterrupt, BrokenPipeError): + pass diff --git a/scripts/tools/v3bw-into-xy.sh b/scripts/tools/v3bw-into-xy.sh deleted file mode 100755 index 059739d..0000000 --- a/scripts/tools/v3bw-into-xy.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# File: v3bw-into-xy.sh -# Author: Matt Traudt -# License: CC0 -# -# Takes one or more v3bw files as arguments. -# -# Looks for lines that contain actual data. That means most of them, since most -# of them start with "node_id=" and those are the ones that are interesting. -# -# Extract the fingerprint and bandwidth values for each of those lines and put -# them on stdout, one per line. Effectively, after ignoring other lines, this: -# node_id=$AAAA...AAAA bw=12345 -# becomes this: -# AAAA...AAAA 12345 -# -# NOTE: If you specify more than v3bw file, this will do NOTHING to tell you -# when the output from one file stops and the next begins -set -e -while [ "$1" != "" ] -do - grep '^node_id=' "$1" | - sed -r 's|^node_id=([$A-Z0-9]+) bw=([0-9]+).*$|\1 \2|' | - sed 's|\$||g' - shift -done

1 0

[sbws/master] Add relay1mbyteMAB relay to integration test network
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 57006a438190d8a76aed13a53c36eb86fedbe4cd Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 19:51:16 2018 -0400 Add relay1mbyteMAB relay to integration test network This relay has MaxAdvertisedBandwidth 1 MByte set in its torrc --- tests/integration/net.tar | Bin 191488 -> 204800 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/integration/net.tar b/tests/integration/net.tar index 2d8f1b9..cde1f17 100644 Binary files a/tests/integration/net.tar and b/tests/integration/net.tar differ

1 0

[sbws/master] Sart an http server during integration tests
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 313eeae2fb805bda6ee8ca46e36f0b0b9998d8d7 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:22:35 2018 -0400 Sart an http server during integration tests --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index a3af1ef..bfe4e97 100644 --- a/tox.ini +++ b/tox.ini @@ -23,13 +23,16 @@ whitelist_externals = tar bash sleep + cat commands = tar -C {envtmpdir} -vxf {toxinidir}/tests/integration/net.tar bash {envtmpdir}/net/start.sh bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" + bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 > {envtmpdir}/http-server.log &" sleep 30 coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest -s {toxinidir}/tests/integration -vv bash {envtmpdir}/net/stop.sh + cat {envtmpdir}/http-server.log [testenv:lint] skip_install = True

1 0

[sbws/master] Python 3.4 doesn't have HTTPStatus
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 815afcd6eacc1ba83669c1e7433e399f7a63f15b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:30:40 2018 -0400 Python 3.4 doesn't have HTTPStatus --- scripts/tools/sbws-http-server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/tools/sbws-http-server.py b/scripts/tools/sbws-http-server.py index d8575d0..f21499f 100755 --- a/scripts/tools/sbws-http-server.py +++ b/scripts/tools/sbws-http-server.py @@ -19,7 +19,6 @@ # Don't breathe too hard or this script might break. from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter import http.server -from http import HTTPStatus # import time FILE_SIZE = 1*1024*1024*1024 # 1 GiB @@ -39,7 +38,7 @@ class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): super().__init__(*a, **kw) def send_head(self, length): - self.send_response(HTTPStatus.OK) + self.send_response(200) self.send_header('Content-Type', 'application/octet-stream') self.send_header('Content-Length', length) # self.send_header('Last-Modified', self.date_time_string(time.time()))

1 0