June 2018 - tor-commits - lists.torproject.org

[sbws/master] Remove v3bw file link if it exist
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 75c37b818138d74ce5174dad4112619b797375ed Author: juga0 <juga(a)riseup.net> Date: Thu Jun 14 16:30:54 2018 +0000 Remove v3bw file link if it exist also fix link path --- sbws/lib/v3bwfile.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 1294d99..40a8fd5 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -402,14 +402,21 @@ class V3BwFile(object): def write(self, output): log.info('Writing v3bw file to %s', output) + out_dir = os.path.dirname(output) + out_link = os.path.join(out_dir, 'latest.v3bw') + if os.path.exists(out_link): + log.debug('Deleting existing symlink before creating a new one.') + os.remove(out_link) # to keep test_generate.py working if output != '/dev/stdout': - with DirectoryLock(os.path.dirname(output)): + with DirectoryLock(out_dir): with open(output, 'wt') as fd: fd.write(str(self.header)) for line in self.bw_lines: fd.write(str(line)) - os.symlink(output, 'latest.v3bw') + log.debug('Creating symlink from {} to {}.' + .format(output, out_link)) + os.symlink(output, out_link) else: with open(output, 'wt') as fd: fd.write(str(self.header))

1 0

[sbws/master] Implement the "MaxAdvertisedBandwidth" hack ...
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 068d521002e0858d91c2fc65d4c16b1c226d8a1b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 19:49:19 2018 -0400 Implement the "MaxAdvertisedBandwidth" hack ... Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound on the measurements we make for it. --- CHANGELOG.md | 5 +++++ sbws/core/scanner.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a79c02..6ce42bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound + on the measurements we make for it. (GH#155) + ### Changed - Change the path where the Bandwidth List files are generated: now they are diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 6af1b0c..2a3aadc 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -165,6 +165,41 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): return chosen +def _clamp_bw_results(relay, bw_results): + ''' + If a relay has MaxAdvertisedBandwidth set, they may be capable of some + large amount of bandwidth but prefer if they didn't receive it. We also + could have managed to measure them faster than their {,Relay}BandwidthRate + somehow. + + See https://github.com/pastly/simple-bw-scanner/issues/155 and + https://trac.torproject.org/projects/tor/ticket/8494 + ''' + upper_bound = relay.average_bandwidth + if upper_bound is None: + log.warning( + 'Could not get average bandwidth from %s\'s descriptor. Not ' + 'capping the results for it to some upper bound.', relay.nickname) + return bw_results + capped_count = 0 + new_results = [] + for result in bw_results: + rate = result['amount'] / result['duration'] + if rate > upper_bound: + capped_count += 1 + new_results.append({ + 'amount': int(upper_bound * result['duration']), + 'duration': result['duration']}) + else: + new_results.append(result) + assert len(new_results) == len(bw_results) + if capped_count > 0: + log.debug( + 'Capped %d results to %d for relay %s', capped_count, upper_bound, + relay.nickname) + return new_results + + def measure_relay(args, conf, destinations, cb, rl, relay): s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) @@ -246,6 +281,7 @@ def measure_relay(args, conf, destinations, cb, rl, relay): ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), ] cb.close_circuit(circ_id) + bw_results = _clamp_bw_results(relay, bw_results) # Finally: store result return [ ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),

1 0

[sbws/master] Add simple HTTP server script
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit f55b84aca876a9c72ce4d364695457f8a0fd124d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:16:43 2018 -0400 Add simple HTTP server script --- scripts/tools/sbws-http-server.py | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/scripts/tools/sbws-http-server.py b/scripts/tools/sbws-http-server.py new file mode 100755 index 0000000..d8575d0 --- /dev/null +++ b/scripts/tools/sbws-http-server.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# File: sbws-http-server.py +# Author: Matt Traudt +# License: CC0 +# +# This script implements just enough of the HTTP protocol to work with Simple +# Bandwidth Scanner. +# +# All requested URLs exist. All return 1 GiB of garbage data. We always speak +# HTTP/1.1 because that's necessary for Keep-Alive request headers +# (used by sbws scanners) to work. +# +# HEAD and GET requests are supported to the minimum extent necessary. +# This essentially means that if the client sends Range request headers just +# like sbws does, then we'll only send back the number of bytes they requested. +# Indeed, this was the motivating reason for the complexity of this script; +# normally I would have used SimpleHTTPRequestHandler unmodified. +# +# Don't breathe too hard or this script might break. +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +import http.server +from http import HTTPStatus +# import time + +FILE_SIZE = 1*1024*1024*1024 # 1 GiB + + +def _get_resp_size_from_range(range_str): + assert range_str.startswith('bytes=') + range_str = range_str[len('bytes='):] + start_byte, end_byte = range_str.split('-') + return int(end_byte) - int(start_byte) + 1 + + +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + + def send_head(self, length): + self.send_response(HTTPStatus.OK) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', length) + # self.send_header('Last-Modified', self.date_time_string(time.time())) + self.end_headers() + + def do_GET(self): + range_hdr = self.headers['Range'] + if not range_hdr: + num_bytes = FILE_SIZE + else: + assert range_hdr.startswith('bytes=') + num_bytes = _get_resp_size_from_range(range_hdr) + self.send_head(num_bytes) + self.wfile.write(b'A' * num_bytes) + + def do_HEAD(self): + self.send_head(FILE_SIZE) + + +def main(args): + addr = ('', args.port) + print('Listening on', addr) + httpd = http.server.HTTPServer(addr, MyHTTPRequestHandler) + httpd.serve_forever() + + +if __name__ == '__main__': + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) + parser.add_argument( + '-p', '--port', default=8000, type=int, help='Port on which to listen') + args = parser.parse_args() + try: + exit(main(args)) + except KeyboardInterrupt: + pass

1 0

[sbws/master] Remove unused member function
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 7f085e23137b0420e2ec485639411fd7b0bbbea5 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 18:35:11 2018 -0400 Remove unused member function --- sbws/lib/v3bwfile.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 3cff29d..e09392e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -379,15 +379,6 @@ class V3BwFile(object): return self.total_bw / self.num_lines @classmethod - def from_results(cls, conf, output, results): - bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] - bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) - header = V3BwHeader.from_results(conf, results) - f = cls(header, bw_lines) - f.write(output) - return f - - @classmethod def from_arg_results(cls, args, conf, results): bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True)

1 0

[sbws/master] Add changelog entry for v3bw refactoring
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 6440da403c36c7eba2e3d736ff3ef7874a2d76bc Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 18:40:47 2018 -0400 Add changelog entry for v3bw refactoring --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e838ac..3a79c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Change the path where the Bandwidth List files are generated: now they are stored in `v3bw` directory, named `YYmmdd_HHMMSS.v3bw`, and previously generated ones are kept. A `latest.v3bw` symlink is updated. (GH#179 GHPR#190) +- Code refactoring in the v3bw classes and generation area ## [0.4.1]

1 0

[sbws/master] Replace v3bw-into-xy bash script with python script
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 09691a0fe7b3809f2cdacd7713c2b37668c6c93b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 22:23:22 2018 -0400 Replace v3bw-into-xy bash script with python script GH: ref #182 --- CHANGELOG.md | 2 ++ scripts/tools/v3bw-into-xy.py | 53 +++++++++++++++++++++++++++++++++++++++++++ scripts/tools/v3bw-into-xy.sh | 26 --------------------- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a79c02..6e40965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. stored in `v3bw` directory, named `YYmmdd_HHMMSS.v3bw`, and previously generated ones are kept. A `latest.v3bw` symlink is updated. (GH#179 GHPR#190) - Code refactoring in the v3bw classes and generation area +- Replace v3bw-into-xy bash script with python script to handle a more complex + v3bw file format (GH#182) ## [0.4.1] diff --git a/scripts/tools/v3bw-into-xy.py b/scripts/tools/v3bw-into-xy.py new file mode 100755 index 0000000..fda5d74 --- /dev/null +++ b/scripts/tools/v3bw-into-xy.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import sys +import re +# File: v3bw-into-xy.py +# Author: Matt Traudt +# License: CC0 +# +# Takes one or more v3bw files as arguments. +# +# Looks for lines that contain actual data. That means most of them, since most +# of them contain "node_id=" and those are the ones that are interesting. +# +# Extract the fingerprint and bandwidth values for each of those lines and put +# them on stdout, one per line. Effectively, after ignoring other lines, this: +# node_id=$AAAA...AAAA bw=12345 +# becomes this: +# AAAA...AAAA 12345 +# +# NOTE: If you specify more than v3bw file, this will do NOTHING to tell you +# when the output from one file stops and the next begins +# +# With v1.1.0 of the v3bw file format, we no longer know if node_id or bw is +# first in the line. Hence two regular expresions and searching for the matched +# item that has 40 chars (the fingerprint) + + +def main(): + re1 = re.compile('.*node_id=\$?([\w]+).* bw=([\d]+).*') # noqa + re2 = re.compile('.*bw=([\d]+).* node_id=\$?([\w]+)') # noqa + for fname in sys.argv[1:]: + with open(fname, 'rt') as fd: + for line in fd: + if 'node_id' not in line: + continue + match = re1.match(line) or re2.match(line) + if not match: + continue + items = match.groups() + assert len(items) == 2 + s = '{} {}\n' + if len(items[0]) == 40: + s = s.format(*items) + else: + s = s.format(*items[::-1]) + sys.stdout.write(s) + return 0 + + +if __name__ == '__main__': + try: + exit(main()) + except (KeyboardInterrupt, BrokenPipeError): + pass diff --git a/scripts/tools/v3bw-into-xy.sh b/scripts/tools/v3bw-into-xy.sh deleted file mode 100755 index 059739d..0000000 --- a/scripts/tools/v3bw-into-xy.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# File: v3bw-into-xy.sh -# Author: Matt Traudt -# License: CC0 -# -# Takes one or more v3bw files as arguments. -# -# Looks for lines that contain actual data. That means most of them, since most -# of them start with "node_id=" and those are the ones that are interesting. -# -# Extract the fingerprint and bandwidth values for each of those lines and put -# them on stdout, one per line. Effectively, after ignoring other lines, this: -# node_id=$AAAA...AAAA bw=12345 -# becomes this: -# AAAA...AAAA 12345 -# -# NOTE: If you specify more than v3bw file, this will do NOTHING to tell you -# when the output from one file stops and the next begins -set -e -while [ "$1" != "" ] -do - grep '^node_id=' "$1" | - sed -r 's|^node_id=([$A-Z0-9]+) bw=([0-9]+).*$|\1 \2|' | - sed 's|\$||g' - shift -done

1 0

[sbws/master] Add relay1mbyteMAB relay to integration test network
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 57006a438190d8a76aed13a53c36eb86fedbe4cd Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 19:51:16 2018 -0400 Add relay1mbyteMAB relay to integration test network This relay has MaxAdvertisedBandwidth 1 MByte set in its torrc --- tests/integration/net.tar | Bin 191488 -> 204800 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/integration/net.tar b/tests/integration/net.tar index 2d8f1b9..cde1f17 100644 Binary files a/tests/integration/net.tar and b/tests/integration/net.tar differ

1 0

[sbws/master] Sart an http server during integration tests
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 313eeae2fb805bda6ee8ca46e36f0b0b9998d8d7 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:22:35 2018 -0400 Sart an http server during integration tests --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index a3af1ef..bfe4e97 100644 --- a/tox.ini +++ b/tox.ini @@ -23,13 +23,16 @@ whitelist_externals = tar bash sleep + cat commands = tar -C {envtmpdir} -vxf {toxinidir}/tests/integration/net.tar bash {envtmpdir}/net/start.sh bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" + bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 > {envtmpdir}/http-server.log &" sleep 30 coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest -s {toxinidir}/tests/integration -vv bash {envtmpdir}/net/stop.sh + cat {envtmpdir}/http-server.log [testenv:lint] skip_install = True

1 0

[sbws/master] Python 3.4 doesn't have HTTPStatus
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 815afcd6eacc1ba83669c1e7433e399f7a63f15b Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:30:40 2018 -0400 Python 3.4 doesn't have HTTPStatus --- scripts/tools/sbws-http-server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/tools/sbws-http-server.py b/scripts/tools/sbws-http-server.py index d8575d0..f21499f 100755 --- a/scripts/tools/sbws-http-server.py +++ b/scripts/tools/sbws-http-server.py @@ -19,7 +19,6 @@ # Don't breathe too hard or this script might break. from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter import http.server -from http import HTTPStatus # import time FILE_SIZE = 1*1024*1024*1024 # 1 GiB @@ -39,7 +38,7 @@ class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): super().__init__(*a, **kw) def send_head(self, length): - self.send_response(HTTPStatus.OK) + self.send_response(200) self.send_header('Content-Type', 'application/octet-stream') self.send_header('Content-Length', length) # self.send_header('Last-Modified', self.date_time_string(time.time()))

1 0

[sbws/master] Hopefully get http server logs if put in {toxinidir}
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 60ef71bcd51427fed6fc4f2d80aeb8c128dd0c38 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 17 21:32:57 2018 -0400 Hopefully get http server logs if put in {toxinidir} --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index bfe4e97..69371bd 100644 --- a/tox.ini +++ b/tox.ini @@ -28,11 +28,11 @@ commands = tar -C {envtmpdir} -vxf {toxinidir}/tests/integration/net.tar bash {envtmpdir}/net/start.sh bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" - bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 > {envtmpdir}/http-server.log &" + bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 > {toxinidir}/http-server.log &" sleep 30 coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest -s {toxinidir}/tests/integration -vv bash {envtmpdir}/net/stop.sh - cat {envtmpdir}/http-server.log + cat {toxinidir}/http-server.log [testenv:lint] skip_install = True

1 0