June 2018 - tor-commits - lists.torproject.org

[sbws/master] Cap results at generate time instead of measurement time
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit c7eda42e58a3fb328def64355e093440d4bca1af Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 14:33:15 2018 -0400 Cap results at generate time instead of measurement time This required a bump in the result version: we add a new field to ResultSuccess. --- sbws/core/scanner.py | 36 ------------------------------------ sbws/globals.py | 2 +- sbws/lib/resultdump.py | 14 +++++++++++--- sbws/lib/v3bwfile.py | 20 +++++++++++++++++++- 4 files changed, 31 insertions(+), 41 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 2a3aadc..6af1b0c 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -165,41 +165,6 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): return chosen -def _clamp_bw_results(relay, bw_results): - ''' - If a relay has MaxAdvertisedBandwidth set, they may be capable of some - large amount of bandwidth but prefer if they didn't receive it. We also - could have managed to measure them faster than their {,Relay}BandwidthRate - somehow. - - See https://github.com/pastly/simple-bw-scanner/issues/155 and - https://trac.torproject.org/projects/tor/ticket/8494 - ''' - upper_bound = relay.average_bandwidth - if upper_bound is None: - log.warning( - 'Could not get average bandwidth from %s\'s descriptor. Not ' - 'capping the results for it to some upper bound.', relay.nickname) - return bw_results - capped_count = 0 - new_results = [] - for result in bw_results: - rate = result['amount'] / result['duration'] - if rate > upper_bound: - capped_count += 1 - new_results.append({ - 'amount': int(upper_bound * result['duration']), - 'duration': result['duration']}) - else: - new_results.append(result) - assert len(new_results) == len(bw_results) - if capped_count > 0: - log.debug( - 'Capped %d results to %d for relay %s', capped_count, upper_bound, - relay.nickname) - return new_results - - def measure_relay(args, conf, destinations, cb, rl, relay): s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) @@ -281,7 +246,6 @@ def measure_relay(args, conf, destinations, cb, rl, relay): ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), ] cb.close_circuit(circ_id) - bw_results = _clamp_bw_results(relay, bw_results) # Finally: store result return [ ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick), diff --git a/sbws/globals.py b/sbws/globals.py index e48b378..3ae7d60 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -4,7 +4,7 @@ import socket log = logging.getLogger(__name__) -RESULT_VERSION = 3 +RESULT_VERSION = 4 WIRE_VERSION = 1 SPEC_VERSION = '1.1.0' diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 32fd1cd..21d7cc0 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -150,15 +150,18 @@ class Result: class Relay: ''' Implements just enough of a stem RouterStatusEntryV3 for this Result class to be happy ''' - def __init__(self, fingerprint, nickname, address, master_key_ed25519): + def __init__(self, fingerprint, nickname, address, master_key_ed25519, + average_bandwidth=None): self.fingerprint = fingerprint self.nickname = nickname self.address = address self.master_key_ed25519 = master_key_ed25519 + self.average_bandwidth = average_bandwidth def __init__(self, relay, circ, dest_url, scanner_nick, t=None): self._relay = Result.Relay(relay.fingerprint, relay.nickname, - relay.address, relay.master_key_ed25519) + relay.address, relay.master_key_ed25519, + relay.average_bandwidth) self._circ = circ self._dest_url = dest_url self._scanner = scanner_nick @@ -169,6 +172,10 @@ class Result: raise NotImplementedError() @property + def relay_average_bandwidth(self): + return self._relay.average_bandwidth + + @property def fingerprint(self): return self._relay.fingerprint @@ -417,7 +424,7 @@ class ResultSuccess(Result): d['rtts'], d['downloads'], Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['master_key_ed25519']), + d['master_key_ed25519'], d['relay_average_bandwidth']), d['circ'], d['dest_url'], d['scanner'], t=d['time']) @@ -426,6 +433,7 @@ class ResultSuccess(Result): d.update({ 'rtts': self.rtts, 'downloads': self.downloads, + 'relay_average_bandwidth': self.relay_average_bandwidth, }) return d diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 5b07f80..64395ef 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -307,8 +307,26 @@ class V3BWLine(object): def bw_from_results(results): median_bw = median([dl['amount'] / dl['duration'] for r in results for dl in r.downloads]) + # If a relay has MaxAdvertisedBandwidth set, they may be capable of + # some large amount of bandwidth but prefer if they didn't receive it. + # We also could have managed to measure them faster than their + # {,Relay}BandwidthRate somehow. + # + # See https://github.com/pastly/simple-bw-scanner/issues/155 and + # https://trac.torproject.org/projects/tor/ticket/8494 + # + # Note how this isn't some measured-by-us average of bandwidth. It's + # the first value on the 'bandwidth' line in the relay's server + # descriptor. + relay_average_bw = median([ + r.relay_average_bandwidth for r in results + if r.relay_average_bandwidth is not None]) + if median_bw > relay_average_bw: + bw = relay_average_bw + else: + bw = median_bw # convert to KB and ensure it's at least 1 - bw_kb = max(round(median_bw / 1024), 1) + bw_kb = max(round(bw / 1024), 1) return bw_kb @staticmethod

1 0

[sbws/master] Remove an assert on _need_refresh()
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit ba873130843856100db78ba204314d1be82cb2fd Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 10:28:26 2018 -0400 Remove an assert on _need_refresh() The chance of us needing a refresh at this point is TINY, but it's probably better to return a list of relays stale by a millisecond than it is to cause the user's program to crash. --- sbws/lib/relaylist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 6afa18a..ddd982a 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -153,7 +153,6 @@ class RelayList: log.debug('No we don\'t need to refresh our relays. ' 'It was done by someone else.') log.debug('Giving back the lock for refreshing relays.') - assert not self._need_refresh() return self._relays @property

1 0

[sbws/master] Fix tests: CircuitBuilder now needs a RelayList
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit e3355d1c567473822c569af8bb5e34f12cee3068 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Sun Jun 24 13:53:54 2018 -0400 Fix tests: CircuitBuilder now needs a RelayList --- tests/integration/core/test_scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/core/test_scanner.py b/tests/integration/core/test_scanner.py index c1a8692..152e939 100644 --- a/tests/integration/core/test_scanner.py +++ b/tests/integration/core/test_scanner.py @@ -28,7 +28,7 @@ def get_everything_to_measure(dotsbws, cont, parser): conf['destinations.foo'] = {} conf['destinations.foo']['url'] = 'http://127.0.0.1:28888/sbws.bin' rl = RelayList(args, conf, cont) - cb = CB(args, conf, cont) + cb = CB(args, conf, cont, rl) dests, error_msg = DestinationList.from_config(conf, cb, rl, cont) assert dests, error_msg return {

1 0

[sbws/master] Make sure we are appropriately blocking when using our list of relays
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit b08694693777e4ba850589c5cf7c898e94ffe7b5 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 09:55:21 2018 -0400 Make sure we are appropriately blocking when using our list of relays I'm not sure if this is necessary, honestly. But it doesn't hurt. --- sbws/lib/relaylist.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 8b19a7c..6245aa1 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -168,16 +168,13 @@ class RelayList: return self._relays_with_flag(Flag.AUTHORITY) def random_relay(self): - relays = self.relays - return self.rng.choice(relays) + return self.rng.choice(self.relays) def _relays_with_flag(self, flag): - relays = self.relays - return [r for r in relays if flag in r.flags] + return [r for r in self.relays if flag in r.flags] def _relays_without_flag(self, flag): - relays = self.relays - return [r for r in relays if flag not in r.flags] + return [r for r in self.relays if flag not in r.flags] def _init_relays(self): c = self._controller

1 0

[sbws/master] Add changelog entry for relay list refreshing
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 17a5c9013b6e20b6fb17472b08e23f5d1b620901 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 10:05:08 2018 -0400 Add changelog entry for relay list refreshing --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18bffbc..524ffab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Make relay priority calculations take only ~5% of the time they used to (3s vs 60s) by using sets instead of lists when selecting non-Authority relays. (GH#204) +- Make relay list refreshing take much less time by not allowing worker threads + to dogpile on the CPU. Before they would all start requesting descriptors +from Tor at roughly the same time, causing us to overload our CPU core and make +the process take unnecessarily long. Now we let one thread do the work so it +can peg the CPU on its own and get the refresh done ASAP. +(GH#205) ### Changed

1 0

[sbws/master] Release 0.5.0
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit af8bb20e96b6a1bf7fab2cf3610efde335b44ce7 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 26 11:11:35 2018 -0400 Release 0.5.0 **Important changes**: - Result format changed, causing a version bump to 4. Updating sbws to 0.5.0 will cause it to ignore results with version less than 4. Summary of changes: - Keep previously-generated v3bw files - Allow a relay to limit its weight based on RelayBandwidthRate/MaxAdvertisedBandwidth - 1 CPU usage optimization - 1 memory usage optimization --- CHANGELOG.md | 20 +++++++++++++++++++- sbws/__init__.py | 2 +- tests/unit/lib/data/v3bw/20180425_131057.v3bw | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1e1c3..a8bbd36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] + + +## [0.5.0] - 2018-06-26 + +**Important changes**: + +- Result format changed, causing a version bump to 4. Updating sbws to 0.5.0 + will cause it to ignore results with version less than 4. + +Summary of changes: + +- Keep previously-generated v3bw files +- Allow a relay to limit its weight based on + RelayBandwidthRate/MaxAdvertisedBandwidth +- 1 CPU usage optimization +- 1 memory usage optimization + ### Added - Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound @@ -67,5 +84,6 @@ So fix that. - `earliest_bandwidth` being the newest bw not the oldest (thanks juga0) - `node_id` was missing the character "$" at the beginning -[Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.1...master +[Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.5.0...master +[0.5.0]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.0...v0.4.1 diff --git a/sbws/__init__.py b/sbws/__init__.py index 62f6024..2b8877c 100644 --- a/sbws/__init__.py +++ b/sbws/__init__.py @@ -1 +1 @@ -__version__ = '0.4.2-dev' +__version__ = '0.5.0' diff --git a/tests/unit/lib/data/v3bw/20180425_131057.v3bw b/tests/unit/lib/data/v3bw/20180425_131057.v3bw index 9bb3ffd..df3dcd4 100644 --- a/tests/unit/lib/data/v3bw/20180425_131057.v3bw +++ b/tests/unit/lib/data/v3bw/20180425_131057.v3bw @@ -5,6 +5,6 @@ file_created=2018-04-25T13:10:57 generator_started=2018-04-16T14:09:05 latest_bandwidth=2018-04-17T14:09:07 software=sbws -software_version=0.4.2-dev +software_version=0.5.0 ==== bw=54 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s nick=A node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 time=2018-04-17T14:09:07

1 0

[sbws/master] Catch json decode error when loading results
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 991fe852683257f23d243c5f79809886adb1f7ea Author: juga0 <juga(a)riseup.net> Date: Mon Jun 25 10:10:09 2018 +0000 Catch json decode error when loading results log the error, ingore the line and continue --- sbws/lib/resultdump.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 9f4f57a..755eb2d 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -44,7 +44,11 @@ def load_result_file(fname, success_only=False): with open(fname, 'rt') as fd: for line in fd: num_total += 1 - r = Result.from_dict(json.loads(line.strip())) + try: + r = Result.from_dict(json.loads(line.strip())) + except json.decoder.JSONDecodeError: + log.warning('Could not decode result %s', line.strip()) + r = None if r is None: num_ignored += 1 continue

1 0

[sbws/master] Bump version to 0.5.1-dev
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit f45706093ae3c7a2f2e923b4e7bf370df4cbb13f Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 26 11:35:51 2018 -0400 Bump version to 0.5.1-dev --- sbws/__init__.py | 2 +- tests/unit/lib/data/v3bw/20180425_131057.v3bw | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/__init__.py b/sbws/__init__.py index 2b8877c..1cb6639 100644 --- a/sbws/__init__.py +++ b/sbws/__init__.py @@ -1 +1 @@ -__version__ = '0.5.0' +__version__ = '0.5.1-dev' diff --git a/tests/unit/lib/data/v3bw/20180425_131057.v3bw b/tests/unit/lib/data/v3bw/20180425_131057.v3bw index df3dcd4..911d483 100644 --- a/tests/unit/lib/data/v3bw/20180425_131057.v3bw +++ b/tests/unit/lib/data/v3bw/20180425_131057.v3bw @@ -5,6 +5,6 @@ file_created=2018-04-25T13:10:57 generator_started=2018-04-16T14:09:05 latest_bandwidth=2018-04-17T14:09:07 software=sbws -software_version=0.5.0 +software_version=0.5.1-dev ==== bw=54 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s nick=A node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 time=2018-04-17T14:09:07

1 0

[sbws/master] Add changelog entry for JSON decode exception
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 58df5663cdc75d97b187f678546eec0be05adb36 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 12:54:57 2018 -0400 Add changelog entry for JSON decode exception --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 524ffab..4b1e1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ from Tor at roughly the same time, causing us to overload our CPU core and make the process take unnecessarily long. Now we let one thread do the work so it can peg the CPU on its own and get the refresh done ASAP. (GH#205) +- Catch a JSON decode exception on malformed results so sbws can continue + gracefully (GH#210 GHPR#212) ### Changed

1 0

[sbws/master] Add log.debug around relay refreshing code
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 979c41e1fbb10e7390ab119f09689c095bd2ca12 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 10:02:21 2018 -0400 Add log.debug around relay refreshing code --- sbws/lib/relaylist.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 6245aa1..6afa18a 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -136,14 +136,23 @@ class RelayList: # See if we can get the list of relays without having to do a refresh, # which is expensive and blocks other threads if self._need_refresh(): + log.debug('We need to refresh our list of relays. ' + 'Going to wait for lock.') # Whelp we couldn't just get the list of relays because the list is # stale. Wait for the lock so we can refresh it. with self._refresh_lock: + log.debug('We got the lock. Now to see if we still ' + 'need to refresh.') # Now we have the lock ... but wait! Maybe someone else already # did the refreshing. So check if it still needs refreshing. If # not, we can do nothing. if self._need_refresh(): + log.debug('Yup we need to refresh our relays. Doing so.') self._refresh() + else: + log.debug('No we don\'t need to refresh our relays. ' + 'It was done by someone else.') + log.debug('Giving back the lock for refreshing relays.') assert not self._need_refresh() return self._relays

1 0