December 2018 - tor-commits - lists.torproject.org

[sbws/master] scanner: log circuit path nicknames and errors
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit a3951327b914298939574a9550b24a68cfe8b552 Author: juga0 <juga(a)riseup.net> Date: Sun Dec 9 13:00:12 2018 +0000 scanner: log circuit path nicknames and errors --- sbws/core/scanner.py | 29 +++++++++++++++++------------ sbws/lib/destination.py | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 6328f71..f614cc4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -192,11 +192,14 @@ def measure_relay(args, conf, destinations, cb, rl, relay): relay, dest, rl, cb.controller, is_exit=False) if helper: circ_fps = [helper.fingerprint, relay.fingerprint] + # stored for debugging + nicknames = [helper.nickname, relay.nickname] else: helper = _pick_ideal_second_hop( relay, dest, rl, cb.controller, is_exit=True) if helper: circ_fps = [relay.fingerprint, helper.fingerprint] + nicknames = [relay.nickname, helper.nickname] if not helper: # TODO: Return ResultError of some sort log.debug('Unable to pick a 2nd hop to help measure %s %s', @@ -208,22 +211,20 @@ def measure_relay(args, conf, destinations, cb, rl, relay): our_nick = conf['scanner']['nickname'] circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: - log.debug('Could not build circuit involving %s', relay.nickname) - msg = 'Unable to complete circuit' + log.debug('Could not build circuit with path %s (%s): %s ', + circ_fps, nicknames, reason) return [ ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, msg=reason), ] - log.debug('Built circ %s %s for relay %s %s', circ_id, - stem_utils.circuit_str(cb.controller, circ_id), relay.nickname, - relay.fingerprint[0:8]) + log.debug('Built circuit with path %s (%s) to measure %s (%s)', + circ_fps, nicknames, relay.fingerprint, relay.nickname) # Make a connection to the destionation webserver and make sure it can # still help us measure is_usable, usable_data = dest.is_usable(circ_id, s, cb.controller) if not is_usable: - log.debug('When measuring %s %s the destination seemed to have ' - 'stopped being usable: %s', relay.nickname, - relay.fingerprint[0:8], usable_data) + log.debug('Destination %s unusable via circuit %s (%s), %s', + dest.url, circ_fps, nicknames, usable_data) cb.close_circuit(circ_id) # TODO: Return a different/new type of ResultError? msg = 'The destination seemed to have stopped being usable' @@ -236,8 +237,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): rtts, reason = measure_rtt_to_server(s, conf, dest, usable_data['content_length']) if rtts is None: - log.debug('Unable to measure RTT to %s via relay %s %s', - dest.url, relay.nickname, relay.fingerprint[0:8]) + log.debug('Unable to measure RTT for %s (%s) to %s via circuit ' + '%s (%s): %s', relay.fingerprint, relay.nickname, + dest.url, circ_fps, nicknames, reason) cb.close_circuit(circ_id) return [ ResultErrorStream(relay, circ_fps, dest.url, our_nick, @@ -247,8 +249,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): bw_results, reason = measure_bandwidth_to_server( s, conf, dest, usable_data['content_length']) if bw_results is None: - log.debug('Unable to measure bandwidth to %s via relay %s %s', - dest.url, relay.nickname, relay.fingerprint[0:8]) + log.debug('Unable to measure bandwidth for %s (%s) to %s via circuit ' + '%s (%s): %s', relay.fingerprint, relay.nickname, + dest.url, circ_fps, nicknames, reason) cb.close_circuit(circ_id) return [ ResultErrorStream(relay, circ_fps, dest.url, our_nick, @@ -256,6 +259,8 @@ def measure_relay(args, conf, destinations, cb, rl, relay): ] cb.close_circuit(circ_id) # Finally: store result + log.debug('Success measurement for %s (%s) via circuit %s (%s) to %s', + relay.fingerprint, relay.nickname, circ_fps, nicknames, dest.url) return [ ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick), ] diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index 0b27f39..bbfab3d 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -195,7 +195,7 @@ class DestinationList: # Pick a random exit exit = self._rng.choice(exits) circ_id, reason = \ - self._cb.build_circuit([None, exit.fingerprint]) + self._cb.build_circuit([None, exit.fingerprint]) if circ_id: break if not circ_id:

1 0

[sbws/master] scanner: log relay long fingerprint
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit b25057189991ba5550dd8f1e32a00a368ccd2a4f Author: juga0 <juga(a)riseup.net> Date: Sun Dec 9 13:02:24 2018 +0000 scanner: log relay long fingerprint Closes bug #28736. Bugfix v0.3.0. --- sbws/core/scanner.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index f614cc4..7246f12 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -180,8 +180,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): # Pick a destionation dest = destinations.next() if not dest: + # XXX: this should return a ResultError log.debug('Unable to get destination to measure %s %s', - relay.nickname, relay.fingerprint[0:8]) + relay.nickname, relay.fingerprint) return None # Pick a relay to help us measure the given relay. If the given relay is an # exit, then pick a non-exit. Otherwise pick an exit. @@ -202,8 +203,8 @@ def measure_relay(args, conf, destinations, cb, rl, relay): nicknames = [relay.nickname, helper.nickname] if not helper: # TODO: Return ResultError of some sort - log.debug('Unable to pick a 2nd hop to help measure %s %s', - relay.nickname, relay.fingerprint[0:8]) + log.debug('Unable to pick a 2nd relay to help measure %s (%s)', + relay.fingerprint, relay.nickname) return None assert helper assert circ_fps is not None and len(circ_fps) == 2 @@ -260,7 +261,7 @@ def measure_relay(args, conf, destinations, cb, rl, relay): cb.close_circuit(circ_id) # Finally: store result log.debug('Success measurement for %s (%s) via circuit %s (%s) to %s', - relay.fingerprint, relay.nickname, circ_fps, nicknames, dest.url) + relay.fingerprint, relay.nickname, circ_fps, nicknames, dest.url) return [ ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick), ]

1 0

[sbws/master] scanner: store the error in the result
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit 90a882d5bac954fd0cb5ba641d305af366fcc681 Author: juga0 <juga(a)riseup.net> Date: Sun Dec 9 11:38:22 2018 +0000 scanner: store the error in the result The error that is returned by an exception, not a custom error. So that it's easier to debug what fails. --- sbws/core/scanner.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index ecc6934..6328f71 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -211,7 +211,8 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.debug('Could not build circuit involving %s', relay.nickname) msg = 'Unable to complete circuit' return [ - ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, msg=msg), + ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, + msg=reason), ] log.debug('Built circ %s %s for relay %s %s', circ_id, stem_utils.circuit_str(cb.controller, circ_id), relay.nickname, @@ -238,10 +239,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.debug('Unable to measure RTT to %s via relay %s %s', dest.url, relay.nickname, relay.fingerprint[0:8]) cb.close_circuit(circ_id) - # TODO: Return a different/new type of ResultError? - msg = 'Something bad happened while measuring RTTs' return [ - ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), + ResultErrorStream(relay, circ_fps, dest.url, our_nick, + msg=str(reason)), ] # SECOND: measure bandwidth bw_results, reason = measure_bandwidth_to_server( @@ -250,10 +250,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.debug('Unable to measure bandwidth to %s via relay %s %s', dest.url, relay.nickname, relay.fingerprint[0:8]) cb.close_circuit(circ_id) - # TODO: Return a different/new type of ResultError? - msg = 'Something bad happened while measuring bandwidth' return [ - ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), + ResultErrorStream(relay, circ_fps, dest.url, our_nick, + msg=str(reason)), ] cb.close_circuit(circ_id) # Finally: store result

1 0

[sbws/master] circuitbuilder: log circuit path with long fingerprints
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit 2be0d501eab90c04d3e981984509b0eeb3f9bb73 Author: juga0 <juga(a)riseup.net> Date: Sun Dec 9 11:05:04 2018 +0000 circuitbuilder: log circuit path with long fingerprints --- sbws/lib/circuitbuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/circuitbuilder.py b/sbws/lib/circuitbuilder.py index 2546f0a..948e680 100644 --- a/sbws/lib/circuitbuilder.py +++ b/sbws/lib/circuitbuilder.py @@ -76,7 +76,7 @@ class CircuitBuilder: raise PathLengthException() c = self.controller timeout = self.circuit_timeout - fp_path = '[' + ' -> '.join([p[0:8] for p in path]) + ']' + fp_path = '[' + ' -> '.join([p for p in path]) + ']' log.debug('Building %s', fp_path) error = None for _ in range(0, 3):

1 0

[sbws/master] Merge branch 'bug28736'
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit b0c67fd711e87b7ea004cc6fafc65cdca734aad1 Merge: b4cfac8 b250571 Author: juga0 <juga(a)riseup.net> Date: Fri Dec 21 09:13:25 2018 +0000 Merge branch 'bug28736' sbws/config.default.ini | 2 +- sbws/config.log.default.ini | 3 ++ sbws/core/scanner.py | 95 ++++++++++++++++++++++++++------------------ sbws/lib/circuitbuilder.py | 14 +++++-- sbws/lib/destination.py | 5 ++- sbws/lib/relayprioritizer.py | 11 ++--- sbws/lib/resultdump.py | 15 ++++++- sbws/util/stem.py | 2 + 8 files changed, 94 insertions(+), 53 deletions(-)

1 0

[sbws/master] CI: check broken links in the docs
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit c6a7a94adf960a412b097ea05f5a5dae380bcd6e Author: juga0 <juga(a)riseup.net> Date: Thu Nov 29 19:00:51 2018 +0000 CI: check broken links in the docs Add a tox environment to check the links and call it from travis. It's not run by tox in order to do not need Internet running tox. The environment clean is also call from travis, in order to have sphinx installed when checking the links. Closes #28670. --- .travis.yml | 7 ++++++- tox.ini | 14 ++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3f19a7d..b6ac5ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,12 @@ python: - "3.5" - "3.6" install: pip install tox-travis -script: tox +script: + - tox + # This is not in included in the tox envlist, in order to don't need Internet + # when running tox + - tox -e doclinks + - tox -e clean before_install: - gpg --version diff --git a/tox.ini b/tox.ini index af89ea8..eb6a958 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,10 @@ [tox] skip_missing_interpreters = True -envlist = py35, py36, inst, setup, unit, integration, lint, stats, doc, clean +envlist = py35, py36, inst, setup, unit, integration, lint, stats, doc [travis] python = - 3.5: py35, inst, setup, unit, integration, lint, doc + 3.5: py35, inst, setup, unit, integration, lint, doc, doclinks 3.6: py36, inst, setup, unit, integration [testenv] @@ -82,6 +82,12 @@ commands = make html # this requires build the pdf images # make latexpdf - # this requires network - # make linkcheck make man + +# this requires Internet, it should not be in envlist +[testenv:doclinks] +deps = .[doc] +whitelist_externals = make +changedir = docs +commands = + make linkcheck

1 0

[sbws/master] Merge branch 'ticket28670'
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit b4cfac81a0def4e135483bd671777318774e6ce0 Merge: e32676c c6a7a94 Author: juga0 <juga(a)riseup.net> Date: Fri Dec 21 09:11:41 2018 +0000 Merge branch 'ticket28670' .travis.yml | 7 ++++++- tox.ini | 14 ++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-)

1 0

[sbws/master] docs: add scanner and destination requirements
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit b4da570af07e7332c600ac91e6d573c31716d818 Author: juga0 <juga(a)riseup.net> Date: Sat Dec 8 18:51:56 2018 +0000 docs: add scanner and destination requirements based on Torflow's documentation. Closes bug #28647. Bugfix v0.4.0 --- DEPLOY.rst | 10 +++++++++- INSTALL.rst | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/DEPLOY.rst b/DEPLOY.rst index e62e17f..1a1ea76 100644 --- a/DEPLOY.rst +++ b/DEPLOY.rst @@ -19,8 +19,16 @@ destination requirements - A Web server installed and running that supports HTTP GET, HEAD and Range (:rfc:`7233`) requests. ``Apache`` HTTP Server and ``Nginx`` support them. -- Optional support for TLS +- TLS support to avoid HTTP content caches at the various exit nodes. +- Certificates can be self-signed. - A large file; at the time of writing, at least 1 GiB in size + It can be created running:: + + head -c $((1024*1024*1024)) /dev/urandom > 1GiB + +- A fixed IP address or a domain name. +- Bandwidth: at least 12.5MB/s (100 Mbit/s). +- Network traffic: around 12-15GB/day. scanner setup ---------------------- diff --git a/INSTALL.rst b/INSTALL.rst index 77944ec..c518a46 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -58,7 +58,7 @@ to configure, deploy and run ``sbws``. System physical requirements ----------------------------- -- Bandwidth: at least 20MB/s (160 Mbit/s). The more the better. +- Bandwidth: at least 12.5MB/s (100 Mbit/s). - Free RAM: at least 1.5GB - Free disk: at least 3GB

1 0

[sbws/master] Merge branch 'bug28647'
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit e32676c868135448285fb67d39884683e54ba756 Merge: 6875f8a b4da570 Author: juga0 <juga(a)riseup.net> Date: Fri Dec 21 09:08:16 2018 +0000 Merge branch 'bug28647' DEPLOY.rst | 10 +++++++++- INSTALL.rst | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-)

1 0

[sbws/master] generate: use round_digs variable name in methods
by juga＠torproject.org 21 Dec '18

21 Dec '18

commit c57ba63ed0107e2ae07271302cfa7c9530774043 Author: juga0 <juga(a)riseup.net> Date: Fri Nov 23 17:40:23 2018 +0000 generate: use round_digs variable name in methods The torflow-round-digs cli argument was kept, but it was added other argument, round-digs, for the same option. argparse takes the first argument as the variable name, but the methods where still using torflow_round_digs. Closes bug #28602. Bugfix 1.0.3-dev0. --- sbws/core/generate.py | 2 +- sbws/lib/v3bwfile.py | 4 ++-- tests/unit/lib/test_v3bwfile.py | 21 ++++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index f9b46cb..971ed9e 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -106,7 +106,7 @@ def main(args, conf): bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant, scaling_method, torflow_cap=args.torflow_bw_margin, - torflow_round_digs=args.torflow_round_digs, + round_digs=args.round_digs, secs_recent=args.secs_recent, secs_away=args.secs_away, min_num=args.min_num, diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index a050744..d0e53f9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -488,7 +488,7 @@ class V3BWFile(object): scaling_method=TORFLOW_SCALING, torflow_obs=TORFLOW_OBS_LAST, torflow_cap=TORFLOW_BW_MARGIN, - torflow_round_digs=PROP276_ROUND_DIG, + round_digs=PROP276_ROUND_DIG, secs_recent=None, secs_away=None, min_num=0, consensus_path=None, max_bw_diff_perc=MAX_BW_DIFF_PERC, reverse=False): @@ -533,7 +533,7 @@ class V3BWFile(object): # log.debug(bw_lines[-1]) elif scaling_method == TORFLOW_SCALING: bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs, - torflow_cap, torflow_round_digs) + torflow_cap, round_digs) # log.debug(bw_lines[-1]) cls.update_progress( cls, bw_lines, header, number_consensus_relays, state) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index f9c50d5..86b979b 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -6,7 +6,7 @@ import os.path from sbws import __version__ as version from sbws.globals import (SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING, - MIN_REPORT) + MIN_REPORT, TORFLOW_ROUND_DIG, PROP276_ROUND_DIG) from sbws.lib.resultdump import Result, load_result_file, ResultSuccess from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V1, num_results_of_type, @@ -266,14 +266,21 @@ def test_sbws_scale(datadir): def test_torflow_scale(datadir): results = load_result_file(str(datadir.join("results.txt"))) - v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING) - assert v3bwfile.bw_lines[0].bw == 520 v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, - torflow_cap=0.0001) - assert v3bwfile.bw_lines[0].bw == 520 + round_digs=TORFLOW_ROUND_DIG) + assert v3bwfile.bw_lines[0].bw == 524 + v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, + torflow_cap=0.0001, + round_digs=TORFLOW_ROUND_DIG) + assert v3bwfile.bw_lines[0].bw == 524 v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, - torflow_cap=1, torflow_round_digs=1) - assert v3bwfile.bw_lines[0].bw == 500 + torflow_cap=1, + round_digs=TORFLOW_ROUND_DIG) + assert v3bwfile.bw_lines[0].bw == 524 + v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, + torflow_cap=1, + round_digs=PROP276_ROUND_DIG) + assert v3bwfile.bw_lines[0].bw == 520 def test_results_away_each_other(datadir):

1 0