March 2019 - tor-commits - lists.torproject.org

[sbws/master] new: state: Add optional default arg to get
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 0ad7e04fcbef48b49acab0845ca83509f9b481b2 Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 15:24:17 2019 +0000 new: state: Add optional default arg to get Part of #28567. --- sbws/util/state.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index 58da699..1d33c3d 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -63,12 +63,16 @@ class State: self._state = self._read() return self._state.__len__() - def get(self, key): + def get(self, key, d=None): + """ + Implements a dictionary ``get`` method reading and locking + a json file. + """ if not isinstance(key, str): raise TypeError( 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() - return self._state.get(key) + return self._state.get(key, d) def __getitem__(self, key): if not isinstance(key, str): @@ -93,7 +97,6 @@ class State: raise TypeError( 'May only store value with type in %s, not %s' % (State._ALLOWED_TYPES, type(value))) - self._state = self._read() self._state.__setitem__(key, value) self._write()

1 0

[sbws/master] new: relaylist: Add measurement attempts
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 148e8e175d2dadadbdffa75f3d83d52d3068fa3f Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 10:05:40 2019 +0000 new: relaylist: Add measurement attempts and number of times in a priority list attrs and methods, to be able to store the data. Part of #28567. --- sbws/lib/relaylist.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index a4dc7fa..6a66069 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -79,6 +79,12 @@ class Relay: log.exception("Exception trying to get desc %s", e) self._consensus_timestamps = [] self._add_consensus_timestamp(timestamp) + # The number of times that a relay is "prioritized" to be measured. + # It is incremented in ``RelayPrioritizer.best_priority`` + self.relay_recent_priority_list_count = 0 + # The number of times that a relay has been queued to be measured. + # It is incremented in ``scanner.main_loop`` + self.relay_recent_measurement_attempt_count = 0 def _from_desc(self, attr): if not self._desc: @@ -231,6 +237,30 @@ class Relay: Flag.EXIT in self.flags and self.can_exit_to_port(port)) + def increment_relay_recent_measurement_attempt_count(self): + """ + Increment The number of times that a relay has been queued + to be measured. + + It is call from :funf:`~sbws.core.scaner.main_loop`. + """ + # If it was not in the previous measurements version, start counting + if self.relay_recent_measurement_attempt_count is None: + self.relay_recent_measurement_attempt_count = 0 + self.relay_recent_measurement_attempt_count += 1 + + def increment_relay_recent_priority_list_count(self): + """ + The number of times that a relay is "prioritized" to be measured. + + It is call from + :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority`. + """ + # If it was not in the previous measurements version, start counting + if self.relay_recent_priority_list_count is None: + self.relay_recent_priority_list_count = 0 + self.relay_recent_priority_list_count += 1 + class RelayList: ''' Keeps a list of all relays in the current Tor network and updates it @@ -252,6 +282,11 @@ class RelayList: # The period of time for which the measurements are keep. self._measurements_period = measurements_period self._state = state + # NOTE: blocking: writes to disk + if self._state: + if self._state.get('recent_measurement_attempt_count', None) \ + is None: + self._state['recent_measurement_attempt_count'] = 0 self._refresh() def _need_refresh(self): @@ -389,3 +424,16 @@ class RelayList: def exits_not_bad_allowing_port(self, port): return [r for r in self.exits if r.is_exit_not_bad_allowing_port(port)] + + def increment_recent_measurement_attempt_count(self): + """ + Increment the number of times that any relay has been queued to be + measured. + + It is call from :funf:`~sbws.core.scaner.main_loop`. + + It is read and stored in a ``state`` file. + """ + # NOTE: blocking, writes to file! + if self._state: + self._state['recent_measurement_attempt_count'] += 1

1 0

[sbws/master] new: resultdump: Store measurements attempts
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit eb4d72dc570bbc0cdc65201e684bf64bbaede92f Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 11:55:50 2019 +0000 new: resultdump: Store measurements attempts and the number of times the relay was in a priority list. Part of #28567. --- sbws/lib/resultdump.py | 137 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 113 insertions(+), 24 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 2149e8d..2ebfa20 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -201,17 +201,39 @@ class _ResultType(_StrEnum): class Result: - ''' A simple struct to pack a measurement result into so that other code - can be confident it is handling a well-formed result. ''' + """A bandwidth measurement for a relay. + + It re-implements :class:`~sbws.lib.relaylist.Relay` as a inner class. + """ class Relay: - ''' Implements just enough of a stem RouterStatusEntryV3 for this - Result class to be happy ''' + """A Tor relay. + + It re-implements :class:`~sbws.lib.relaylist.Relay` + with the attributes needed. + + .. note:: in a future refactor it would be simpler if a ``Relay`` has + measurements and a measurement has a relay, + instead of every measurement re-implementing ``Relay``. + """ def __init__(self, fingerprint, nickname, address, master_key_ed25519, average_bandwidth=None, burst_bandwidth=None, observed_bandwidth=None, consensus_bandwidth=None, consensus_bandwidth_is_unmeasured=None, - relay_in_recent_consensus_count=None): + # Counters to be stored by relay and not per measurement, + # since the measurements might fail. + relay_in_recent_consensus_count=None, + relay_recent_measurement_attempt_count=None, + relay_recent_priority_list_count=None): + """ + Initializes a ``Result.Relay``. + + .. note:: in a future refactor the attributes should be dinamic + to easy adding/removing them. + They are shared by :class:`~sbws.lib.relaylist.Relay` and + :class:`~sbws.lib.v3bwfile.V3BWLine` and there should not be + repeated in every class. + """ self.fingerprint = fingerprint self.nickname = nickname self.address = address @@ -222,20 +244,30 @@ class Result: self.consensus_bandwidth = consensus_bandwidth self.consensus_bandwidth_is_unmeasured = \ consensus_bandwidth_is_unmeasured - # The number of times the relay was in a consensus. self.relay_in_recent_consensus_count = \ relay_in_recent_consensus_count + self.relay_recent_measurement_attempt_count = \ + relay_recent_measurement_attempt_count + self.relay_recent_priority_list_count = \ + relay_recent_priority_list_count def __init__(self, relay, circ, dest_url, scanner_nick, t=None, relay_in_recent_consensus_count=None): - self._relay = Result.Relay(relay.fingerprint, relay.nickname, - relay.address, relay.master_key_ed25519, - relay.average_bandwidth, - relay.burst_bandwidth, - relay.observed_bandwidth, - relay.consensus_bandwidth, - relay.consensus_bandwidth_is_unmeasured, - relay.relay_in_recent_consensus_count) + """ + Initilizes the measurement and the relay with all the relay attributes. + """ + self._relay = Result.Relay( + relay.fingerprint, relay.nickname, + relay.address, relay.master_key_ed25519, + relay.average_bandwidth, + relay.burst_bandwidth, + relay.observed_bandwidth, + relay.consensus_bandwidth, + relay.consensus_bandwidth_is_unmeasured, + relay.relay_in_recent_consensus_count, + relay.relay_recent_measurement_attempt_count, + relay.relay_recent_priority_list_count + ) self._circ = circ self._dest_url = dest_url self._scanner = scanner_nick @@ -287,6 +319,24 @@ class Result: return self._relay.relay_in_recent_consensus_count @property + def relay_recent_measurement_attempt_count(self): + """Returns the relay recent measurements attemps. + + It is initialized in :class:`~sbws.lib.relaylist.Relay` and + incremented in :func:`~sbws.core.scanner.main_loop`. + """ + return self._relay.relay_recent_measurement_attempt_count + + @property + def relay_recent_priority_list_count(self): + """Returns the relay recent "prioritization"s to be measured. + + It is initialized in :class:`~sbws.lib.relaylist.Relay` and + incremented in :func:`~sbws.core.scanner.main_loop`. + """ + return self._relay.relay_recent_priority_list_count + + @property def circ(self): return self._circ @@ -320,15 +370,29 @@ class Result: 'version': self.version, 'relay_in_recent_consensus_count': self.relay_in_recent_consensus_count, + 'relay_recent_measurement_attempt_count': + self.relay_recent_measurement_attempt_count, + 'relay_recent_priority_list_count': + self.relay_recent_priority_list_count, } @staticmethod def from_dict(d): - ''' Given a dict, returns the Result* subtype that is represented by - the dict. If we don't know how to parse the dict into a Result and it's - likely because the programmer forgot to implement something, raises - NotImplementedError. If we can't parse the dict for some other reason, - return None. ''' + """ + Returns a :class:`~sbws.lib.resultdump.Result` subclass from a + dictionary. + + Returns None if the ``version`` attribute is not + :const:`~sbws.globals.RESULT_VERSION` + + It raises ``NotImplementedError`` when the dictionary ``type`` can not + be parsed. + + .. note:: in a future refactor, the conversions to/from + object-dictionary will be simpler using ``setattr`` and ``__dict__`` + + ``version`` is not being used and should be removed. + """ assert 'version' in d if d['version'] != RESULT_VERSION: return None @@ -393,7 +457,12 @@ class ResultError(Result): d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None)), # noqa + d.get('relay_in_recent_consensus_count', None), # noqa + relay_recent_measurement_attempt_count= # noqa + d.get('relay_recent_measurement_attempt_count', None), # noqa + relay_recent_priority_list_count= # noqa + d.get('relay_recent_priority_list_count', None), # noqa + ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -436,7 +505,12 @@ class ResultErrorCircuit(ResultError): d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None)), # noqa + d.get('relay_in_recent_consensus_count', None), # noqa + relay_recent_measurement_attempt_count= # noqa + d.get('relay_recent_measurement_attempt_count', None), # noqa + relay_recent_priority_list_count= # noqa + d.get('relay_recent_priority_list_count', None), # noqa + ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -461,7 +535,12 @@ class ResultErrorStream(ResultError): d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None)), # noqa + d.get('relay_in_recent_consensus_count', None), # noqa + relay_recent_measurement_attempt_count= # noqa + d.get('relay_recent_measurement_attempt_count', None), # noqa + relay_recent_priority_list_count= # noqa + d.get('relay_recent_priority_list_count', None), # noqa + ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -576,7 +655,12 @@ class ResultErrorAuth(ResultError): d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None)), # noqa + d.get('relay_in_recent_consensus_count', None), # noqa + relay_recent_measurement_attempt_count= # noqa + d.get('relay_recent_measurement_attempt_count', None), # noqa + relay_recent_priority_list_count= # noqa + d.get('relay_recent_priority_list_count', None), # noqa + ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -615,7 +699,12 @@ class ResultSuccess(Result): d.get('consensus_bandwidth'), d.get('consensus_bandwidth_is_unmeasured'), relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None)), # noqa + d.get('relay_in_recent_consensus_count', None), # noqa + relay_recent_measurement_attempt_count= # noqa + d.get('relay_recent_measurement_attempt_count', None), # noqa + relay_recent_priority_list_count= # noqa + d.get('relay_recent_priority_list_count', None), # noqa + ), d['circ'], d['dest_url'], d['scanner'], t=d['time'])

1 0

[sbws/master] chg: tests: Remove test that fails sometimes in Travis
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 24d61b651073991fecf196d06d511f5e6572ac66 Author: juga0 <juga(a)riseup.net> Date: Mon Mar 11 16:34:28 2019 +0000 chg: tests: Remove test that fails sometimes in Travis because the http server is not concurrent/asynchronous and there are as many processes as tox environments querying the http server. --- tests/integration/util/test_requests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integration/util/test_requests.py b/tests/integration/util/test_requests.py index ecb22ad..9c55640 100644 --- a/tests/integration/util/test_requests.py +++ b/tests/integration/util/test_requests.py @@ -21,9 +21,3 @@ def test_make_session(conf, persistent_launch_tor, dests): except requests.exceptions.ConnectTimeout: pass assert response is None - - # Disable socks proxies to test a session request not over Tor - # without timeout. - session.proxies = {} - response = session.get(dests.next().url, verify=False) - print(response.request.__dict__)

1 0

[sbws/master] new: v3bwfile: Add note about not resetting counters
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit e8f7d9de0be74a3e9332d32d7eb9c3aa560180ce Author: juga0 <juga(a)riseup.net> Date: Wed Mar 13 13:49:26 2019 +0000 new: v3bwfile: Add note about not resetting counters --- sbws/lib/v3bwfile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 4788840..048f1b3 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -44,6 +44,10 @@ STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] # Added in #29591 +# NOTE: recent_consensus_count, recent_priority_list_count, +# recent_measurement_attempt_count and recent_priority_relay_count +# are not reset when the scanner is stop. +# They will accumulate the values since the scanner was ever started. BW_HEADER_KEYVALUES_MONITOR = [ # 1.1 header: the number of different consensuses, that sbws has seen, # since the last 5 days

1 0

[sbws/master] new: v3bwfile: Set relays exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 5fd38798a8070818d0b521b56b5ca50943af4ca7 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:53:08 2019 +0000 new: v3bwfile: Set relays exclusion KeyValues Part of #28565. --- sbws/lib/v3bwfile.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 411d324..2c1e745 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -438,6 +438,11 @@ class V3BWHeader(object): [setattr(self, k, str(v)) for k, v in kwargs.items() if k in STATS_KEYVALUES] + def add_relays_excluded_counters(self, exclusion_dict): + log.debug("Adding relays excluded counters.") + for k, v in exclusion_dict.items(): + setattr(self, k, str(v)) + class V3BWLine(object): """ @@ -761,12 +766,22 @@ class V3BWFile(object): number_consensus_relays = cls.read_number_consensus_relays( consensus_path) state = State(state_fpath) + + # Initiliaze exclusion counts dictionary with 0 + exclusion_dict = dict( + [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + ) for fp, values in results.items(): # log.debug("Relay fp %s", fp) line, reason = V3BWLine.from_results(values, secs_recent, secs_away, min_num) if line is not None: bw_lines_raw.append(line) + else: + exclusion_dict[reason] = exclusion_dict.get(reason, 0) + 1 + # Add the headers with the number of excluded relays by reason + header.add_relays_excluded_counters(exclusion_dict) + if not bw_lines_raw: log.info("After applying restrictions to the raw results, " "there is not any. Scaling can not be applied.")

1 0

[sbws/master] fix: v3bwfile: Convert consensus count to string
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 702327c871c5665031f94789de89651a7598fe13 Author: juga0 <juga(a)riseup.net> Date: Mon Mar 11 13:05:10 2019 +0000 fix: v3bwfile: Convert consensus count to string It was forgotten in #28566 --- sbws/lib/v3bwfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 521f304..4788840 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -483,7 +483,7 @@ class V3BWLine(object): if getattr(r, 'relay_in_recent_consensus_count', None)] if consensuses_count: consensus_count = max(consensuses_count) - kwargs['relay_in_recent_consensus_count'] = consensus_count + kwargs['relay_in_recent_consensus_count'] = str(consensus_count) measurements_attempts = \ [r.relay_recent_measurement_attempt_count for r in results

1 0

[sbws/master] new: v3bwfile: Set measurements exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit c201e4f64a04a3b8bb2f10ca096fa76c8b5ce2ff Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:46:39 2019 +0000 new: v3bwfile: Set measurements exclusion KeyValues and return reason why the relay was excluded to be able to count the relays excluded in the header. Part of #28565. --- sbws/lib/v3bwfile.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index fca82ac..411d324 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -510,20 +510,29 @@ class V3BWLine(object): str(max(relay_recent_priority_list_counts)) success_results = [r for r in results if isinstance(r, ResultSuccess)] + kwargs['relay_recent_measurement_exclusion_not_success_count'] = \ + len(results) - len(success_results) if not success_results: - return None + return None, 'recent_measurement_exclusion_not_success_count' results_away = \ cls.results_away_each_other(success_results, secs_away) + kwargs['relay_recent_measurement_exclusion_not_distanciated_count'] = \ + len(success_results) - len(results_away) if not results_away: - return None + return None, 'recent_measurement_exclusion_not_distanciated_count' # log.debug("Results away from each other: %s", # [unixts_to_isodt_str(r.time) for r in results_away]) results_recent = cls.results_recent_than(results_away, secs_recent) + kwargs['relay_recent_measurement_exclusion_not_recent_count'] = \ + len(results_away) - len(results_recent) if not results_recent: - return None + return None, 'recent_measurement_exclusion_not_recent_count' + kwargs['relay_recent_measurement_exclusion_not_min_num_count'] = \ + len(results_recent) if not len(results_recent) >= min_num: # log.debug('The number of results is less than %s', min_num) - return None + return None, 'recent_measurement_exclusion_not_min_num_count' + rtt = cls.rtt_from_results(results_recent) if rtt: kwargs['rtt'] = rtt @@ -545,7 +554,7 @@ class V3BWLine(object): kwargs['desc_bw_obs_mean'] = \ cls.desc_bw_obs_mean_from_results(results_recent) bwl = cls(node_id, bw, **kwargs) - return bwl + return bwl, None @classmethod def from_data(cls, data, fingerprint): @@ -754,8 +763,8 @@ class V3BWFile(object): state = State(state_fpath) for fp, values in results.items(): # log.debug("Relay fp %s", fp) - line = V3BWLine.from_results(values, secs_recent, secs_away, - min_num) + line, reason = V3BWLine.from_results(values, secs_recent, + secs_away, min_num) if line is not None: bw_lines_raw.append(line) if not bw_lines_raw:

1 0

[sbws/master] new: tests: Update header and lines with new KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 5c600c6d03459ca0e7f48354da46510be80a7e95 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:56:37 2019 +0000 new: tests: Update header and lines with new KeyValues Part of #28565. --- tests/unit/lib/test_v3bwfile.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index c4c6a20..8c95a5e 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -8,9 +8,12 @@ from sbws import __version__ as version from sbws.globals import (SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING, MIN_REPORT, TORFLOW_ROUND_DIG, PROP276_ROUND_DIG) from sbws.lib.resultdump import Result, load_result_file, ResultSuccess -from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, - KEYVALUE_SEP_V1, num_results_of_type, - V3BWFile, round_sig_dig) +from sbws.lib.v3bwfile import ( + V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, + KEYVALUE_SEP_V1, num_results_of_type, + V3BWFile, round_sig_dig, + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED + ) from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts timestamp = 1523974147 @@ -61,6 +64,10 @@ raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "\ "relay_recent_measurement_attempt_count=2 "\ + "relay_recent_measurement_exclusion_not_distanciated_count=0 "\ + "relay_recent_measurement_exclusion_not_min_num_count=1 "\ + "relay_recent_measurement_exclusion_not_recent_count=0 "\ + "relay_recent_measurement_exclusion_not_success_count=1 "\ "relay_recent_priority_list_count=3 "\ "rtt=456 success=1 " \ "time=2018-04-17T14:09:07\n" @@ -237,7 +244,7 @@ def test_v3bwline_from_results_file(datadir): if fp not in d: d[fp] = [] d[fp].append(r) - bwl = V3BWLine.from_data(d, fp) + bwl, _ = V3BWLine.from_data(d, fp) # bw store now B, not KB bwl.bw = round(bwl.bw / 1000) assert raw_bwl_str == str(bwl) @@ -248,7 +255,11 @@ def test_from_results_read(datadir, tmpdir, conf, args): expected_header = V3BWHeader(timestamp_l, earliest_bandwidth=earliest_bandwidth, latest_bandwidth=latest_bandwidth) - raw_bwls = [V3BWLine.from_results(results[fp]) for fp in results] + exclusion_dict = dict( + [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + ) + expected_header.add_relays_excluded_counters(exclusion_dict) + raw_bwls = [V3BWLine.from_results(results[fp])[0] for fp in results] # Scale BWLines using torflow method, since it's the default and BWLines # bandwidth is the raw bandwidth. expected_bwls = V3BWFile.bw_torflow_scale(raw_bwls) @@ -343,7 +354,7 @@ def test_measured_progress_stats(datadir): results = load_result_file(str(datadir.join("results_away.txt"))) for fp, values in results.items(): # log.debug("Relay fp %s", fp) - line = V3BWLine.from_results(values) + line, _ = V3BWLine.from_results(values) if line is not None: bw_lines_raw.append(line) assert len(bw_lines_raw) == 3

1 0

[sbws/master] new: globals: Add constants to recover destination
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 7cc6de1d9f3f5c5f750b15ae971d6297cea1b17e Author: juga0 <juga(a)riseup.net> Date: Sat Mar 16 10:13:18 2019 +0000 new: globals: Add constants to recover destination Part of #29589. --- sbws/globals.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sbws/globals.py b/sbws/globals.py index 015e857..5003973 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -122,9 +122,18 @@ HTTP_GET_HEADERS = { DESTINATION_VERIFY_CERTIFICATE = True # This number might need adjusted depending on the percentage of circuits and # HTTP requests failures. -# While the scanner can not recover from some/all failing destionations, -# set a big number so that it continues trying. -MAXIMUM_NUMBER_DESTINATION_FAILURES = 100 + +# Number of attempts to use a destination, that are stored, in order to decide +# whether the destination is functional or not. +NUM_DESTINATION_ATTEMPTS_STORED = 10 +# Time to wait before trying again a destination that wasn't functional. +DELTA_SECONDS_RETRY_DESTINATION = 60 * 60 * 3 +# Number of consecutive times a destination can fail before considering it +# not functional. +MAX_NUM_DESTINATION_FAILURES = 3 +# By which factor to multiply DELTA_SECONDS_RETRY_DESTINATION when the +# destination fail again. +FACTOR_INCREMENT_DESTINATION_RETRY = 2 def fail_hard(*a, **kw):

1 0