March 2019 - tor-commits - lists.torproject.org

[sbws/master] new: relaylist: Store total consensus count
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 1387f91ce469f2c54dfdd067cfd2cd0b3b9099a1 Author: juga0 <juga(a)riseup.net> Date: Fri Feb 15 16:06:53 2019 +0000 new: relaylist: Store total consensus count to the state file. To know the total number of consensus seen in the last days. Part of #28566. --- sbws/core/scanner.py | 2 +- sbws/lib/relaylist.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 503f9cc..73d0b0e 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -627,7 +627,7 @@ def run_speedtest(args, conf): # To do not have to pass args and conf to RelayList, pass an extra # argument with the data_period measurements_period = conf.getint('general', 'data_period') - rl = RelayList(args, conf, controller, measurements_period) + rl = RelayList(args, conf, controller, measurements_period, state) cb = CB(args, conf, controller, rl) rd = ResultDump(args, conf) rp = RelayPrioritizer(args, conf, rl, rd) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 4cac543..3857718 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -234,7 +234,7 @@ class RelayList: ''' def __init__(self, args, conf, controller, - measurements_period=MEASUREMENTS_PERIOD): + measurements_period=MEASUREMENTS_PERIOD, state=None): self._controller = controller self.rng = random.SystemRandom() self._refresh_lock = Lock() @@ -246,6 +246,7 @@ class RelayList: self._relays = [] # The period of time for which the measurements are keep. self._measurements_period = measurements_period + self._state = state self._refresh() def _need_refresh(self): @@ -368,6 +369,13 @@ class RelayList: # Set a new list of relays. self._relays = self._init_relays() + log.info("Number of consensuses obtained in the last %s days: %s.", + int(self._measurements_period / 24 / 60 / 60), + self.consensus_count) + # NOTE: blocking, writes to file! + if self._state is not None: + self._state['consensus_count'] = self.consensus_count + @property def consensus_count(self): """Number of times a new consensus was obtained."""

1 0

[sbws/master] new: v3bwfile: Include consensus count header
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 0bc03df5ee5e8d4ec3ead47d9a9761ea3c8b7978 Author: juga0 <juga(a)riseup.net> Date: Fri Feb 15 16:54:10 2019 +0000 new: v3bwfile: Include consensus count header The total number of consensuses seen in the last days. Part of #28566. --- sbws/lib/v3bwfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index d827029..fa05fbc 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -229,7 +229,9 @@ class V3BWHeader(object): kwargs = dict() latest_bandwidth = cls.latest_bandwidth_from_results(results) earliest_bandwidth = cls.earliest_bandwidth_from_results(results) + # NOTE: Blocking, reads file generator_started = cls.generator_started_from_file(state_fpath) + consensus_count = cls.consensus_count_from_file(state_fpath) timestamp = str(latest_bandwidth) kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) @@ -240,6 +242,8 @@ class V3BWHeader(object): kwargs['scanner_country'] = scanner_country if destinations_countries is not None: kwargs['destinations_countries'] = destinations_countries + if consensus_count is not None: + kwargs['recent_consensus_count'] = str(consensus_count) h = cls(timestamp, **kwargs) return h @@ -297,6 +301,14 @@ class V3BWHeader(object): return None @staticmethod + def consensus_count_from_file(state_fpath): + state = State(state_fpath) + if 'consensus_count' in state: + return state['consensus_count'] + else: + return None + + @staticmethod def latest_bandwidth_from_results(results): return round(max([r.time for fp in results for r in results[fp]]))

1 0

[sbws/master] new: tests: Add new possible measurement errors
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 846d878f530fcdbe9f5b9cb4139d862650be8636 Author: juga0 <juga(a)riseup.net> Date: Sat Feb 23 07:15:46 2019 +0000 new: tests: Add new possible measurement errors in the results file. Part of #28567. --- tests/unit/lib/test_v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 0448c69..a6de526 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -48,7 +48,8 @@ header_extra_str = LINE_SEP.join(header_extra_ls) + LINE_SEP raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "consensus_bandwidth=600000 consensus_bandwidth_is_unmeasured=False "\ "desc_bw_avg=1000000000 desc_bw_bur=123456 desc_bw_obs_last=524288 "\ - "desc_bw_obs_mean=524288 error_circ=0 error_misc=0 error_stream=1 " \ + "desc_bw_obs_mean=524288 error_circ=0 error_destination=0 error_misc=0 " \ + "error_second_relay=0 error_stream=1 " \ "master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s " \ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 " \

1 0

[sbws/master] new: resultdump: Add more ResultError types
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit eb4047e22d3b0c732e5a4de2c425c3710ff2cdf8 Author: juga0 <juga(a)riseup.net> Date: Thu Feb 7 20:03:09 2019 +0000 new: resultdump: Add more ResultError types to store all the possible errors while measuring a relay. It adds more technical debt, since there should be only a ResultError class. Part of #28567. --- sbws/lib/resultdump.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index f904925..2149e8d 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -192,6 +192,12 @@ class _ResultType(_StrEnum): ErrorCircuit = 'error-circ' ErrorStream = 'error-stream' ErrorAuth = 'error-auth' + # When it can not be found a second relay suitable to measure a relay. + # It is used in ``ResultErrorSecondRelay``. + ErrorSecondRelay = 'error-second-relay' + # When there is not a working destination Web Server. + # It is used in ``ResultErrorDestionation``. + ErrorDestination = 'error-destination' class Result: @@ -337,6 +343,10 @@ class Result: return ResultErrorStream.from_dict(d) elif d['type'] == _ResultType.ErrorAuth.value: return ResultErrorAuth.from_dict(d) + elif d['type'] == _ResultType.ErrorSecondRelay.value: + return ResultErrorSecondRelay.from_dict(d) + elif d['type'] == _ResultType.ErrorDestination.value: + return ResultErrorDestination.from_dict(d) else: raise NotImplementedError( 'Unknown result type {}'.format(d['type'])) @@ -460,6 +470,83 @@ class ResultErrorStream(ResultError): return d +class ResultErrorSecondRelay(ResultError): + """ + Error when it could not be found a second relay suitable to measure + a relay. + + A second suitable relay is a relay that: + - Has at least equal bandwidth as the relay to measure. + - If the relay to measure is not an exit, + the second relay is an exit without `bad` flag and can exit to port 443. + - If the relay to measure is an exit, the second relay is not an exit. + + It is instanciated in :func:`~sbws.core.scanner.measure_relay`. + + .. note:: this duplicates code and add more tech-debt, + since it's the same as the other + :class:`~sbws.lib.resultdump.ResultError` classes except for the + ``type``. + In a future refactor, there should be only one ``ResultError`` class + and assign the type in the ``scanner`` module. + """ + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + + @property + def type(self): + return _ResultType.ErrorSecondRelay + + @staticmethod + def from_dict(d): + assert isinstance(d, dict) + return ResultErrorSecondRelay( + Result.Relay( + d['fingerprint'], d['nickname'], d['address'], + d['master_key_ed25519']), + d['circ'], d['dest_url'], d['scanner'], + msg=d['msg'], t=d['time']) + + def to_dict(self): + d = super().to_dict() + return d + + +class ResultErrorDestination(ResultError): + """ + Error when there is not a working destination Web Server. + + It is instanciated in :func:`~sbws.core.scanner.measure_relay`. + + .. note:: this duplicates code and add more tech-debt, + since it's the same as the other + :class:`~sbws.lib.resultdump.ResultError` classes except for the + ``type``. + In a future refactor, there should be only one ``ResultError`` class + and assign the type in the ``scanner`` module. + """ + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + + @property + def type(self): + return _ResultType.ErrorSecondRelay + + @staticmethod + def from_dict(d): + assert isinstance(d, dict) + return ResultErrorSecondRelay( + Result.Relay( + d['fingerprint'], d['nickname'], d['address'], + d['master_key_ed25519']), + d['circ'], d['dest_url'], d['scanner'], + msg=d['msg'], t=d['time']) + + def to_dict(self): + d = super().to_dict() + return d + + class ResultErrorAuth(ResultError): def __init__(self, *a, **kw): super().__init__(*a, **kw)

1 0

[sbws/master] chg: !minor Replace consensus count by the name
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 7a802e22fa12398ab33207f67414dbc1c7d4c223 Author: juga0 <juga(a)riseup.net> Date: Fri Mar 8 16:06:41 2019 +0000 chg: !minor Replace consensus count by the name used in the bandwidth file, so that it is less confusing and less prone to errors. --- sbws/lib/relaylist.py | 8 ++++---- sbws/lib/resultdump.py | 31 +++++++++++++++++++------------ sbws/lib/v3bwfile.py | 15 ++++++++------- tests/unit/conftest.py | 4 ++-- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 3857718..880f88e 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -206,7 +206,7 @@ class Relay: self._remove_old_consensus_timestamps() @property - def consensus_count(self): + def relay_in_recent_consensus_count(self): """Number of times the relay was in a conensus.""" return len(self._consensus_timestamps) @@ -371,13 +371,13 @@ class RelayList: log.info("Number of consensuses obtained in the last %s days: %s.", int(self._measurements_period / 24 / 60 / 60), - self.consensus_count) + self.recent_consensus_count) # NOTE: blocking, writes to file! if self._state is not None: - self._state['consensus_count'] = self.consensus_count + self._state['recent_consensus_count'] = self.recent_consensus_count @property - def consensus_count(self): + def recent_consensus_count(self): """Number of times a new consensus was obtained.""" return len(self._consensus_timestamps) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index c081b61..f904925 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -205,7 +205,7 @@ class Result: average_bandwidth=None, burst_bandwidth=None, observed_bandwidth=None, consensus_bandwidth=None, consensus_bandwidth_is_unmeasured=None, - consensus_count=None): + relay_in_recent_consensus_count=None): self.fingerprint = fingerprint self.nickname = nickname self.address = address @@ -217,10 +217,11 @@ class Result: self.consensus_bandwidth_is_unmeasured = \ consensus_bandwidth_is_unmeasured # The number of times the relay was in a consensus. - self.consensus_count = consensus_count + self.relay_in_recent_consensus_count = \ + relay_in_recent_consensus_count def __init__(self, relay, circ, dest_url, scanner_nick, t=None, - consensus_count=None): + relay_in_recent_consensus_count=None): self._relay = Result.Relay(relay.fingerprint, relay.nickname, relay.address, relay.master_key_ed25519, relay.average_bandwidth, @@ -228,7 +229,7 @@ class Result: relay.observed_bandwidth, relay.consensus_bandwidth, relay.consensus_bandwidth_is_unmeasured, - relay.consensus_count) + relay.relay_in_recent_consensus_count) self._circ = circ self._dest_url = dest_url self._scanner = scanner_nick @@ -275,9 +276,9 @@ class Result: return self._relay.master_key_ed25519 @property - def consensus_count(self): + def relay_in_recent_consensus_count(self): """Number of times the relay was in a consensus.""" - return self._relay.consensus_count + return self._relay.relay_in_recent_consensus_count @property def circ(self): @@ -311,7 +312,8 @@ class Result: 'type': self.type, 'scanner': self.scanner, 'version': self.version, - 'consensus_count': self.consensus_count, + 'relay_in_recent_consensus_count': + self.relay_in_recent_consensus_count, } @staticmethod @@ -380,7 +382,8 @@ class ResultError(Result): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - consensus_count=d.get('consensus_count', None)), + relay_in_recent_consensus_count= # noqa + d.get('relay_in_recent_consensus_count', None)), # noqa d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -422,7 +425,8 @@ class ResultErrorCircuit(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - consensus_count=d.get('consensus_count', None)), + relay_in_recent_consensus_count= # noqa + d.get('relay_in_recent_consensus_count', None)), # noqa d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -446,7 +450,8 @@ class ResultErrorStream(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - consensus_count=d.get('consensus_count', None)), + relay_in_recent_consensus_count= # noqa + d.get('relay_in_recent_consensus_count', None)), # noqa d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -483,7 +488,8 @@ class ResultErrorAuth(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - consensus_count=d.get('consensus_count', None)), + relay_in_recent_consensus_count= # noqa + d.get('relay_in_recent_consensus_count', None)), # noqa d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -521,7 +527,8 @@ class ResultSuccess(Result): d.get('relay_burst_bandwidth'), d['relay_observed_bandwidth'], d.get('consensus_bandwidth'), d.get('consensus_bandwidth_is_unmeasured'), - consensus_count=d.get('consensus_count', None)), + relay_in_recent_consensus_count= # noqa + d.get('relay_in_recent_consensus_count', None)), # noqa d['circ'], d['dest_url'], d['scanner'], t=d['time']) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index fa05fbc..997e35c 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -231,7 +231,7 @@ class V3BWHeader(object): earliest_bandwidth = cls.earliest_bandwidth_from_results(results) # NOTE: Blocking, reads file generator_started = cls.generator_started_from_file(state_fpath) - consensus_count = cls.consensus_count_from_file(state_fpath) + recent_consensus_count = cls.consensus_count_from_file(state_fpath) timestamp = str(latest_bandwidth) kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) @@ -242,8 +242,8 @@ class V3BWHeader(object): kwargs['scanner_country'] = scanner_country if destinations_countries is not None: kwargs['destinations_countries'] = destinations_countries - if consensus_count is not None: - kwargs['recent_consensus_count'] = str(consensus_count) + if recent_consensus_count is not None: + kwargs['recent_consensus_count'] = str(recent_consensus_count) h = cls(timestamp, **kwargs) return h @@ -303,8 +303,8 @@ class V3BWHeader(object): @staticmethod def consensus_count_from_file(state_fpath): state = State(state_fpath) - if 'consensus_count' in state: - return state['consensus_count'] + if 'recent_consensus_count' in state: + return state['recent_consensus_count'] else: return None @@ -416,8 +416,9 @@ class V3BWLine(object): kwargs['master_key_ed25519'] = results[0].master_key_ed25519 kwargs['time'] = cls.last_time_from_results(results) kwargs.update(cls.result_types_from_results(results)) - consensuses_count = [r.consensus_count for r in results - if getattr(r, 'consensus_count', None)] + consensuses_count = \ + [r.relay_in_recent_consensus_count for r in results + if getattr(r, 'relay_in_recent_consensus_count', None)] if consensuses_count: consensus_count = max(consensuses_count) kwargs['relay_in_recent_consensus_count'] = consensus_count diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index f5d3d05..fa1143d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -72,7 +72,7 @@ RELAY1 = Result.Relay(FP1, NICK1, IP1, ED25519, average_bandwidth=AVG_BW, burst_bandwidth=BUR_BW, observed_bandwidth=OBS_BW, consensus_bandwidth=BW, consensus_bandwidth_is_unmeasured=UNMEASURED, - consensus_count=2) + relay_in_recent_consensus_count=2) RELAY2 = Result.Relay(FP2, NICK2, IP2, ED25519) RESULT = Result(RELAY1, CIRC12, DEST_URL, SCANNER, t=TIME1) @@ -96,7 +96,7 @@ RELAY_DICT = { "relay_observed_bandwidth": OBS_BW, "consensus_bandwidth": BW, "consensus_bandwidth_is_unmeasured": UNMEASURED, - "consensus_count": 2, + "relay_in_recent_consensus_count": 2, } BASE_RESULT_NO_RELAY_DICT = {

1 0

[sbws/master] new: v3bwfile: Add measurement attempts, failures
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 58d5a4a5c9380fc9b6524f372f982797c3416a00 Author: juga0 <juga(a)riseup.net> Date: Thu Feb 7 20:12:47 2019 +0000 new: v3bwfile: Add measurement attempts, failures and priority list counters Part of #28567. --- sbws/lib/v3bwfile.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 997e35c..521f304 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -114,6 +114,9 @@ BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ # 1.2 relay: the number of different consensuses, that sbws has seen, # since the last 5 days, that have this relay 'relay_in_recent_consensus_count', + # 2.6 relay: the number of times a relay was "prioritized" to be measured + # in the recent days (by default 5). + 'relay_recent_priority_list_count', # 3.8 relay: the number of times that sbws has tried to measure # this relay, since the last 5 days # This would be the number of times a relay was in a priority list (2.6) @@ -244,6 +247,35 @@ class V3BWHeader(object): kwargs['destinations_countries'] = destinations_countries if recent_consensus_count is not None: kwargs['recent_consensus_count'] = str(recent_consensus_count) + + recent_measurement_attempt_count = \ + cls.recent_measurement_attempt_count_from_file(state_fpath) + if recent_measurement_attempt_count is not None: + kwargs['recent_measurement_attempt_count'] = \ + str(recent_measurement_attempt_count) + + # If it is a failure that is not a ResultError, then + # failures = attempts - all mesaurements + # Works only in the case that old measurements files already had + # measurements count + if recent_measurement_attempt_count is not None: + all_measurements = 0 + for result_list in results.values(): + all_measurements += len(result_list) + measurement_failures = (recent_measurement_attempt_count + - all_measurements) + kwargs['recent_measurement_failure_count'] = \ + str(measurement_failures) + + priority_lists = cls.recent_priority_list_count_from_file(state_fpath) + if priority_lists is not None: + kwargs['recent_priority_list_count'] = str(priority_lists) + + priority_relays = \ + cls.recent_priority_relay_count_from_file(state_fpath) + if priority_relays is not None: + kwargs['recent_priority_relay_count'] = str(priority_relays) + h = cls(timestamp, **kwargs) return h @@ -308,6 +340,36 @@ class V3BWHeader(object): else: return None + # NOTE: in future refactor store state in the class + @staticmethod + def recent_measurement_attempt_count_from_file(state_fpath): + """ + Returns the number of times any relay was queued to be measured + in the recent (by default 5) days from the state file. + """ + state = State(state_fpath) + return state.get('recent_measurement_attempt_count', None) + + @staticmethod + def recent_priority_list_count_from_file(state_fpath): + """ + Returns the number of times + :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority` + was run + in the recent (by default 5) days from the state file. + """ + state = State(state_fpath) + return state.get('recent_priority_list_count', None) + + @staticmethod + def recent_priority_relay_count_from_file(state_fpath): + """ + Returns the number of times any relay was "prioritized" to be measured + in the recent (by default 5) days from the state file. + """ + state = State(state_fpath) + return state.get('recent_priority_relay_count', None) + @staticmethod def latest_bandwidth_from_results(results): return round(max([r.time for fp in results for r in results[fp]])) @@ -423,6 +485,20 @@ class V3BWLine(object): consensus_count = max(consensuses_count) kwargs['relay_in_recent_consensus_count'] = consensus_count + measurements_attempts = \ + [r.relay_recent_measurement_attempt_count for r in results + if getattr(r, 'relay_recent_measurement_attempt_count', None)] + if measurements_attempts: + kwargs['relay_recent_measurement_attempt_count'] = \ + str(max(measurements_attempts)) + + relay_recent_priority_list_counts = \ + [r.relay_recent_priority_list_count for r in results + if getattr(r, 'relay_recent_priority_list_count', None)] + if relay_recent_priority_list_counts: + kwargs['relay_recent_priority_list_count'] = \ + str(max(relay_recent_priority_list_counts)) + success_results = [r for r in results if isinstance(r, ResultSuccess)] if not success_results: return None

1 0

[sbws/master] new: relayprioritizer: Increment relay counter
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit badda6875db961e041a137f6dee31a476ecec85c Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 11:57:49 2019 +0000 new: relayprioritizer: Increment relay counter about the number of times it was in a priority list. Part of #28567. --- sbws/lib/relayprioritizer.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sbws/lib/relayprioritizer.py b/sbws/lib/relayprioritizer.py index e920bd1..bb64b06 100644 --- a/sbws/lib/relayprioritizer.py +++ b/sbws/lib/relayprioritizer.py @@ -6,6 +6,8 @@ import copy import time import logging +from ..util import state + log = logging.getLogger(__name__) @@ -21,6 +23,31 @@ class RelayPrioritizer: self.min_to_return = conf.getint('relayprioritizer', 'min_relays') self.fraction_to_return = conf.getfloat( 'relayprioritizer', 'fraction_relays') + self._state = state.State(conf.getpath('paths', 'state_fname')) + if self._state is not None: + # If it was not in previous state versions, initialize it to 0 + if self._state.get('recent_priority_list_count', None) is None: + self._state['recent_priority_list_count'] = 0 + if self._state.get('recent_priority_relay_count') is None: + self._state['recent_priority_relay_count'] = 0 + + def increment_priority_lists(self): + """ + Increment the number of times that + :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority` + has been run. + """ + # NOTE: blocking, writes to file! + self._state['recent_priority_list_count'] += 1 + + def increment_priority_relays(self, relays_count): + """ + Increment the number of relays that have been "prioritized" to be + measured in a + :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority`. + """ + # NOTE: blocking, writes to file! + self._state['recent_priority_relay_count'] += relays_count def best_priority(self, prioritize_result_error=False, return_fraction=True): @@ -112,10 +139,18 @@ class RelayPrioritizer: cutoff = max(int(len(relays) * self.fraction_to_return), self.min_to_return) upper_limit = cutoff if return_fraction else len(relays) + # NOTE: these two are blocking, write to disk + # Increment the number of times ``best_priority`` has been run. + self.increment_priority_lists() + # Increment the number of relays that have been "prioritized". + self.increment_priority_relays(upper_limit) for relay in relays[0:upper_limit]: log.debug('Returning next relay %s with priority %f', relay.nickname, relay.priority) # In a future refactor, a new attribute should not be created, # then no need to remove it. del(relay.priority) + # Increment the number of times a realy was "prioritized" to be + # measured. + relay.increment_relay_recent_priority_list_count() yield relay

1 0

[sbws/master] new: scanner: Return the new possible errors
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit cc14d90d7537ac1266cb3bc1b631a8a5da35284a Author: juga0 <juga(a)riseup.net> Date: Thu Feb 7 20:05:40 2019 +0000 new: scanner: Return the new possible errors when mesuring the relay. Part of #28567. --- sbws/core/scanner.py | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 73d0b0e..98fa305 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -11,8 +11,10 @@ from multiprocessing.context import TimeoutError from ..lib.circuitbuilder import GapsCircuitBuilder as CB from ..lib.resultdump import ResultDump -from ..lib.resultdump import ResultSuccess, ResultErrorCircuit -from ..lib.resultdump import ResultErrorStream +from ..lib.resultdump import ( + ResultSuccess, ResultErrorCircuit, ResultErrorStream, + ResultErrorSecondRelay, ResultError, # ResultErrorDestination + ) from ..lib.relaylist import RelayList from ..lib.relayprioritizer import RelayPrioritizer from ..lib.destination import (DestinationList, @@ -240,11 +242,22 @@ def measure_relay(args, conf, destinations, cb, rl, relay): """ log.debug('Measuring %s %s', relay.nickname, relay.fingerprint) + our_nick = conf['scanner']['nickname'] s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) # Probably because the scanner is stopping. if s is None: - return None + if settings.end_event.is_set(): + return None + else: + # In future refactor this should be returned from the make_session + reason = "Unable to get proxies." + log.debug(reason + ' to measure %s %s', + relay.nickname, relay.fingerprint) + return [ + ResultError(relay, [], '', our_nick, + msg=reason), + ] # Pick a destionation dest = destinations.next() # If there is no any destination at this point, it can not continue. @@ -256,6 +269,14 @@ def measure_relay(args, conf, destinations, cb, rl, relay): "the scanner can continue if one fails.") # Exit the scanner with error stopping threads first. stop_threads(signal.SIGTERM, None, 1) + # When the destinations can recover would be implemented; + # reason = 'Unable to get destination' + # log.debug(reason + ' to measure %s %s', + # relay.nickname, relay.fingerprint) + # return [ + # ResultErrorDestination(relay, [], dest.url, our_nick, + # msg=reason), + # ] # Pick a relay to help us measure the given relay. If the given relay is an # exit, then pick a non-exit. Otherwise pick an exit. helper = None @@ -274,14 +295,15 @@ def measure_relay(args, conf, destinations, cb, rl, relay): circ_fps = [relay.fingerprint, helper.fingerprint] nicknames = [relay.nickname, helper.nickname] if not helper: - # TODO: Return ResultError of some sort - log.debug('Unable to pick a 2nd relay to help measure %s (%s)', + reason = 'Unable to select a second relay' + log.debug(reason + ' to help measure %s (%s)', relay.fingerprint, relay.nickname) - return None - assert helper - assert circ_fps is not None and len(circ_fps) == 2 + return [ + ResultErrorSecondRelay(relay, [], dest.url, our_nick, + msg=reason), + ] + # Build the circuit - our_nick = conf['scanner']['nickname'] circ_id, reason = cb.build_circuit(circ_fps) if not circ_id: log.debug('Could not build circuit with path %s (%s): %s ',

1 0

[sbws/master] new: tests: assert relay's in priority list increased
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 1fa8a32450072970986b1144347856b731aab595 Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 11:59:55 2019 +0000 new: tests: assert relay's in priority list increased Part of #28567. --- tests/integration/lib/test_relayprioritizer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/lib/test_relayprioritizer.py b/tests/integration/lib/test_relayprioritizer.py index 76c5969..0cbc98e 100644 --- a/tests/integration/lib/test_relayprioritizer.py +++ b/tests/integration/lib/test_relayprioritizer.py @@ -65,5 +65,6 @@ def test_relayprioritizer_general(time_mock, sbwshome_empty, args, pos = i * -1 relay = best_list[pos] assert relay.nickname == nick + assert relay.relay_recent_priority_list_count == 1 finally: settings.end_event.set()

1 0

[sbws/master] new: scanner: Increment measurement attempt counter
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit a4b4bbae41e1618f61eb0dcccd8764bd03a73b99 Author: juga0 <juga(a)riseup.net> Date: Thu Mar 7 11:58:52 2019 +0000 new: scanner: Increment measurement attempt counter Part of #28567. --- sbws/core/scanner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 98fa305..24a975b 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -492,6 +492,8 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump, # Don't start measuring a relay if sbws is stopping. if settings.end_event.is_set(): break + relay_list.increment_recent_measurement_attempt_count() + target.increment_relay_recent_measurement_attempt_count() num_relays += 1 # callback and callback_err must be non-blocking callback = result_putter(result_dump)

1 0