- tor-commits - lists.torproject.org

[sbws/master] new: destination: Add configuration option
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 1667e592c82e2e4823986f3542bceb6ab6a2feff Author: juga0 <juga(a)riseup.net> Date: Sat Mar 16 13:15:56 2019 +0000 new: destination: Add configuration option for the maximum number of failures before the destination is not functional. Part of #29589. --- docs/source/examples/sbws.example.ini | 4 ++++ sbws/lib/destination.py | 10 +++++++++- sbws/util/config.py | 6 ++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/source/examples/sbws.example.ini b/docs/source/examples/sbws.example.ini index 5b9b756..1a2d6a4 100644 --- a/docs/source/examples/sbws.example.ini +++ b/docs/source/examples/sbws.example.ini @@ -25,6 +25,10 @@ verify = False # Use ZZ if the location is unknown (for instance, a CDN). country = ZZ +# Number of consecutive times that a destination could not be used to measure +# before stopping to try to use it for a while that by default is 3h. +max_num_failures = 3 + ## The following logging options are set by default. ## There is no need to change them unless other options are prefered. ; [logging] diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index 59474a6..d2ffa84 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -286,7 +286,15 @@ class Destination: assert 'url' in conf_section url = conf_section['url'] verify = _parse_verify_option(conf_section) - return Destination(url, max_dl, verify) + try: + max_num_failures = conf_section.getint('max_num_failures') + except ValueError: + log.warning("Configuration max_num_failures is wrong, ignoring.") + max_num_failures = None + if max_num_failures: + return Destination(url, max_dl, verify, max_num_failures) + else: + return Destination(url, max_dl, verify) class DestinationList: diff --git a/sbws/util/config.py b/sbws/util/config.py index 6622ba3..e1bc429 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -406,14 +406,16 @@ def _validate_destinations(conf): urls = { 'url': {}, } - all_valid_keys = list(urls.keys()) + ['verify', 'country'] + all_valid_keys = list(urls.keys()) \ + + ['verify', 'country', 'max_num_failures'] for sec in dest_sections: if sec not in conf: errors.append('{} is an enabled destination but is not a ' 'section in the config'.format(sec)) continue errors.extend(_validate_section_keys( - conf, sec, all_valid_keys, err_tmpl, allow_missing=['verify'])) + conf, sec, all_valid_keys, err_tmpl, + allow_missing=['verify', 'max_num_failures'])) errors.extend(_validate_section_urls(conf, sec, urls, err_tmpl)) errors.extend(_validate_country(conf, sec, 'country', err_tmpl)) return errors

1 0

[sbws/master] new: v3bwfile: Add header and lines exclusion keys
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit b222416198c0bae810a7024ddb91a15c509e122b Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:44:18 2019 +0000 new: v3bwfile: Add header and lines exclusion keys Part of #28565. --- sbws/lib/v3bwfile.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 048f1b3..fca82ac 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -43,6 +43,18 @@ EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] +BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ + # The number of success results should be: + # the number of attempts - the number of failures + # 4.6 header: the number of successful results, created in the last 5 days, + # that were excluded by a filter + # This is the sum of the following 3 + not success results + # 'recent_measurement_exclusion_count', + 'recent_measurement_exclusion_not_success_count', + 'recent_measurement_exclusion_not_distanciated_count', + 'recent_measurement_exclusion_not_recent_count', + 'recent_measurement_exclusion_not_min_num_count', +] # Added in #29591 # NOTE: recent_consensus_count, recent_priority_list_count, # recent_measurement_attempt_count and recent_priority_relay_count @@ -67,16 +79,7 @@ BW_HEADER_KEYVALUES_MONITOR = [ # something else we don't know yet # So far is the number of ResultError 'recent_measurement_failure_count', - # The number of success results should be: - # the number of attempts - the number of failures - # 4.6 header: the number of successful results, created in the last 5 days, - # that were excluded by a filter - # This is the sum of the following 3 + not success results - # 'recent_measurement_exclusion_count', - 'recent_measurement_exclusion_not_distanciated_count', - 'recent_measurement_exclusion_not_recent_count', - 'recent_measurement_exclusion_not_min_num_count', -] +] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED BANDWIDTH_HEADER_KEY_VALUES_INIT = \ ['earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries']\ @@ -135,11 +138,14 @@ BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ 'relay_recent_measurement_failure_count', # The number of success results should be: # the number of attempts - the number of failures + # Number of error results, it can be deduced counting the number of errors + # but this way seem clearer. + 'relay_recent_measurement_exclusion_not_success_count', # 4.8 relay: the number of successful results, created in the last 5 days, - # that were excluded by a rule, for this relay + # that were excluded by a rule, for this relay. # This would be the sum of the following 3 + the number of not success - 'relay_recent_measurement_exclusion_count', - 'relay_recent_measurement_exclusion_not_distanciated', + # 'relay_recent_measurement_exclusion_count', + 'relay_recent_measurement_exclusion_not_distanciated_count', 'relay_recent_measurement_exclusion_not_recent_count', 'relay_recent_measurement_exclusion_not_min_num_count', ]

1 0

[sbws/master] new: tests: include measurements count KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 3c2c03512375c11dfbaf1d42b82ef2e889e23c7e Author: juga0 <juga(a)riseup.net> Date: Fri Feb 8 11:08:00 2019 +0000 new: tests: include measurements count KeyValues Closes: #28567 --- tests/unit/lib/data/results.txt | 4 ++-- tests/unit/lib/test_v3bwfile.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/lib/data/results.txt b/tests/unit/lib/data/results.txt index 59a35b4..4a95b58 100644 --- a/tests/unit/lib/data/results.txt +++ b/tests/unit/lib/data/results.txt @@ -1,2 +1,2 @@ -{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000, "relay_burst_bandwidth": 123456, "relay_observed_bandwidth": 524288, "consensus_ba ndwidth": 600000, "consensus_bandwidth_is_unmeasured": false} -{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000, "relay_burst_bandwidth": 123456, "relay_observed_bandwidth": 524288, "consensus_ba ndwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 3} +{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_recent_measurement_attempt_count": 2, "relay_recent_priority_list_count": 3} diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index a6de526..c4c6a20 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -28,8 +28,15 @@ file_created_l = KEYVALUE_SEP_V1.join(['file_created', file_created]) latest_bandwidth = '2018-04-17T14:09:07' latest_bandwidth_l = KEYVALUE_SEP_V1.join(['latest_bandwidth', latest_bandwidth]) +attempts = '1' +attempts_l = KEYVALUE_SEP_V1.join(['recent_measurement_attempt_count', + attempts]) +failure = '0' +failure_l = KEYVALUE_SEP_V1.join(['recent_measurement_failure_count', + failure]) header_ls = [timestamp_l, version_l, destinations_countries_l, file_created_l, latest_bandwidth_l, + # attempts_l, failure_l, scanner_country_l, software_l, software_version_l, TERMINATOR] header_str = LINE_SEP.join(header_ls) + LINE_SEP earliest_bandwidth = '2018-04-16T14:09:07' @@ -52,7 +59,10 @@ raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "error_second_relay=0 error_stream=1 " \ "master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s " \ "nick=A " \ - "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 " \ + "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "\ + "relay_recent_measurement_attempt_count=2 "\ + "relay_recent_priority_list_count=3 "\ + "rtt=456 success=1 " \ "time=2018-04-17T14:09:07\n" v3bw_str = header_extra_str + raw_bwl_str

1 0

[sbws/master] fixup! new: v3bwfile: Set relays exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 26380423ca8466bd1bb3047ac76b50e5e5e94314 Author: juga0 <juga(a)riseup.net> Date: Thu Mar 14 11:52:33 2019 +0000 fixup! new: v3bwfile: Set relays exclusion KeyValues --- sbws/lib/v3bwfile.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 80d4ca6..245c75a 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -452,6 +452,10 @@ class V3BWHeader(object): if k in STATS_KEYVALUES] def add_relays_excluded_counters(self, exclusion_dict): + """ + Add the monitoring KeyValues to the header about the number of + relays not included because they were not ``eligible``. + """ log.debug("Adding relays excluded counters.") for k, v in exclusion_dict.items(): setattr(self, k, str(v)) @@ -792,7 +796,13 @@ class V3BWFile(object): consensus_path) state = State(state_fpath) - # Initiliaze exclusion counts dictionary with 0 + # Create a dictionary with the number of relays excluded by any of the + # of the filtering rules that makes relays non-`eligible`. + # NOTE: In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is + # explained what are the KeyValues. + # They were introduced in #28061 and #27338. + # Using the KeyValues introduced in #28565 + # Initiliaze exclusion counts dictionary with 0. exclusion_dict = dict( [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] )

1 0

[sbws/master] new: destination: Recover destination when it failed
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 083e7c702313ffd3e4bf2be31ec19cb90bc046a1 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 16 10:21:22 2019 +0000 new: destination: Recover destination when it failed Closes: #29589. --- sbws/core/scanner.py | 24 ++--- sbws/lib/destination.py | 171 +++++++++++++++++++++++------- tests/integration/lib/test_destination.py | 52 ++++----- tests/unit/lib/test_destination.py | 54 ++++++++++ 4 files changed, 222 insertions(+), 79 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 24a975b..4b06225 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -267,16 +267,13 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.critical("There are not any functional destinations.\n" "It is recommended to set several destinations so that " "the scanner can continue if one fails.") - # Exit the scanner with error stopping threads first. - stop_threads(signal.SIGTERM, None, 1) - # When the destinations can recover would be implemented; - # reason = 'Unable to get destination' - # log.debug(reason + ' to measure %s %s', - # relay.nickname, relay.fingerprint) - # return [ - # ResultErrorDestination(relay, [], dest.url, our_nick, - # msg=reason), - # ] + # NOTE: Because this is executed in a thread, stop_threads can not + # be call from here, it has to be call from the main thread. + # Instead set the singleton end event, that will call stop_threads + # from the main process. + # Errors with only one destination are set in ResultErrorStream. + settings.end_event.set() + return None # Pick a relay to help us measure the given relay. If the given relay is an # exit, then pick a non-exit. Otherwise pick an exit. helper = None @@ -321,10 +318,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.debug('Destination %s unusable via circuit %s (%s), %s', dest.url, circ_fps, nicknames, usable_data) cb.close_circuit(circ_id) - # TODO: Return a different/new type of ResultError? - msg = 'The destination seemed to have stopped being usable' return [ - ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg), + ResultErrorStream(relay, circ_fps, dest.url, our_nick, + msg=usable_data), ] assert is_usable assert 'content_length' in usable_data @@ -561,7 +557,7 @@ def wait_for_results(num_relays_to_measure, pending_results): than the time to request over the network.) """ num_last_measured = 1 - while num_last_measured > 0: + while num_last_measured > 0 and not settings.end_event.is_set(): log.info("Pending measurements: %s out of %s: ", len(pending_results), num_relays_to_measure) time.sleep(TIMEOUT_MEASUREMENTS) diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index a92df61..59474a6 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -1,3 +1,5 @@ +import collections +import datetime import logging import random import requests @@ -6,8 +8,13 @@ from stem.control import EventType from sbws.globals import DESTINATION_VERIFY_CERTIFICATE import sbws.util.stem as stem_utils +from ..globals import ( + MAX_NUM_DESTINATION_FAILURES, + DELTA_SECONDS_RETRY_DESTINATION, + NUM_DESTINATION_ATTEMPTS_STORED, + FACTOR_INCREMENT_DESTINATION_RETRY + ) -from ..globals import MAXIMUM_NUMBER_DESTINATION_FAILURES log = logging.getLogger(__name__) @@ -98,73 +105,155 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): try: head = session.head(dest.url, verify=dest.verify) except requests.exceptions.RequestException as e: - dest.set_failure() + dest.add_failure() return False, 'Could not connect to {} over circ {} {}: {}'.format( dest.url, circ_id, stem_utils.circuit_str(cont, circ_id), e) finally: stem_utils.remove_event_listener(cont, listener) if head.status_code != requests.codes.ok: - dest.set_failure() + dest.add_failure() return False, error_prefix + 'we expected HTTP code '\ '{} not {}'.format(requests.codes.ok, head.status_code) if 'content-length' not in head.headers: - dest.set_failure() + dest.add_failure() return False, error_prefix + 'we except the header Content-Length '\ 'to exist in the response' content_length = int(head.headers['content-length']) if max_dl > content_length: - dest.set_failure() + dest.add_failure() return False, error_prefix + 'our maximum configured download size '\ 'is {} but the content is only {}'.format(max_dl, content_length) log.debug('Connected to %s over circuit %s', dest.url, circ_id) - # Any failure connecting to the destination will call set_failure, - # which will set `failed` to True and count consecutives failures. - # It can not be set at the start, to be able to know if it failed a - # a previous time, which is checked by set_failure. - # Future improvement: use a list to count consecutive failures - # or calculate it from the results. - dest.failed = False + # Any failure connecting to the destination will call add_failure, + # It can not be set at the start, to be able to know whether it is + # failing consecutive times. + dest.add_success() return True, {'content_length': content_length} class Destination: - def __init__(self, url, max_dl, verify): + """Web server from which data is downloaded to measure bandwidth. + """ + # NOTE: max_dl and verify should be optional and have defaults + def __init__(self, url, max_dl, verify, + max_num_failures=MAX_NUM_DESTINATION_FAILURES, + delta_seconds_retry=DELTA_SECONDS_RETRY_DESTINATION, + num_attempts_stored=NUM_DESTINATION_ATTEMPTS_STORED, + factor_increment_retry=FACTOR_INCREMENT_DESTINATION_RETRY): + """Initalizes the Web server from which the data is downloaded. + + :param str url: Web server data URL to download. + :param int max_dl: Maximum size of the the data to download. + :param bool verify: Whether to verify or not the TLS certificate. + :param int max_num_failures: Number of consecutive failures when the + destination is not considered functional. + :param int delta_seconds_retry: Delta time to try a destination + that was not functional. + :param int num_attempts_stored: Number of attempts to store. + :param int factor_increment_retry: Factor to increment delta by + before trying to use a destination again. + """ self._max_dl = max_dl u = urlparse(url) self._url = u self._verify = verify - # Flag to record whether this destination failed in the last - # measurement. - # Failures can happen if: - # - an HTTPS request can not be made over Tor - # (which might be the relays fault, not the destination being - # unreachable) - # - the destination does not support HTTP Range requests. - self.failed = False - self.consecutive_failures = 0 - @property - def is_functional(self): + # Attributes to decide whether a destination is functional or not. + self._max_num_failures = max_num_failures + self._num_attempts_stored = num_attempts_stored + # Default delta time to try a destination that was not functional. + self._default_delta_seconds_retry = delta_seconds_retry + self._delta_seconds_retry = delta_seconds_retry + # Using a deque (FIFO) to do not grow forever and + # to do not have to remove old attempts. + # Store tuples of timestamp and whether the destination succed or not + # (succed, 1, failed, 0). + # Initialize it as if it never failed. + self._attempts = collections.deque([(datetime.datetime.utcnow(), 1), ], + maxlen=self._num_attempts_stored) + self._factor = factor_increment_retry + + def _last_attempts(self, n=None): + """Return the last ``n`` attempts the destination was used.""" + # deque does not accept slices, + # a new deque is returned with the last n items + # (or less if there were less). + return collections.deque(self._attempts, + maxlen=(n or self._max_num_failures)) + + def _are_last_attempts_failures(self, n=None): + """ + Return True if the last`` n`` times the destination was used + and failed. + """ + # Count the number that there was a failure when used + n = n if n else self._max_num_failures + return ([i[1] for i in self._last_attempts(n)].count(0) + >= self._max_num_failures) + + def _increment_time_to_retry(self, factor=None): + """ + Increment the time a destination will be tried again by a ``factor``. """ - Returns True if there has not been a number consecutive measurements. - Otherwise warn about it and return False. + self._delta_seconds_retry *= factor or self._factor + log.info("Incremented the time to try destination %s to %s hours.", + self.url, self._delta_seconds_retry / 60 / 60) + def _is_last_try_old_enough(self, n=None): + """ + Return True if the last time it was used it was ``n`` seconds ago. """ - if self.consecutive_failures > MAXIMUM_NUMBER_DESTINATION_FAILURES: - log.warning("Destination %s is not functional. Please check that " - "it is correct.", self._url) + # Timestamp of the last attempt. + last_time = self._attempts[-1][0] + # If the last attempt is older than _delta_seconds_retry, + if (datetime.datetime.utcnow() + - datetime.timedelta(seconds=self._delta_seconds_retry) + > last_time): + # And try again. + return True + return False + + def is_functional(self): + """Whether connections to a destination are failing or not. + + Return True if: + - It did not fail more than n (by default 3) consecutive times. + - The last time the destination was tried + was x (by default 3h) seconds ago. + And False otherwise. + + When the destination is tried again after the consecutive failures, + the time to try again is incremented and resetted as soon as the + destination does not fail. + """ + # Failed the last X consecutive times + if self._are_last_attempts_failures(): + log.warning("The last %s times the destination %s failed." + "It will not be used again in %s hours.\n", + self._max_num_failures, self.url, + self._delta_seconds_retry / 60 / 60) + log.warning("Please, add more destinations or increment the " + "number of maximum number of consecutive failures " + "in the configuration.") + # It was not used for a while and the last time it was used + # was long ago, then try again + if self._is_last_try_old_enough(): + log.info("The destination %s was not tried for %s hours, " + "it is going to by tried again.") + # Set the next time to retry higher, in case this attempt fails + self._increment_time_to_retry() + return True return False + # Reset the time to retry to the initial value + # In case it was incrememented + self._delta_seconds_retry = self._default_delta_seconds_retry return True - def set_failure(self): - """Set failed to True and increase the number of consecutive failures. - Only if it also failed in the previous measuremnt. + def add_failure(self, dt=None): + self._attempts.append((dt or datetime.datetime.utcnow(), 0)) - """ - # if it failed in the last measurement - if self.failed: - self.consecutive_failures += 1 - self.failed = True + def add_success(self, dt=None): + self._attempts.append((dt or datetime.datetime.utcnow(), 1)) @property def url(self): @@ -213,7 +302,7 @@ class DestinationList: @property def functional_destinations(self): - return [d for d in self._all_dests if d.is_functional] + return [d for d in self._all_dests if d.is_functional()] @staticmethod def from_config(conf, circuit_builder, relay_list, controller): @@ -250,4 +339,8 @@ class DestinationList: # This removes the need for an extra lock for every measurement. # Do not change the order of the destinations, just return a # destination. - return self._rng.choice(self.functional_destinations) + # random.choice raises IndexError with an empty list. + if self.functional_destinations: + return self._rng.choice(self.functional_destinations) + else: + return None diff --git a/tests/integration/lib/test_destination.py b/tests/integration/lib/test_destination.py index 54cbacc..98ed89f 100644 --- a/tests/integration/lib/test_destination.py +++ b/tests/integration/lib/test_destination.py @@ -1,5 +1,4 @@ """Integration tests for destination.py""" -from sbws.globals import MAXIMUM_NUMBER_DESTINATION_FAILURES import sbws.util.requests as requests_utils from sbws.lib.destination import (DestinationList, Destination, connect_to_destination_over_circuit) @@ -36,16 +35,12 @@ def test_connect_to_destination_over_circuit_success(persistent_launch_tor, destination, circuit_id, session, persistent_launch_tor, 1024) assert is_usable is True assert 'content_length' in response - assert not destination.failed - assert destination.consecutive_failures == 0 - assert destination.is_functional + assert destination.is_functional() def test_connect_to_destination_over_circuit_fail(persistent_launch_tor, dests, cb, rl): bad_destination = Destination('https://example.example', 1024, False) - # dests._all_dests.append(bad_destination) - # dests._usable_dests.append(bad_destination) session = requests_utils.make_session(persistent_launch_tor, 10) # Choose a relay that is not an exit relay = [r for r in rl.relays @@ -61,35 +56,40 @@ def test_connect_to_destination_over_circuit_fail(persistent_launch_tor, assert is_usable is False # because it is the first time it fails, failures aren't count - assert bad_destination.failed - assert bad_destination.consecutive_failures == 0 - assert bad_destination.is_functional + assert bad_destination.is_functional() - # fail twice in a row + # fail three times in a row is_usable, response = connect_to_destination_over_circuit( bad_destination, circuit_id, session, persistent_launch_tor, 1024) - assert bad_destination.failed - assert bad_destination.consecutive_failures == 1 - assert bad_destination.is_functional + is_usable, response = connect_to_destination_over_circuit( + bad_destination, circuit_id, session, persistent_launch_tor, 1024) + assert not bad_destination.is_functional() def test_functional_destinations(conf, cb, rl, persistent_launch_tor): good_destination = Destination('https://127.0.0.1:28888', 1024, False) - # Mock that it failed before and just now, but it's still considered - # functional. - good_destination.consecutive_failures = 3 - good_destination.failed = True bad_destination = Destination('https://example.example', 1024, False) - # Mock that it didn't fail now, but it already failed 11 consecutive - # times. - bad_destination.consecutive_failures = \ - MAXIMUM_NUMBER_DESTINATION_FAILURES + 1 - bad_destination.failed = False - # None of the arguments are used, move to unit tests when this get - # refactored + + session = requests_utils.make_session(persistent_launch_tor, 10) + # Choose a relay that is not an exit + relay = [r for r in rl.relays + if r.nickname == 'relay1mbyteMAB'][0] + # Choose an exit, for this test it does not matter the bandwidth + helper = rl.exits_not_bad_allowing_port(bad_destination.port)[0] + circuit_path = [relay.fingerprint, helper.fingerprint] + # Build a circuit. + circuit_id, _ = cb.build_circuit(circuit_path) + + # fail three times in a row + is_usable, response = connect_to_destination_over_circuit( + bad_destination, circuit_id, session, persistent_launch_tor, 1024) + is_usable, response = connect_to_destination_over_circuit( + bad_destination, circuit_id, session, persistent_launch_tor, 1024) + is_usable, response = connect_to_destination_over_circuit( + bad_destination, circuit_id, session, persistent_launch_tor, 1024) + destination_list = DestinationList( conf, [good_destination, bad_destination], cb, rl, persistent_launch_tor) - expected_functional_destinations = [good_destination] functional_destinations = destination_list.functional_destinations - assert expected_functional_destinations == functional_destinations + assert [good_destination] == functional_destinations diff --git a/tests/unit/lib/test_destination.py b/tests/unit/lib/test_destination.py new file mode 100644 index 0000000..fc8b489 --- /dev/null +++ b/tests/unit/lib/test_destination.py @@ -0,0 +1,54 @@ +"""Unit tests for sbws.lib.destination.""" +from datetime import datetime, timedelta + +from sbws.lib import destination + + +def test_destination_is_functional(): + eight_hours_ago = datetime.utcnow() - timedelta(hours=8) + four_hours_ago = datetime.utcnow() - timedelta(hours=4) + two_hours_ago = datetime.utcnow() - timedelta(hours=2) + + d = destination.Destination('unexistenturl', 0, False) + assert d.is_functional() + + # Fail 3 consecutive times + d.add_failure() + d.add_failure() + d.add_failure() + assert d._are_last_attempts_failures() + assert not d._is_last_try_old_enough() + assert not d.is_functional() + + # Then doesn't fail and it's functional again + d.add_success() + assert not d._are_last_attempts_failures() + assert d.is_functional() + + # Fail again 3 times + d.add_failure() + d.add_failure() + # And last failure was 2h ago + d.add_failure(two_hours_ago) + assert d._are_last_attempts_failures() + assert not d._is_last_try_old_enough() + assert not d.is_functional() + + # But if the last failure was 4h ago, try to use it again + # And last failure was 4h ago + d.add_failure(four_hours_ago) + assert d._is_last_try_old_enough() + assert d.is_functional() + + # If last failure was 8h ago, try to use it again again + d.add_failure(eight_hours_ago) + assert d._is_last_try_old_enough() + assert d.is_functional() + + # Whenever it does not fail again, reset the time to try again + # on 3 consecutive failures + d.add_success() + assert not d._are_last_attempts_failures() + assert d.is_functional() + # And the delta to try is resetted + assert not d._is_last_try_old_enough()

1 0

[sbws/master] fixup! new: v3bwfile: Set measurements exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit b60684b032b453a1af80692c44af9950fa1d345c Author: juga0 <juga(a)riseup.net> Date: Thu Mar 14 11:52:00 2019 +0000 fixup! new: v3bwfile: Set measurements exclusion KeyValues --- sbws/lib/v3bwfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 99b3914..80d4ca6 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -528,6 +528,18 @@ class V3BWLine(object): str(max(relay_recent_priority_list_counts)) success_results = [r for r in results if isinstance(r, ResultSuccess)] + + # NOTE: The following 4 conditions exclude relays from the bandwidth + # file when the measurements does not satisfy some rules, what makes + # the relay non-`eligible`. + # In BANDWIDTH_LINE_KEY_VALUES_MONITOR it is explained what they mean. + # In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is also + # explained the what it means the strings returned. + # They rules were introduced in #28061 and #27338 + # In #28565 we introduce the KeyValues to know why they're excluded. + # In #28563 will include again these lines, but make them unparseable + # by Tor. + # This might confirm #28042. kwargs['relay_recent_measurement_exclusion_not_success_count'] = \ len(results) - len(success_results) if not success_results:

1 0

[sbws/master] new: destination: do not attach stream to destination
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit cc314712c3ead555c6997be18e292a4fcc69888e Author: juga0 <juga(a)riseup.net> Date: Sat Mar 16 13:18:31 2019 +0000 new: destination: do not attach stream to destination if sbws is shutting down. --- sbws/lib/destination.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index d2ffa84..21a907c 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -14,6 +14,7 @@ from ..globals import ( NUM_DESTINATION_ATTEMPTS_STORED, FACTOR_INCREMENT_DESTINATION_RETRY ) +from sbws import settings log = logging.getLogger(__name__) @@ -98,6 +99,10 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): should commence. False and an error string otherwise. ''' assert isinstance(dest, Destination) + log.debug("Connecting to destination over circuit.") + # Do not start if sbws is stopping + if settings.end_event.is_set(): + return False, "Shutting down." error_prefix = 'When sending HTTP HEAD to {}, '.format(dest.url) with stem_utils.stream_building_lock: listener = stem_utils.attach_stream_to_circuit_listener(cont, circ_id) @@ -205,12 +210,10 @@ class Destination: """ # Timestamp of the last attempt. last_time = self._attempts[-1][0] - # If the last attempt is older than _delta_seconds_retry, - if (datetime.datetime.utcnow() + # If the last attempt is older than _delta_seconds_retry, try again + return (datetime.datetime.utcnow() - datetime.timedelta(seconds=self._delta_seconds_retry) - > last_time): - # And try again. - return True + > last_time) return False def is_functional(self):

1 0

[sbws/master] new: v3bwfile: Add header and lines exclusion keys
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 98443e0bdd2a9f96ccef31ebec0b7b1db3be3676 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:44:18 2019 +0000 new: v3bwfile: Add header and lines exclusion keys Part of #28565. --- sbws/lib/v3bwfile.py | 59 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 048f1b3..086d17c 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -43,6 +43,28 @@ EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] + +# KeyValues that count the number of relays that are in the bandwidth file, +# but ignored by Tor when voting, because they do not have a +# measured bandwidth. +BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ + # Number of relays that were measured but all the measurements failed + # because of network failures or it was + # not found a suitable helper relay + 'recent_measurements_excluded_error_count', + # Number of relays that have successful measurements but the measurements + # were not away from each other in X time (by default 1 day). + 'recent_measurements_excluded_near_count', + # Number of relays that have successful measurements and they are away from + # each other but they are not X time recent. + # By default this is 5 days, which is the same time the older + # the measurements can be by default. + 'recent_measurements_excluded_old_count', + # Number of relays that have successful measurements and they are away from + # each other and recent + # but the number of measurements are less than X (by default 2). + 'recent_measurements_excluded_few_count', +] # Added in #29591 # NOTE: recent_consensus_count, recent_priority_list_count, # recent_measurement_attempt_count and recent_priority_relay_count @@ -67,16 +89,7 @@ BW_HEADER_KEYVALUES_MONITOR = [ # something else we don't know yet # So far is the number of ResultError 'recent_measurement_failure_count', - # The number of success results should be: - # the number of attempts - the number of failures - # 4.6 header: the number of successful results, created in the last 5 days, - # that were excluded by a filter - # This is the sum of the following 3 + not success results - # 'recent_measurement_exclusion_count', - 'recent_measurement_exclusion_not_distanciated_count', - 'recent_measurement_exclusion_not_recent_count', - 'recent_measurement_exclusion_not_min_num_count', -] +] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED BANDWIDTH_HEADER_KEY_VALUES_INIT = \ ['earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries']\ @@ -133,15 +146,23 @@ BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ # something else we don't know yet # So far is the number of ResultError 'relay_recent_measurement_failure_count', - # The number of success results should be: - # the number of attempts - the number of failures - # 4.8 relay: the number of successful results, created in the last 5 days, - # that were excluded by a rule, for this relay - # This would be the sum of the following 3 + the number of not success - 'relay_recent_measurement_exclusion_count', - 'relay_recent_measurement_exclusion_not_distanciated', - 'relay_recent_measurement_exclusion_not_recent_count', - 'relay_recent_measurement_exclusion_not_min_num_count', + # Number of error results created in the last 5 days that are excluded. + # This is the sum of all the errors. + 'relay_recent_measurements_excluded_error_count', + # The number of successful results, created in the last 5 days, + # that were excluded by a rule, for this relay. + # 'relay_recent_measurements_excluded_error_count' would be the + # sum of the following 3 + the number of error results. + + # The number of successful measurements that are not X time away + # from each other (by default 1 day). + 'relay_recent_measurements_excluded_near_count', + # The number of successful measurements that are away from each other + # but not X time recent (by default 5 days). + 'relay_recent_measurements_excluded_old_count', + # The number of measurements excluded because they are not at least X + # (by default 2). + 'relay_recent_measurements_excluded_few_count', ] BW_KEYVALUES_EXTRA = BW_KEYVALUES_FILE + BW_KEYVALUES_EXTRA_BWS \ + BANDWIDTH_LINE_KEY_VALUES_MONITOR

1 0

[sbws/master] fixup! new: v3bwfile: Set measurements exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 39739fa38acf91853f62e99234eaaf0b878783b8 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 17 13:10:30 2019 +0000 fixup! new: v3bwfile: Set measurements exclusion KeyValues --- sbws/lib/v3bwfile.py | 5 +++-- tests/unit/lib/test_v3bwfile.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 245c75a..0a95510 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -561,9 +561,10 @@ class V3BWLine(object): len(results_away) - len(results_recent) if not results_recent: return None, 'recent_measurement_exclusion_not_recent_count' - kwargs['relay_recent_measurement_exclusion_not_min_num_count'] = \ - len(results_recent) + if not len(results_recent) >= min_num: + kwargs['relay_recent_measurement_exclusion_not_min_num_count'] = \ + len(results_recent) # log.debug('The number of results is less than %s', min_num) return None, 'recent_measurement_exclusion_not_min_num_count' diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 8c95a5e..022e070 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -65,7 +65,6 @@ raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "\ "relay_recent_measurement_attempt_count=2 "\ "relay_recent_measurement_exclusion_not_distanciated_count=0 "\ - "relay_recent_measurement_exclusion_not_min_num_count=1 "\ "relay_recent_measurement_exclusion_not_recent_count=0 "\ "relay_recent_measurement_exclusion_not_success_count=1 "\ "relay_recent_priority_list_count=3 "\

1 0

[sbws/master] new: v3bwfile: Set measurements exclusion KeyValues
by juga＠torproject.org 21 Mar '19

21 Mar '19

commit 718a3145190792392ed618d6adf88e089bc8fdb0 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 12 10:46:39 2019 +0000 new: v3bwfile: Set measurements exclusion KeyValues and return reason why the relay was excluded to be able to count the relays excluded in the header. Part of #28565. --- sbws/lib/v3bwfile.py | 47 ++++++++++++++++++++++++++++++++++------- tests/unit/lib/test_v3bwfile.py | 27 ++++++++++++++++++++++- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 086d17c..cb6d29c 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -525,20 +525,51 @@ class V3BWLine(object): str(max(relay_recent_priority_list_counts)) success_results = [r for r in results if isinstance(r, ResultSuccess)] + + # NOTE: The following 4 conditions exclude relays from the bandwidth + # file when the measurements does not satisfy some rules, what makes + # the relay non-`eligible`. + # In BANDWIDTH_LINE_KEY_VALUES_MONITOR it is explained what they mean. + # In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is also + # explained the what it means the strings returned. + # They rules were introduced in #28061 and #27338 + # In #28565 we introduce the KeyValues to know why they're excluded. + # In #28563 we report these relays, but make Tor ignore them. + # This might confirm #28042. + + number_excluded_error = len(results) - len(success_results) + if number_excluded_error > 0: + # then the number of error results is the number of results + kwargs['relay_recent_measurements_excluded_error_count'] = \ + number_excluded_error if not success_results: - return None + return None, 'recent_measurements_excluded_error_count' + results_away = \ cls.results_away_each_other(success_results, secs_away) + number_excluded_near = len(success_results) - len(results_away) + if number_excluded_near > 0: + kwargs['relay_recent_measurements_excluded_near_count'] = \ + len(success_results) - len(results_away) if not results_away: - return None + return None, 'recent_measurements_excluded_near_count' + # log.debug("Results away from each other: %s", # [unixts_to_isodt_str(r.time) for r in results_away]) results_recent = cls.results_recent_than(results_away, secs_recent) + number_excluded_old = len(results_away) - len(results_recent) + if number_excluded_old > 0: + kwargs['relay_recent_measurements_excluded_old_count'] = \ + number_excluded_old if not results_recent: - return None + return None, 'recent_measurements_excluded_old_count' + if not len(results_recent) >= min_num: + kwargs['relay_recent_measurements_excluded_few_count'] = \ + len(results_recent) # log.debug('The number of results is less than %s', min_num) - return None + return None, 'recent_measurements_excluded_few_count' + rtt = cls.rtt_from_results(results_recent) if rtt: kwargs['rtt'] = rtt @@ -560,7 +591,7 @@ class V3BWLine(object): kwargs['desc_bw_obs_mean'] = \ cls.desc_bw_obs_mean_from_results(results_recent) bwl = cls(node_id, bw, **kwargs) - return bwl + return bwl, None @classmethod def from_data(cls, data, fingerprint): @@ -594,7 +625,7 @@ class V3BWLine(object): return results # log.debug("Results are NOT away from each other in at least %ss: %s", # secs_away, [unixts_to_isodt_str(r.time) for r in results]) - return None + return [] @staticmethod def results_recent_than(results, secs_recent=None): @@ -769,8 +800,8 @@ class V3BWFile(object): state = State(state_fpath) for fp, values in results.items(): # log.debug("Relay fp %s", fp) - line = V3BWLine.from_results(values, secs_recent, secs_away, - min_num) + line, reason = V3BWLine.from_results(values, secs_recent, + secs_away, min_num) if line is not None: bw_lines_raw.append(line) if not bw_lines_raw: diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index c4c6a20..da7b456 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -61,6 +61,9 @@ raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "\ "relay_recent_measurement_attempt_count=2 "\ + "relay_recent_measurement_exclusion_not_distanciated_count=0 "\ + "relay_recent_measurement_exclusion_not_recent_count=0 "\ + "relay_recent_measurement_exclusion_not_success_count=1 "\ "relay_recent_priority_list_count=3 "\ "rtt=456 success=1 " \ "time=2018-04-17T14:09:07\n" @@ -309,16 +312,31 @@ def test_results_away_each_other(datadir): results = load_result_file(str(datadir.join("results_away.txt"))) # A has 4 results, 3 are success, 2 are 1 day away, 1 is 12h away values = results["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"] + + # There is one result excluded, but the relay is not excluded + bwl, reason = V3BWLine.from_results(values, secs_away=secs_away, min_num=2) + assert bwl.relay_recent_measurement_exclusion_error_count == 1 + assert reason is None + success_results = [r for r in values if isinstance(r, ResultSuccess)] assert len(success_results) >= min_num results_away = V3BWLine.results_away_each_other(success_results, secs_away) assert len(results_away) == 3 + # B has 2 results, 12h away from each other values = results["BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"] + + # Two measurements are excluded and there were only 2, + # the relay is excluded + bwl, reason = V3BWLine.from_results(values, secs_away=secs_away, min_num=2) + # TODO ticket28563: uncomment + # assert bwl.relay_recent_measurement_exclusion_near_count == 2 + assert reason == 'recent_measurement_exclusion_near_count' + success_results = [r for r in values if isinstance(r, ResultSuccess)] assert len(success_results) >= min_num results_away = V3BWLine.results_away_each_other(success_results, secs_away) - assert results_away is None + assert not results_away secs_away = 43200 # 12h values = results["BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"] success_results = [r for r in values if isinstance(r, ResultSuccess)] @@ -327,6 +345,13 @@ def test_results_away_each_other(datadir): assert len(results_away) == 2 # C has 1 result values = results["CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"] + + # There is only 1 result, the relay is excluded + bwl, reason = V3BWLine.from_results(values, min_num=2) + # TODO ticket28563: uncomment + # assert bwl.recent_measurement_exclusion_few_count == 1 + assert reason == 'recent_measurement_exclusion_few_count' + success_results = [r for r in values if isinstance(r, ResultSuccess)] assert len(success_results) < min_num

1 0