- tor-commits - lists.torproject.org

[sbws/master] chg: relaylist, v3bwfile: Count consensus with timestamps
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit e098ebb9540144fbd04854f05ea3cf958eb006bf Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 13:42:18 2020 +0000 chg: relaylist, v3bwfile: Count consensus with timestamps in RelayList: - Rename consensus_timestamps to recent_consensus - Rename recent_consensus_count to recent_consensus when there is no counting - Use the timestamps class to manage/count consensuses - Remove method not needed anymore --- sbws/globals.py | 4 ++++ sbws/lib/relaylist.py | 35 +++++++++++---------------------- sbws/lib/v3bwfile.py | 10 +++++----- tests/data/.sbws/state.dat | 42 ++++++++++++++++++++++++++++++++-------- tests/unit/lib/test_relaylist.py | 10 ++++++---- tests/unit/lib/test_v3bwfile.py | 17 +++++++++++++--- 6 files changed, 74 insertions(+), 44 deletions(-) diff --git a/sbws/globals.py b/sbws/globals.py index a71fbc6..a9dca10 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3 # destination fail again. FACTOR_INCREMENT_DESTINATION_RETRY = 2 +# Constants to check health KeyValues in the bandwidth file +PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60)) +MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120 + def fail_hard(*a, **kw): ''' Log something ... and then exit as fast as possible ''' diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 9ffff49..5ae1c2d 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -8,8 +8,11 @@ import random import logging from threading import Lock -from ..globals import MEASUREMENTS_PERIOD -from ..util import timestamp +from ..globals import ( + MAX_RECENT_CONSENSUS_COUNT, + MEASUREMENTS_PERIOD +) +from ..util import timestamp, timestamps log = logging.getLogger(__name__) @@ -322,7 +325,9 @@ class RelayList: self.rng = random.SystemRandom() self._refresh_lock = Lock() # To track all the consensus seen. - self._consensus_timestamps = [] + self._recent_consensus = timestamps.DateTimeSeq( + [], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus" + ) # Initialize so that there's no error trying to access to it. # In future refactor, change to a dictionary, where the keys are # the relays' fingerprint. @@ -345,15 +350,7 @@ class RelayList: @property def last_consensus_timestamp(self): """Returns the datetime when the last consensus was obtained.""" - if (getattr(self, "_consensus_timestamps") - and self._consensus_timestamps): - return self._consensus_timestamps[-1] - # If the object was not created from __init__, it won't have - # consensus_timestamps attribute or it might be empty. - # In this case force new update. - # Anytime more than 1h in the past will be old. - self._consensus_timestamps = [] - return datetime.utcnow() - timedelta(seconds=60*61) + return self._recent_consensus.last() @property def relays(self): @@ -419,12 +416,6 @@ class RelayList: def _relays_without_flag(self, flag): return [r for r in self.relays if flag not in r.flags] - def _remove_old_consensus_timestamps(self): - self._consensus_timestamps = remove_old_consensus_timestamps( - copy.deepcopy(self._consensus_timestamps), - self._measurements_period - ) - def _init_relays(self): """Returns a new list of relays that are in the current consensus. And update the consensus timestamp list with the current one. @@ -441,8 +432,7 @@ class RelayList: # Find the timestamp of the last consensus. timestamp = valid_after_from_network_statuses(network_statuses) - self._consensus_timestamps.append(timestamp) - self._remove_old_consensus_timestamps() + self._recent_consensus.update(timestamp) new_relays = [] @@ -502,14 +492,11 @@ class RelayList: log.info("Number of consensuses obtained in the last %s days: %s.", int(self._measurements_period / 24 / 60 / 60), self.recent_consensus_count) - # NOTE: blocking, writes to file! - if self._state is not None: - self._state['recent_consensus_count'] = self.recent_consensus_count @property def recent_consensus_count(self): """Number of times a new consensus was obtained.""" - return len(self._consensus_timestamps) + return len(self._recent_consensus) def exits_not_bad_allowing_port(self, port): return [r for r in self.exits diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 80f1e34..70304a9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -367,7 +367,7 @@ class V3BWHeader(object): if destinations_countries is not None: kwargs['destinations_countries'] = destinations_countries if recent_consensus_count is not None: - kwargs['recent_consensus_count'] = str(recent_consensus_count) + kwargs['recent_consensus_count'] = recent_consensus_count recent_measurement_attempt_count = \ cls.recent_measurement_attempt_count_from_file(state_fpath) @@ -457,10 +457,10 @@ class V3BWHeader(object): @staticmethod def consensus_count_from_file(state_fpath): state = State(state_fpath) - if 'recent_consensus_count' in state: - return state['recent_consensus_count'] - else: - return None + count = state.count("recent_consensus") + if count: + return str(count) + return None # NOTE: in future refactor store state in the class @staticmethod diff --git a/tests/data/.sbws/state.dat b/tests/data/.sbws/state.dat index 38acf6c..8793d73 100644 --- a/tests/data/.sbws/state.dat +++ b/tests/data/.sbws/state.dat @@ -1,9 +1,35 @@ { - "uuid": "806218a0-3ce5-4778-b839-d6faf6798405", - "scanner_started": "2019-03-25T13:03:06", - "recent_consensus_count": 1, - "recent_priority_list_count": 1, - "recent_measurement_attempt_count": 15, - "min_perc_reached": null, - "recent_priority_relay_count": 15 -} \ No newline at end of file + "scanner_started": "2020-02-29T10:00:00", + "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de", + "tor_version": "0.4.2.6", + "recent_consensus": [ + "2020-03-18T13:26:46" + ], + "recent_priority_list": [ + "2020-03-18T13:26:46" + ], + "recent_priority_relay": [ + [ + "2020-03-18T13:26:46", + 15 + ] + ], + "recent_measurement_attempt": [ + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46" + ], + "min_perc_reached": null +} diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 846677f..1b22425 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta from freezegun import freeze_time from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps +from sbws.util.state import State def test_remove_old_consensus_timestamps(): @@ -31,13 +32,14 @@ def test_init_relays( new consensus is received. Test that the number of consesus timesamps and relays is correct. """ + state = State(conf['paths']['state_fpath']) # There is no need to mock datetime to update the consensus, since the # actual date will be always later. # But it's needed to have the correct list of timestamps both for RelayList # and Relay. with freeze_time("2020-02-29 10:00:00"): - relay_list = RelayList(args, conf, controller=controller) - assert len(relay_list._consensus_timestamps) == 1 + relay_list = RelayList(args, conf, controller, state=state) + assert relay_list.recent_consensus_count == 1 assert len(relay_list._relays[0]._consensus_timestamps) == 1 # The actual number of relays in the consensus assert len(relay_list._relays) == 6433 @@ -48,7 +50,7 @@ def test_init_relays( with freeze_time("2020-02-29 11:00:00"): # Call relays update the list of relays. relay_list.relays - assert len(relay_list._consensus_timestamps) == 2 + assert relay_list.recent_consensus_count == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2 # Check that the number of relays is now the previous one plus the relays # that are in the new consensus that there were not in the previous one. @@ -61,7 +63,7 @@ def test_init_relays( relay_list._controller = controller_5days_later with freeze_time("2020-03-05 10:00:01"): relay_list.relays - assert len(relay_list._consensus_timestamps) == 2 + assert relay_list.recent_consensus_count == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2 fps_5days_later = {r.fingerprint for r in relay_list._relays} # The number of added relays will be the number of relays in this diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 8fe2dab..c9c0e93 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import ( V3BWFile, round_sig_dig, HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS ) +from sbws.util.state import CustomDecoder from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts timestamp = 1523974147 @@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir): lines = datadir.readlines('results.txt') d = dict() for line in lines: - r = Result.from_dict(json.loads(line.strip())) + r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder)) fp = r.fingerprint if fp not in d: d[fp] = [] @@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir): def test_generator_started(root_data_path, datadir): state_fpath = os.path.join(root_data_path, '.sbws/state.dat') # The method is correct - assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file( + assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file( state_fpath ) # `results` does not matter here, using them to not have an empty list. results = load_result_file(str(datadir.join("results.txt"))) header = V3BWHeader.from_results(results, '', '', state_fpath) # And the header is correct - assert "2019-03-25T13:03:06" == header.generator_started + assert "2020-02-29T10:00:00" == header.generator_started + + +def test_recent_consensus_count(root_data_path, datadir): + # This state has recent_consensus_count + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + assert "1" == V3BWHeader.consensus_count_from_file(state_fpath) + # `results` does not matter here, using them to not have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + assert "1" == header.recent_consensus_count

1 0

[sbws/master] chg: relaylist: Count measurements with timestamps
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 89801c5331b80776cced97d9da3a2990516279b3 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 14:01:15 2020 +0000 chg: relaylist: Count measurements with timestamps in RelayList: - Rename recent_measurement_attempt_count to recent_measurement_attempt when there is no counting - Use the timestamps class to manage/count measurement attempts --- sbws/core/scanner.py | 2 +- sbws/globals.py | 16 +++++++++++++++ sbws/lib/relaylist.py | 20 ++++++++++--------- sbws/lib/v3bwfile.py | 2 +- tests/unit/lib/test_relaylist.py | 42 ++++++++++++++++++++++++++++++++++++++++ tests/unit/lib/test_v3bwfile.py | 11 +++++++++++ 6 files changed, 82 insertions(+), 11 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 60c8ca4..06381b8 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -515,7 +515,7 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump, # Don't start measuring a relay if sbws is stopping. if settings.end_event.is_set(): break - relay_list.increment_recent_measurement_attempt_count() + relay_list.increment_recent_measurement_attempt() target.increment_relay_recent_measurement_attempt_count() num_relays += 1 # callback and callback_err must be non-blocking diff --git a/sbws/globals.py b/sbws/globals.py index a9dca10..01a147b 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -151,6 +151,22 @@ FACTOR_INCREMENT_DESTINATION_RETRY = 2 # Constants to check health KeyValues in the bandwidth file PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60)) MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120 +# XXX: This was only defined in `config.default.ini`, it should be read from +# here. +FRACTION_RELAYS = 0.05 +# A priority list currently takes more than 3h, ideally it should only take 1h. +MIN_HOURS_PRIORITY_LIST = 1 +# As of 2020, there're less than 7000 relays. +MAX_RELAYS = 8000 +# 120 +MAX_RECENT_PRIORITY_LIST_COUNT = int( + PERIOD_DAYS * 24 / MIN_HOURS_PRIORITY_LIST +) +MAX_RELAYS_PER_PRIORITY_LIST = int(MAX_RELAYS * FRACTION_RELAYS) # 400 +# 48000 +MAX_RECENT_PRIORITY_RELAY_COUNT = ( + MAX_RECENT_PRIORITY_LIST_COUNT * MAX_RELAYS_PER_PRIORITY_LIST +) def fail_hard(*a, **kw): diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 5ae1c2d..d5e5754 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -10,6 +10,7 @@ from threading import Lock from ..globals import ( MAX_RECENT_CONSENSUS_COUNT, + MAX_RECENT_PRIORITY_RELAY_COUNT, MEASUREMENTS_PERIOD ) from ..util import timestamp, timestamps @@ -334,12 +335,10 @@ class RelayList: self._relays = [] # The period of time for which the measurements are keep. self._measurements_period = measurements_period - self._state = state - # NOTE: blocking: writes to disk - if self._state: - if self._state.get('recent_measurement_attempt_count', None) \ - is None: - self._state['recent_measurement_attempt_count'] = 0 + self._recent_measurement_attempt = timestamps.DateTimeSeq( + [], MAX_RECENT_PRIORITY_RELAY_COUNT, state, + "recent_measurement_attempt" + ) self._refresh() def _need_refresh(self): @@ -502,7 +501,7 @@ class RelayList: return [r for r in self.exits if r.is_exit_not_bad_allowing_port(port)] - def increment_recent_measurement_attempt_count(self): + def increment_recent_measurement_attempt(self): """ Increment the number of times that any relay has been queued to be measured. @@ -512,5 +511,8 @@ class RelayList: It is read and stored in a ``state`` file. """ # NOTE: blocking, writes to file! - if self._state: - self._state['recent_measurement_attempt_count'] += 1 + self._recent_measurement_attempt.update() + + @property + def recent_measurement_attempt_count(self): + return len(self._recent_measurement_attempt) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 70304a9..f81ef82 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -470,7 +470,7 @@ class V3BWHeader(object): in the recent (by default 5) days from the state file. """ state = State(state_fpath) - return state.get('recent_measurement_attempt_count', None) + return state.count('recent_measurement_attempt') @staticmethod def recent_priority_list_count_from_file(state_fpath): diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 1b22425..ea13b7b 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -75,3 +75,45 @@ def test_init_relays( # The number of relays will be the number of relays in the cosensus plus # the added ones minus the removed ones. assert 6925 == 6505 + len(added_fps) - len(removed_fps) + + +def test_increment_recent_measurement_attempt(args, conf, controller): + """Test that incrementing the measurement attempts does not go on forever + + And instead it only counts the number of attempts in the last days. + It also tests that the state file is updated correctly. + """ + state = State(conf['paths']['state_fpath']) + # For this test it does not matter that the consensus timestamps or relays + # are not correct. + relay_list = RelayList(args, conf, controller=controller, state=state) + # The initial count is 0 and the state does not have that key. + assert 0 == relay_list.recent_measurement_attempt_count + assert not state.get("recent_measurement_attempt", None) + + # Pretend that a measurement attempt is made. + with freeze_time("2020-02-29 10:00:00"): + relay_list.increment_recent_measurement_attempt() + assert 1 == relay_list.recent_measurement_attempt_count + assert [datetime(2020, 2, 29, 10, 0)] == state[ + "recent_measurement_attempt" + ] + + # And a second measurement attempt is made 4 days later. + with freeze_time("2020-03-04 10:00:00"): + relay_list.increment_recent_measurement_attempt() + assert 2 == relay_list.recent_measurement_attempt_count + assert 2 == len(state["recent_measurement_attempt"]) + + # And a third measurement attempt is made 5 days later. + with freeze_time("2020-03-05 10:00:00"): + relay_list.increment_recent_measurement_attempt() + assert 3 == relay_list.recent_measurement_attempt_count + assert 3 == len(state["recent_measurement_attempt"]) + + # And a fourth measurement attempt is made 6 days later. The first one is + # now removed and not counted. + with freeze_time("2020-03-06 10:00:00"): + relay_list.increment_recent_measurement_attempt() + assert 3 == relay_list.recent_measurement_attempt_count + assert 3 == len(state["recent_measurement_attempt"]) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index c9c0e93..2ce34ff 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -540,3 +540,14 @@ def test_recent_consensus_count(root_data_path, datadir): results = load_result_file(str(datadir.join("results.txt"))) header = V3BWHeader.from_results(results, '', '', state_fpath) assert "1" == header.recent_consensus_count + + +def test_recent_measurement_attempt_count(root_data_path, datadir): + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file( + state_fpath + ) + # `results` does not matter here, using them to not have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + assert "15" == header.recent_measurement_attempt_count

1 0

[sbws/master] chg: relayprioritizer: Count priorities with timestamps
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit c7c91370f5008aa4757340799e880e175d3b8830 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 14:49:50 2020 +0000 chg: relayprioritizer: Count priorities with timestamps in RelayPrioritizer: - Rename recent_priority_list_count to recent_priority_list when there is no counting - Rename recent_priority_relay_count to recent_priority_relay when there is no counting - Use the timestamps class to manage/count priority lists/relays --- sbws/lib/relayprioritizer.py | 38 +++++++++++------ sbws/lib/v3bwfile.py | 4 +- tests/conftest.py | 30 +++++++++++++ tests/unit/conftest.py | 11 +++++ tests/unit/lib/test_relayprioritizer.py | 76 +++++++++++++++++++++++++++++++++ tests/unit/lib/test_v3bwfile.py | 20 +++++++++ 6 files changed, 163 insertions(+), 16 deletions(-) diff --git a/sbws/lib/relayprioritizer.py b/sbws/lib/relayprioritizer.py index 4932ea8..3647a8b 100644 --- a/sbws/lib/relayprioritizer.py +++ b/sbws/lib/relayprioritizer.py @@ -6,7 +6,8 @@ import copy import time import logging -from ..util import state +from ..globals import MAX_RECENT_PRIORITY_RELAY_COUNT +from ..util import state, timestamps log = logging.getLogger(__name__) @@ -24,30 +25,39 @@ class RelayPrioritizer: self.fraction_to_return = conf.getfloat( 'relayprioritizer', 'fraction_relays') self._state = state.State(conf.getpath('paths', 'state_fname')) - if self._state is not None: - # If it was not in previous state versions, initialize it to 0 - if self._state.get('recent_priority_list_count', None) is None: - self._state['recent_priority_list_count'] = 0 - if self._state.get('recent_priority_relay_count') is None: - self._state['recent_priority_relay_count'] = 0 - - def increment_priority_lists(self): + self._recent_priority_list = timestamps.DateTimeSeq( + [], 120, self._state, "recent_priority_list" + ) + self._recent_priority_relay = timestamps.DateTimeIntSeq( + [], MAX_RECENT_PRIORITY_RELAY_COUNT, self._state, + "recent_priority_relay" + ) + + def increment_recent_priority_list(self): """ Increment the number of times that :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority` has been run. """ # NOTE: blocking, writes to file! - self._state['recent_priority_list_count'] += 1 + self._recent_priority_list.update() - def increment_priority_relays(self, relays_count): + @property + def recent_priority_list_count(self): + return len(self._recent_priority_list) + + def increment_recent_priority_relay(self, relays_count): """ Increment the number of relays that have been "prioritized" to be measured in a :meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority`. """ # NOTE: blocking, writes to file! - self._state['recent_priority_relay_count'] += relays_count + self._recent_priority_relay.update(number=relays_count) + + @property + def recent_priority_relay_count(self): + return len(self._recent_priority_relay) def best_priority(self, prioritize_result_error=False, return_fraction=True): @@ -141,11 +151,11 @@ class RelayPrioritizer: upper_limit = cutoff if return_fraction else len(relays) # NOTE: these two are blocking, write to disk # Increment the number of times ``best_priority`` has been run. - self.increment_priority_lists() + self.increment_recent_priority_list() # Increment the number of relays that have been "prioritized". # Because in a small testing network the upper limit could be smaller # than the number of relays in the network, use the length of the list. - self.increment_priority_relays(len(relays[0:upper_limit])) + self.increment_recent_priority_relay(len(relays[0:upper_limit])) for relay in relays[0:upper_limit]: log.debug('Returning next relay %s with priority %f', relay.nickname, relay.priority) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index f81ef82..ee9fd17 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -481,7 +481,7 @@ class V3BWHeader(object): in the recent (by default 5) days from the state file. """ state = State(state_fpath) - return state.get('recent_priority_list_count', None) + return state.count('recent_priority_list') @staticmethod def recent_priority_relay_count_from_file(state_fpath): @@ -490,7 +490,7 @@ class V3BWHeader(object): in the recent (by default 5) days from the state file. """ state = State(state_fpath) - return state.get('recent_priority_relay_count', None) + return state.count('recent_priority_relay') @staticmethod def latest_bandwidth_from_results(results): diff --git a/tests/conftest.py b/tests/conftest.py index d968cac..b2b54d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,10 @@ from unittest import mock from stem import descriptor +from sbws import settings +from sbws.lib import relaylist +from sbws.lib import relayprioritizer +from sbws.lib import resultdump from sbws.util.parser import create_parser @@ -87,3 +91,29 @@ def controller_5days_later(router_statuses_5days_later): controller = mock.Mock() controller.get_network_statuses.return_value = router_statuses_5days_later return controller + + +# Because of the function scoped `args` in `tests.unit.conftest`, this has to +# be function scoped too. +(a)pytest.fixture(scope='function') +def relay_list(args, conf, controller): + """Returns a RelayList containing the Relays in the controller""" + return relaylist.RelayList(args, conf, controller) + + +(a)pytest.fixture(scope='function') +def result_dump(args, conf): + """Returns a ResultDump without Results""" + # To stop the thread that would be waiting for new results + settings.set_end_event() + return resultdump.ResultDump(args, conf) + + +(a)pytest.fixture(scope="function") +def relay_prioritizer(args, conf_results, relay_list, result_dump): + """ + Returns a RelayPrioritizer with a RelayList and a ResultDump. + """ + return relayprioritizer.RelayPrioritizer( + args, conf_results, relay_list, result_dump + ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 295e777..bc812f0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -187,6 +187,17 @@ def conf(sbwshome_empty, tmpdir): return conf +(a)pytest.fixture(scope='function') +def conf_results(sbwshome_success_result_two_relays, conf): + """Minimal configuration having a datadir + + So that `ResultDump` does not raise AssertionError. + + """ + conf['paths']['sbws_home'] = sbwshome_success_result_two_relays + return conf + + @pytest.fixture() def result(): return RESULT diff --git a/tests/unit/lib/test_relayprioritizer.py b/tests/unit/lib/test_relayprioritizer.py new file mode 100644 index 0000000..e692343 --- /dev/null +++ b/tests/unit/lib/test_relayprioritizer.py @@ -0,0 +1,76 @@ +"""relayprioritizer.py unit tests.""" +from freezegun import freeze_time + + +def test_increment_recent_priority_list(relay_prioritizer): + """Test that incrementing the priority lists do not go on forever. + + And instead it only counts the number of priority lists in the last days. + """ + + state = relay_prioritizer._state + assert 0 == relay_prioritizer.recent_priority_list_count + assert not state.get("recent_priority_list", None) + + # Pretend that a priority list is made. + with freeze_time("2020-02-29 10:00:00"): + relay_prioritizer.increment_recent_priority_list() + assert 1 == relay_prioritizer.recent_priority_list_count + assert 1 == len(state["recent_priority_list"]) + + # And a second priority list is made 4 days later. + with freeze_time("2020-03-04 10:00:00"): + relay_prioritizer.increment_recent_priority_list() + assert 2 == relay_prioritizer.recent_priority_list_count + assert 2 == len(state["recent_priority_list"]) + + # And a third priority list is made 5 days later. + with freeze_time("2020-03-05 10:00:00"): + relay_prioritizer.increment_recent_priority_list() + assert 3 == relay_prioritizer.recent_priority_list_count + assert 3 == len(state["recent_priority_list"]) + + # And a fourth priority list is made 6 days later. The first one is + # now removed and not counted. + with freeze_time("2020-03-06 10:00:00"): + relay_prioritizer.increment_recent_priority_list() + assert 3 == relay_prioritizer.recent_priority_list_count + assert 3 == len(state["recent_priority_list"]) + + +def test_increment_priority_relay(relay_prioritizer): + """Test that incrementing the number of relays in the priority lists + do not go on forever. + + And instead it only counts number of relays in priority lists in the last + days. + """ + + state = relay_prioritizer._state + assert 0 == relay_prioritizer.recent_priority_relay_count + assert not state.get("recent_priority_relay", None) + + # Pretend that a priority list is made. + with freeze_time("2020-02-29 10:00:00"): + relay_prioritizer.increment_recent_priority_relay(2) + assert 2 == relay_prioritizer.recent_priority_relay_count + assert 2 == state.count("recent_priority_relay") + + # And a second priority list is made 4 days later. + with freeze_time("2020-03-04 10:00:00"): + relay_prioritizer.increment_recent_priority_relay(2) + assert 4 == relay_prioritizer.recent_priority_relay_count + assert 4 == state.count("recent_priority_relay") + + # And a third priority list is made 5 days later. + with freeze_time("2020-03-05 10:00:00"): + relay_prioritizer.increment_recent_priority_relay(2) + assert 6 == relay_prioritizer.recent_priority_relay_count + assert 6 == state.count("recent_priority_relay") + + # And a fourth priority list is made 6 days later. The first one is + # now removed and the relays are not counted. + with freeze_time("2020-03-06 10:00:00"): + relay_prioritizer.increment_recent_priority_relay(2) + assert 6 == relay_prioritizer.recent_priority_relay_count + assert 6 == state.count("recent_priority_relay") diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 2ce34ff..7d26b8d 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -551,3 +551,23 @@ def test_recent_measurement_attempt_count(root_data_path, datadir): results = load_result_file(str(datadir.join("results.txt"))) header = V3BWHeader.from_results(results, '', '', state_fpath) assert "15" == header.recent_measurement_attempt_count + + +def test_recent_priority_list_count(root_data_path, datadir): + # This state has recent_priority_list + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + assert 1 == V3BWHeader.recent_priority_list_count_from_file(state_fpath) + # `results` does not matter here, using them to don't have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + assert "1" == header.recent_priority_list_count + + +def test_recent_priority_relay_count(root_data_path, datadir): + # This state has recent_priority_relay_count + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + assert 15 == V3BWHeader.recent_priority_relay_count_from_file(state_fpath) + # `results` does not matter here, using them to don't have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + assert "15" == header.recent_priority_relay_count

1 0

[sbws/master] fix: v3bwfile: Stop calculating failures with 0 attempts
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 1225c5d9ea738723e272288d6734dbc651cf2d7b Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 07:09:19 2020 +0000 fix: v3bwfile: Stop calculating failures with 0 attempts --- sbws/lib/v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 6465892..245ffa6 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -379,7 +379,8 @@ class V3BWHeader(object): # failures = attempts - all mesaurements # Works only in the case that old measurements files already had # measurements count - if recent_measurement_attempt_count is not None: + # If this is None or 0, the failures can't be calculated + if recent_measurement_attempt_count: all_measurements = 0 for result_list in results.values(): all_measurements += len(result_list)

1 0

[sbws/master] fix: tests: Check the files generated in test net
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 4e2ccf3c34df54139954256528813305b48596c0 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 16:28:16 2020 +0000 fix: tests: Check the files generated in test net Test that the results, state and bandwidth file generated by running the scanner and the generator in the test network are correct. --- tests/integration/conftest.py | 11 ++++----- tests/integration/test_files.py | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b1312d9..170ea51 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -36,15 +36,15 @@ def tmpdir(tmpdir_factory, request): @pytest.fixture(scope='session') def sbwshome_empty(tmpdir): - """Create sbws home inside of the tests tmp dir without initializing.""" - home = tmpdir.join('.sbws') - os.makedirs(home.strpath, exist_ok=True) - return home.strpath + """Create sbws home inside of the test net tmp dir without initializing.""" + home = "/tmp/.sbws" + os.makedirs(home, exist_ok=True) + return home @pytest.fixture(scope='session') def sbwshome_dir(sbwshome_empty): - """Create sbws home inside of the tests tmp dir without initializing.""" + """Create sbws home inside of the test net tmp dir without initializing.""" os.makedirs(os.path.join(sbwshome_empty, 'datadir'), exist_ok=True) return sbwshome_empty @@ -69,6 +69,7 @@ def conf(sbwshome_dir): """Default configuration with sbws home in the tmp test dir.""" conf = _get_default_config() conf['paths']['sbws_home'] = sbwshome_dir + conf["paths"]["state_fpath"] = os.path.join(sbwshome_dir, "state.dat") conf['tor']['run_dpath'] = os.path.join(sbwshome_dir, 'tor', 'run') conf['destinations']['foo'] = 'on' conf['destinations.foo'] = {} diff --git a/tests/integration/test_files.py b/tests/integration/test_files.py new file mode 100644 index 0000000..aa7973d --- /dev/null +++ b/tests/integration/test_files.py @@ -0,0 +1,49 @@ +""" +Integration tests for the files with data to be used by the bandwidth file. + +""" +from sbws.lib.resultdump import load_recent_results_in_datadir +from sbws.lib.v3bwfile import V3BWFile +from sbws.util.state import State + + +def test_results(conf): + results = load_recent_results_in_datadir(5, conf["paths"]["datadir"]) + for fp, values in results.items(): + count = max( + [ + len(getattr(r, "relay_recent_measurement_attempt", [])) + for r in values + ] + ) + assert count == 1 + count = max( + [len(getattr(r, "relay_in_recent_consensus", [])) for r in values] + ) + assert count == 1 + count = max( + [len(getattr(r, "relay_recent_priority_list", [])) for r in values] + ) + assert count == 1 + + +def test_state(conf): + state = State(conf["paths"]["state_fpath"]) + assert 1 == state.count("recent_consensus") + assert 1 == state.count("recent_priority_list") + assert 15 == state.count("recent_priority_relay") + assert 15 == state.count("recent_measurement_attempt") + + +def test_v3bwfile(conf): + bwfile = V3BWFile.from_v1_fpath( + conf["paths"]["v3bw_fname"].format("latest") + ) + assert "1" == bwfile.header.recent_consensus_count + assert "1" == bwfile.header.recent_priority_list_count + assert "15" == bwfile.header.recent_priority_relay_count + assert "15" == bwfile.header.recent_measurement_attempt_count + for bwline in bwfile.bw_lines: + assert 1 == bwline.relay_in_recent_consensus_count + assert 1 == bwline.relay_recent_priority_list_count + assert 1 == bwline.relay_recent_measurement_attempt_count

1 0

[sbws/master] chg: v3bwfile: Count recent relay's monitoring numbers
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit c02d9b1e5ab0df3aa8ef19b1dbff8c23332ba682 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 18:21:03 2020 +0000 chg: v3bwfile: Count recent relay's monitoring numbers using timestamps class. Also add one more result to the tests data and change the test accordingly. --- sbws/lib/v3bwfile.py | 43 ++++++++++++++++++++++------------------- tests/unit/lib/data/results.txt | 5 +++-- tests/unit/lib/test_v3bwfile.py | 26 +++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index ee9fd17..6465892 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -656,26 +656,29 @@ class V3BWLine(object): kwargs['master_key_ed25519'] = results[0].master_key_ed25519 kwargs['time'] = cls.last_time_from_results(results) kwargs.update(cls.result_types_from_results(results)) - consensuses_count = \ - [r.relay_in_recent_consensus_count for r in results - if getattr(r, 'relay_in_recent_consensus_count', None)] - if consensuses_count: - consensus_count = max(consensuses_count) - kwargs['relay_in_recent_consensus_count'] = str(consensus_count) - - measurements_attempts = \ - [r.relay_recent_measurement_attempt_count for r in results - if getattr(r, 'relay_recent_measurement_attempt_count', None)] - if measurements_attempts: - kwargs['relay_recent_measurement_attempt_count'] = \ - str(max(measurements_attempts)) - - relay_recent_priority_list_counts = \ - [r.relay_recent_priority_list_count for r in results - if getattr(r, 'relay_recent_priority_list_count', None)] - if relay_recent_priority_list_counts: - kwargs['relay_recent_priority_list_count'] = \ - str(max(relay_recent_priority_list_counts)) + + # If it has not the attribute, return list to be able to call len + # If it has the attribute, but it is None, return also list + kwargs['relay_in_recent_consensus_count'] = str( + max([ + len(getattr(r, 'relay_in_recent_consensus', []) or []) + for r in results + ]) + ) + + kwargs['relay_recent_priority_list_count'] = str( + max([ + len(getattr(r, 'relay_recent_priority_list', []) or []) + for r in results + ]) + ) + + kwargs['relay_recent_measurement_attempt_count'] = str( + max([ + len(getattr(r, 'relay_recent_measurement_attempt', []) or []) + for r in results + ]) + ) success_results = [r for r in results if isinstance(r, ResultSuccess)] diff --git a/tests/unit/lib/data/results.txt b/tests/unit/lib/data/results.txt index 4a95b58..36df3d9 100644 --- a/tests/unit/lib/data/results.txt +++ b/tests/unit/lib/data/results.txt @@ -1,2 +1,3 @@ -{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000, "relay_burst_bandwidth": 123456, "relay_observed_bandwidth": 524288, "consensus_ba ndwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 3} -{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_recent_measurement_attempt_count": 2, "relay_recent_priority_list_count": 3} +{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000, "relay_burst_bandwidth": 123456, "relay_observed_bandwidth": 524288, "consensus_ba ndwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} +{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 7d26b8d..1028370 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -69,12 +69,13 @@ raw_bwl_str = "bw=56 bw_mean=61423 bw_median=55656 "\ "consensus_bandwidth=600000 consensus_bandwidth_is_unmeasured=False "\ "desc_bw_avg=1000000000 desc_bw_bur=123456 desc_bw_obs_last=524288 "\ "desc_bw_obs_mean=524288 error_circ=0 error_destination=0 error_misc=0 " \ - "error_second_relay=0 error_stream=1 " \ + "error_second_relay=0 error_stream=2 " \ "master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s " \ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "\ + "relay_in_recent_consensus_count=3 "\ "relay_recent_measurement_attempt_count=2 "\ - "relay_recent_measurements_excluded_error_count=1 "\ + "relay_recent_measurements_excluded_error_count=2 "\ "relay_recent_priority_list_count=3 "\ "rtt=456 success=1 " \ "time=2018-04-17T14:09:07\n" @@ -571,3 +572,24 @@ def test_recent_priority_relay_count(root_data_path, datadir): results = load_result_file(str(datadir.join("results.txt"))) header = V3BWHeader.from_results(results, '', '', state_fpath) assert "15" == header.recent_priority_relay_count + + +def test_relay_recent_measurement_attempt_count(root_data_path, datadir): + results = load_result_file(str(datadir.join("results.txt"))) + for fp, values in results.items(): + line = V3BWLine.from_results(values) + assert "2" == line[0].relay_recent_measurement_attempt_count + + +def test_relay_recent_priority_list_count(root_data_path, datadir): + results = load_result_file(str(datadir.join("results.txt"))) + for fp, values in results.items(): + line = V3BWLine.from_results(values) + assert "3" == line[0].relay_recent_priority_list_count + + +def test_relay_in_recent_consensus_count(root_data_path, datadir): + results = load_result_file(str(datadir.join("results.txt"))) + for fp, values in results.items(): + line = V3BWLine.from_results(values) + assert "3" == line[0].relay_in_recent_consensus_count

1 0

[sbws/master] fix: tests: Add results incrementing relays'
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 5c45e321b3ddb574fc204be2eb42c66a7bbedbe5 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 08:36:54 2020 +0000 fix: tests: Add results incrementing relays' monitoring KeyValues. --- tests/unit/lib/test_results.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/unit/lib/test_results.py b/tests/unit/lib/test_results.py index 3c2a52e..dccca09 100644 --- a/tests/unit/lib/test_results.py +++ b/tests/unit/lib/test_results.py @@ -9,6 +9,8 @@ from sbws.lib.resultdump import ResultErrorStream from sbws.lib.resultdump import _ResultType from tests.unit.globals import monotonic_time +from sbws.lib.relaylist import Relay + def test_Result(result): ''' @@ -232,3 +234,60 @@ def test_ResultErrorAuth_from_dict(time_mock): assert isinstance(r1, ResultErrorAuth) assert isinstance(r2, ResultErrorAuth) assert str(r1) == str(r2) + + +def test_relay_in_recent_consensus_count( + args, conf_results, controller, router_status, server_descriptor +): + relay = Relay( + router_status.fingerprint, + controller, + ns=router_status, + desc=server_descriptor, + ) + # Initialize the ResultSuccess as `measure_relay` does + r = ResultSuccess( + [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + assert 1 == len(r.relay_in_recent_consensus) + relay.update_relay_in_recent_consensus() + r = ResultSuccess( + [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + assert 2 == len(r.relay_in_recent_consensus) + + +def test_relay_recent_measurement_attempt_count( + args, conf_results, controller, router_status, server_descriptor +): + relay = Relay( + router_status.fingerprint, + controller, + ns=router_status, + desc=server_descriptor, + ) + relay.increment_relay_recent_measurement_attempt() + relay.increment_relay_recent_measurement_attempt() + # Initialize the ResultSuccess as `measure_relay` does + r = ResultSuccess( + [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + assert 2 == len(r.relay_recent_measurement_attempt) + + +def test_relay_recent_priority_list_count( + args, conf_results, controller, router_status, server_descriptor +): + relay = Relay( + router_status.fingerprint, + controller, + ns=router_status, + desc=server_descriptor, + ) + relay.increment_relay_recent_priority_list() + relay.increment_relay_recent_priority_list() + # Initialize the ResultSuccess as `measure_relay` does + r = ResultSuccess( + [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + assert 2 == len(r.relay_recent_priority_list)

1 0

[sbws/master] Use freeze_time() in other parts of our tests, too
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 74ad2d91e38124e005fb3adc36ac38181ee155af Author: Georg Koppen <gk(a)torproject.org> Date: Thu Apr 2 07:48:22 2020 +0000 Use freeze_time() in other parts of our tests, too When using `_relays_with_flags()` and similar methods it's possible that tests start to hang without time freezing. See bug 33748 for more details. We work around this by providing the necessary `freeze_time()` calls meanwhile. --- tests/conftest.py | 4 +++- tests/unit/lib/test_relaylist.py | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ffa7bb..5d156f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ import pytest import os.path from unittest import mock +from freezegun import freeze_time from stem import descriptor from sbws import settings @@ -120,7 +121,8 @@ def router_status(server_descriptor, router_statuses): @pytest.fixture(scope='function') def relay_list(args, conf, controller): """Returns a RelayList containing the Relays in the controller""" - return relaylist.RelayList(args, conf, controller) + with freeze_time("2020-02-29 10:00:00"): + return relaylist.RelayList(args, conf, controller) @pytest.fixture(scope='function') diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 5561d60..88f9db5 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -71,9 +71,8 @@ def test_increment_recent_measurement_attempt(args, conf, controller): It also tests that the state file is updated correctly. """ state = State(conf['paths']['state_fpath']) - # For this test it does not matter that the consensus timestamps or relays - # are not correct. - relay_list = RelayList(args, conf, controller=controller, state=state) + with freeze_time("2020-02-29 10:00:00"): + relay_list = RelayList(args, conf, controller=controller, state=state) # The initial count is 0 and the state does not have that key. assert 0 == relay_list.recent_measurement_attempt_count assert not state.get("recent_measurement_attempt", None)

1 0

[sbws/master] fix: tests: Add tests loading results
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit fa1c0a4c6a101cdd62e93ee395ce3767fd1d6e53 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 08:42:49 2020 +0000 fix: tests: Add tests loading results in ResultDump and incrementing relay's monitoring KeyValues. --- tests/unit/conftest.py | 12 ++----- tests/unit/lib/test_resultdump.py | 66 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index a74d03e..49db357 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -296,14 +296,6 @@ def end_event(): @pytest.fixture(scope='function') -def rd(args, conf, end_event): +def rd(args, conf_results): from sbws.lib.resultdump import ResultDump - # in Travis the next line gives the error: - # TypeError: __init__() takes 3 positional arguments but 4 were given - # No idea why. - # Returning None to disable the test in case ResultDump can not be - # initialized. - try: - return ResultDump(args, conf, end_event) - except TypeError: - return None + return ResultDump(args, conf_results) diff --git a/tests/unit/lib/test_resultdump.py b/tests/unit/lib/test_resultdump.py index c956dea..1180bdb 100644 --- a/tests/unit/lib/test_resultdump.py +++ b/tests/unit/lib/test_resultdump.py @@ -1,6 +1,16 @@ # -*- coding: utf-8 -*- """Unit tests for resultdump.""" -from sbws.lib.resultdump import trim_results_ip_changed + +import datetime + +from sbws.lib.relaylist import Relay +from sbws.lib.resultdump import ( + ResultError, + ResultErrorStream, + ResultSuccess, + trim_results_ip_changed, + load_result_file +) def test_trim_results_ip_changed_defaults(resultdict_ip_not_changed): @@ -31,3 +41,57 @@ def test_trim_results_ip_changed_on_changed_ipv6(caplog, assert record.levelname == 'WARNING' assert 'Reseting bandwidth results when IPv6 changes, ' \ 'is not yet implemented.\n' in caplog.text + + +def test_resultdump( + rd, args, conf_results, controller, router_status, server_descriptor +): + from sbws import settings + relay = Relay( + router_status.fingerprint, + controller, + ns=router_status, + desc=server_descriptor, + ) + relay.increment_relay_recent_priority_list() + relay.increment_relay_recent_measurement_attempt() + r = ResultSuccess( + [], 2000, relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + # Storing the result with `rd.queue.put` will not store the result to disk + # because the thread is not spawned with pytest. + rd.store_result(r) + results = rd.results_for_relay(relay) + # It has stored the result + assert 1 == len(results) + # The result has the correct attribute + assert 1 == len(results[0].relay_recent_priority_list) + # Store a second result for the sme relay + r = ResultError( + relay, ["A", "B"], "http://localhost/bw", "scanner_nick", + ) + rd.store_result(r) + assert 2 == len(results) + assert 1 == len(results[1].relay_recent_priority_list) + settings.set_end_event() + + +def test_load(datadir): + results = load_result_file(str(datadir.join("results.txt"))) + results = [v for values in results.values() for v in values] + r1 = results[1] + assert isinstance(r1, ResultSuccess) + assert isinstance( + r1.relay_recent_measurement_attempt[0], datetime.datetime + ) + assert 2 == len(r1.relay_recent_measurement_attempt) + assert 3 == len(r1.relay_recent_priority_list) + assert 3 == len(r1.relay_in_recent_consensus) + r2 = results[2] + assert isinstance(r2, ResultErrorStream) + assert isinstance( + r2.relay_recent_measurement_attempt[0], datetime.datetime + ) + assert 2 == len(r2.relay_recent_measurement_attempt) + assert 3 == len(r2.relay_recent_priority_list) + assert 3 == len(r2.relay_in_recent_consensus)

1 0

[sbws/master] fix: test: Assert that caplog messages were found
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit e9d5bb084aa0ba9a1e0aaac22b6732ffa3e40623 Author: juga0 <juga(a)riseup.net> Date: Mon Mar 23 09:19:08 2020 +0000 fix: test: Assert that caplog messages were found --- tests/unit/lib/test_heartbeat.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/lib/test_heartbeat.py b/tests/unit/lib/test_heartbeat.py index 5c82a4b..f3d8f52 100644 --- a/tests/unit/lib/test_heartbeat.py +++ b/tests/unit/lib/test_heartbeat.py @@ -19,5 +19,9 @@ def test_total_measured_percent(conf, caplog): hbeat.print_heartbeat_message() assert hbeat.previous_measurement_percent == 67 - caplog.records[1].getMessage().find("Measured in total 2 (67%)") - caplog.records[2].getMessage().find("1 relays still not measured") + assert 0 == caplog.records[1].getMessage().find( + "Measured in total 2 (67%)" + ) + assert 0 == caplog.records[2].getMessage().find( + "1 relays still not measured" + )

1 0