commit e098ebb9540144fbd04854f05ea3cf958eb006bf Author: juga0 juga@riseup.net Date: Sat Mar 21 13:42:18 2020 +0000
chg: relaylist, v3bwfile: Count consensus with timestamps
in RelayList: - Rename consensus_timestamps to recent_consensus - Rename recent_consensus_count to recent_consensus when there is no counting - Use the timestamps class to manage/count consensuses - Remove method not needed anymore --- sbws/globals.py | 4 ++++ sbws/lib/relaylist.py | 35 +++++++++++---------------------- sbws/lib/v3bwfile.py | 10 +++++----- tests/data/.sbws/state.dat | 42 ++++++++++++++++++++++++++++++++-------- tests/unit/lib/test_relaylist.py | 10 ++++++---- tests/unit/lib/test_v3bwfile.py | 17 +++++++++++++--- 6 files changed, 74 insertions(+), 44 deletions(-)
diff --git a/sbws/globals.py b/sbws/globals.py index a71fbc6..a9dca10 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3 # destination fail again. FACTOR_INCREMENT_DESTINATION_RETRY = 2
+# Constants to check health KeyValues in the bandwidth file +PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60)) +MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120 +
def fail_hard(*a, **kw): ''' Log something ... and then exit as fast as possible ''' diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 9ffff49..5ae1c2d 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -8,8 +8,11 @@ import random import logging from threading import Lock
-from ..globals import MEASUREMENTS_PERIOD -from ..util import timestamp +from ..globals import ( + MAX_RECENT_CONSENSUS_COUNT, + MEASUREMENTS_PERIOD +) +from ..util import timestamp, timestamps
log = logging.getLogger(__name__)
@@ -322,7 +325,9 @@ class RelayList: self.rng = random.SystemRandom() self._refresh_lock = Lock() # To track all the consensus seen. - self._consensus_timestamps = [] + self._recent_consensus = timestamps.DateTimeSeq( + [], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus" + ) # Initialize so that there's no error trying to access to it. # In future refactor, change to a dictionary, where the keys are # the relays' fingerprint. @@ -345,15 +350,7 @@ class RelayList: @property def last_consensus_timestamp(self): """Returns the datetime when the last consensus was obtained.""" - if (getattr(self, "_consensus_timestamps") - and self._consensus_timestamps): - return self._consensus_timestamps[-1] - # If the object was not created from __init__, it won't have - # consensus_timestamps attribute or it might be empty. - # In this case force new update. - # Anytime more than 1h in the past will be old. - self._consensus_timestamps = [] - return datetime.utcnow() - timedelta(seconds=60*61) + return self._recent_consensus.last()
@property def relays(self): @@ -419,12 +416,6 @@ class RelayList: def _relays_without_flag(self, flag): return [r for r in self.relays if flag not in r.flags]
- def _remove_old_consensus_timestamps(self): - self._consensus_timestamps = remove_old_consensus_timestamps( - copy.deepcopy(self._consensus_timestamps), - self._measurements_period - ) - def _init_relays(self): """Returns a new list of relays that are in the current consensus. And update the consensus timestamp list with the current one. @@ -441,8 +432,7 @@ class RelayList:
# Find the timestamp of the last consensus. timestamp = valid_after_from_network_statuses(network_statuses) - self._consensus_timestamps.append(timestamp) - self._remove_old_consensus_timestamps() + self._recent_consensus.update(timestamp)
new_relays = []
@@ -502,14 +492,11 @@ class RelayList: log.info("Number of consensuses obtained in the last %s days: %s.", int(self._measurements_period / 24 / 60 / 60), self.recent_consensus_count) - # NOTE: blocking, writes to file! - if self._state is not None: - self._state['recent_consensus_count'] = self.recent_consensus_count
@property def recent_consensus_count(self): """Number of times a new consensus was obtained.""" - return len(self._consensus_timestamps) + return len(self._recent_consensus)
def exits_not_bad_allowing_port(self, port): return [r for r in self.exits diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 80f1e34..70304a9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -367,7 +367,7 @@ class V3BWHeader(object): if destinations_countries is not None: kwargs['destinations_countries'] = destinations_countries if recent_consensus_count is not None: - kwargs['recent_consensus_count'] = str(recent_consensus_count) + kwargs['recent_consensus_count'] = recent_consensus_count
recent_measurement_attempt_count = \ cls.recent_measurement_attempt_count_from_file(state_fpath) @@ -457,10 +457,10 @@ class V3BWHeader(object): @staticmethod def consensus_count_from_file(state_fpath): state = State(state_fpath) - if 'recent_consensus_count' in state: - return state['recent_consensus_count'] - else: - return None + count = state.count("recent_consensus") + if count: + return str(count) + return None
# NOTE: in future refactor store state in the class @staticmethod diff --git a/tests/data/.sbws/state.dat b/tests/data/.sbws/state.dat index 38acf6c..8793d73 100644 --- a/tests/data/.sbws/state.dat +++ b/tests/data/.sbws/state.dat @@ -1,9 +1,35 @@ { - "uuid": "806218a0-3ce5-4778-b839-d6faf6798405", - "scanner_started": "2019-03-25T13:03:06", - "recent_consensus_count": 1, - "recent_priority_list_count": 1, - "recent_measurement_attempt_count": 15, - "min_perc_reached": null, - "recent_priority_relay_count": 15 -} \ No newline at end of file + "scanner_started": "2020-02-29T10:00:00", + "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de", + "tor_version": "0.4.2.6", + "recent_consensus": [ + "2020-03-18T13:26:46" + ], + "recent_priority_list": [ + "2020-03-18T13:26:46" + ], + "recent_priority_relay": [ + [ + "2020-03-18T13:26:46", + 15 + ] + ], + "recent_measurement_attempt": [ + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46", + "2020-03-18T13:26:46" + ], + "min_perc_reached": null +} diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 846677f..1b22425 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps +from sbws.util.state import State
def test_remove_old_consensus_timestamps(): @@ -31,13 +32,14 @@ def test_init_relays( new consensus is received. Test that the number of consesus timesamps and relays is correct. """ + state = State(conf['paths']['state_fpath']) # There is no need to mock datetime to update the consensus, since the # actual date will be always later. # But it's needed to have the correct list of timestamps both for RelayList # and Relay. with freeze_time("2020-02-29 10:00:00"): - relay_list = RelayList(args, conf, controller=controller) - assert len(relay_list._consensus_timestamps) == 1 + relay_list = RelayList(args, conf, controller, state=state) + assert relay_list.recent_consensus_count == 1 assert len(relay_list._relays[0]._consensus_timestamps) == 1 # The actual number of relays in the consensus assert len(relay_list._relays) == 6433 @@ -48,7 +50,7 @@ def test_init_relays( with freeze_time("2020-02-29 11:00:00"): # Call relays update the list of relays. relay_list.relays - assert len(relay_list._consensus_timestamps) == 2 + assert relay_list.recent_consensus_count == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2 # Check that the number of relays is now the previous one plus the relays # that are in the new consensus that there were not in the previous one. @@ -61,7 +63,7 @@ def test_init_relays( relay_list._controller = controller_5days_later with freeze_time("2020-03-05 10:00:01"): relay_list.relays - assert len(relay_list._consensus_timestamps) == 2 + assert relay_list.recent_consensus_count == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2 fps_5days_later = {r.fingerprint for r in relay_list._relays} # The number of added relays will be the number of relays in this diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 8fe2dab..c9c0e93 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import ( V3BWFile, round_sig_dig, HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS ) +from sbws.util.state import CustomDecoder from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts
timestamp = 1523974147 @@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir): lines = datadir.readlines('results.txt') d = dict() for line in lines: - r = Result.from_dict(json.loads(line.strip())) + r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder)) fp = r.fingerprint if fp not in d: d[fp] = [] @@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir): def test_generator_started(root_data_path, datadir): state_fpath = os.path.join(root_data_path, '.sbws/state.dat') # The method is correct - assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file( + assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file( state_fpath ) # `results` does not matter here, using them to not have an empty list. results = load_result_file(str(datadir.join("results.txt"))) header = V3BWHeader.from_results(results, '', '', state_fpath) # And the header is correct - assert "2019-03-25T13:03:06" == header.generator_started + assert "2020-02-29T10:00:00" == header.generator_started + + +def test_recent_consensus_count(root_data_path, datadir): + # This state has recent_consensus_count + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + assert "1" == V3BWHeader.consensus_count_from_file(state_fpath) + # `results` does not matter here, using them to not have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + assert "1" == header.recent_consensus_count