[tor-commits] [sbws/master] chg: relaylist, v3bwfile: Count consensus with timestamps
juga at torproject.org
juga at torproject.org
Tue Apr 14 13:54:37 UTC 2020
commit e098ebb9540144fbd04854f05ea3cf958eb006bf
Author: juga0 <juga at riseup.net>
Date: Sat Mar 21 13:42:18 2020 +0000
chg: relaylist, v3bwfile: Count consensus with timestamps
in RelayList:
- Rename consensus_timestamps to recent_consensus
- Rename recent_consensus_count to recent_consensus when there is
no counting
- Use the timestamps class to manage/count consensuses
- Remove method not needed anymore
---
sbws/globals.py | 4 ++++
sbws/lib/relaylist.py | 35 +++++++++++----------------------
sbws/lib/v3bwfile.py | 10 +++++-----
tests/data/.sbws/state.dat | 42 ++++++++++++++++++++++++++++++++--------
tests/unit/lib/test_relaylist.py | 10 ++++++----
tests/unit/lib/test_v3bwfile.py | 17 +++++++++++++---
6 files changed, 74 insertions(+), 44 deletions(-)
diff --git a/sbws/globals.py b/sbws/globals.py
index a71fbc6..a9dca10 100644
--- a/sbws/globals.py
+++ b/sbws/globals.py
@@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3
# destination fail again.
FACTOR_INCREMENT_DESTINATION_RETRY = 2
+# Constants to check health KeyValues in the bandwidth file
+PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60))
+MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120
+
def fail_hard(*a, **kw):
''' Log something ... and then exit as fast as possible '''
diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py
index 9ffff49..5ae1c2d 100644
--- a/sbws/lib/relaylist.py
+++ b/sbws/lib/relaylist.py
@@ -8,8 +8,11 @@ import random
import logging
from threading import Lock
-from ..globals import MEASUREMENTS_PERIOD
-from ..util import timestamp
+from ..globals import (
+ MAX_RECENT_CONSENSUS_COUNT,
+ MEASUREMENTS_PERIOD
+)
+from ..util import timestamp, timestamps
log = logging.getLogger(__name__)
@@ -322,7 +325,9 @@ class RelayList:
self.rng = random.SystemRandom()
self._refresh_lock = Lock()
# To track all the consensus seen.
- self._consensus_timestamps = []
+ self._recent_consensus = timestamps.DateTimeSeq(
+ [], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus"
+ )
# Initialize so that there's no error trying to access to it.
# In future refactor, change to a dictionary, where the keys are
# the relays' fingerprint.
@@ -345,15 +350,7 @@ class RelayList:
@property
def last_consensus_timestamp(self):
"""Returns the datetime when the last consensus was obtained."""
- if (getattr(self, "_consensus_timestamps")
- and self._consensus_timestamps):
- return self._consensus_timestamps[-1]
- # If the object was not created from __init__, it won't have
- # consensus_timestamps attribute or it might be empty.
- # In this case force new update.
- # Anytime more than 1h in the past will be old.
- self._consensus_timestamps = []
- return datetime.utcnow() - timedelta(seconds=60*61)
+ return self._recent_consensus.last()
@property
def relays(self):
@@ -419,12 +416,6 @@ class RelayList:
def _relays_without_flag(self, flag):
return [r for r in self.relays if flag not in r.flags]
- def _remove_old_consensus_timestamps(self):
- self._consensus_timestamps = remove_old_consensus_timestamps(
- copy.deepcopy(self._consensus_timestamps),
- self._measurements_period
- )
-
def _init_relays(self):
"""Returns a new list of relays that are in the current consensus.
And update the consensus timestamp list with the current one.
@@ -441,8 +432,7 @@ class RelayList:
# Find the timestamp of the last consensus.
timestamp = valid_after_from_network_statuses(network_statuses)
- self._consensus_timestamps.append(timestamp)
- self._remove_old_consensus_timestamps()
+ self._recent_consensus.update(timestamp)
new_relays = []
@@ -502,14 +492,11 @@ class RelayList:
log.info("Number of consensuses obtained in the last %s days: %s.",
int(self._measurements_period / 24 / 60 / 60),
self.recent_consensus_count)
- # NOTE: blocking, writes to file!
- if self._state is not None:
- self._state['recent_consensus_count'] = self.recent_consensus_count
@property
def recent_consensus_count(self):
"""Number of times a new consensus was obtained."""
- return len(self._consensus_timestamps)
+ return len(self._recent_consensus)
def exits_not_bad_allowing_port(self, port):
return [r for r in self.exits
diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py
index 80f1e34..70304a9 100644
--- a/sbws/lib/v3bwfile.py
+++ b/sbws/lib/v3bwfile.py
@@ -367,7 +367,7 @@ class V3BWHeader(object):
if destinations_countries is not None:
kwargs['destinations_countries'] = destinations_countries
if recent_consensus_count is not None:
- kwargs['recent_consensus_count'] = str(recent_consensus_count)
+ kwargs['recent_consensus_count'] = recent_consensus_count
recent_measurement_attempt_count = \
cls.recent_measurement_attempt_count_from_file(state_fpath)
@@ -457,10 +457,10 @@ class V3BWHeader(object):
@staticmethod
def consensus_count_from_file(state_fpath):
state = State(state_fpath)
- if 'recent_consensus_count' in state:
- return state['recent_consensus_count']
- else:
- return None
+ count = state.count("recent_consensus")
+ if count:
+ return str(count)
+ return None
# NOTE: in future refactor store state in the class
@staticmethod
diff --git a/tests/data/.sbws/state.dat b/tests/data/.sbws/state.dat
index 38acf6c..8793d73 100644
--- a/tests/data/.sbws/state.dat
+++ b/tests/data/.sbws/state.dat
@@ -1,9 +1,35 @@
{
- "uuid": "806218a0-3ce5-4778-b839-d6faf6798405",
- "scanner_started": "2019-03-25T13:03:06",
- "recent_consensus_count": 1,
- "recent_priority_list_count": 1,
- "recent_measurement_attempt_count": 15,
- "min_perc_reached": null,
- "recent_priority_relay_count": 15
-}
\ No newline at end of file
+ "scanner_started": "2020-02-29T10:00:00",
+ "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de",
+ "tor_version": "0.4.2.6",
+ "recent_consensus": [
+ "2020-03-18T13:26:46"
+ ],
+ "recent_priority_list": [
+ "2020-03-18T13:26:46"
+ ],
+ "recent_priority_relay": [
+ [
+ "2020-03-18T13:26:46",
+ 15
+ ]
+ ],
+ "recent_measurement_attempt": [
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46",
+ "2020-03-18T13:26:46"
+ ],
+ "min_perc_reached": null
+}
diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py
index 846677f..1b22425 100644
--- a/tests/unit/lib/test_relaylist.py
+++ b/tests/unit/lib/test_relaylist.py
@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
+from sbws.util.state import State
def test_remove_old_consensus_timestamps():
@@ -31,13 +32,14 @@ def test_init_relays(
new consensus is received.
Test that the number of consesus timesamps and relays is correct.
"""
+ state = State(conf['paths']['state_fpath'])
# There is no need to mock datetime to update the consensus, since the
# actual date will be always later.
# But it's needed to have the correct list of timestamps both for RelayList
# and Relay.
with freeze_time("2020-02-29 10:00:00"):
- relay_list = RelayList(args, conf, controller=controller)
- assert len(relay_list._consensus_timestamps) == 1
+ relay_list = RelayList(args, conf, controller, state=state)
+ assert relay_list.recent_consensus_count == 1
assert len(relay_list._relays[0]._consensus_timestamps) == 1
# The actual number of relays in the consensus
assert len(relay_list._relays) == 6433
@@ -48,7 +50,7 @@ def test_init_relays(
with freeze_time("2020-02-29 11:00:00"):
# Call relays update the list of relays.
relay_list.relays
- assert len(relay_list._consensus_timestamps) == 2
+ assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
# Check that the number of relays is now the previous one plus the relays
# that are in the new consensus that there were not in the previous one.
@@ -61,7 +63,7 @@ def test_init_relays(
relay_list._controller = controller_5days_later
with freeze_time("2020-03-05 10:00:01"):
relay_list.relays
- assert len(relay_list._consensus_timestamps) == 2
+ assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
fps_5days_later = {r.fingerprint for r in relay_list._relays}
# The number of added relays will be the number of relays in this
diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py
index 8fe2dab..c9c0e93 100644
--- a/tests/unit/lib/test_v3bwfile.py
+++ b/tests/unit/lib/test_v3bwfile.py
@@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import (
V3BWFile, round_sig_dig,
HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS
)
+from sbws.util.state import CustomDecoder
from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts
timestamp = 1523974147
@@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir):
lines = datadir.readlines('results.txt')
d = dict()
for line in lines:
- r = Result.from_dict(json.loads(line.strip()))
+ r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder))
fp = r.fingerprint
if fp not in d:
d[fp] = []
@@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir):
def test_generator_started(root_data_path, datadir):
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
# The method is correct
- assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file(
+ assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file(
state_fpath
)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
# And the header is correct
- assert "2019-03-25T13:03:06" == header.generator_started
+ assert "2020-02-29T10:00:00" == header.generator_started
+
+
+def test_recent_consensus_count(root_data_path, datadir):
+ # This state has recent_consensus_count
+ state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
+ assert "1" == V3BWHeader.consensus_count_from_file(state_fpath)
+ # `results` does not matter here, using them to not have an empty list.
+ results = load_result_file(str(datadir.join("results.txt")))
+ header = V3BWHeader.from_results(results, '', '', state_fpath)
+ assert "1" == header.recent_consensus_count
More information about the tor-commits
mailing list