April 2020 - tor-commits - lists.torproject.org

[sbws/maint-1.1] Merge branch 'bug_33009_v6' into 'maint-1.1'
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 303342187dc164eb56f1a8d49ef859e69ae72c06 Merge: 3efb686 20df35c Author: Juga <juga(a)torproject.org> Date: Tue Apr 14 13:47:57 2020 +0000 Merge branch 'bug_33009_v6' into 'maint-1.1' Bug 33009 v6 See merge request torproject/network-health/sbws!6 sbws/core/scanner.py | 6 ++++++ sbws/lib/relaylist.py | 35 +++++++++++++++++++++++++++++++++++ tests/conftest.py | 4 +++- tests/unit/lib/test_relaylist.py | 16 +++++++++++++--- 4 files changed, 57 insertions(+), 4 deletions(-)

1 0

[sbws/maint-1.1] fix: tests: Test state file consistency
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 89efc2b6722c71b009ef3746a613b9fa512f01a8 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 17:02:43 2020 +0000 fix: tests: Test state file consistency Test that two different instances of state don't overwrite each other. This test don't past in this commit, will pass in the next bugfix. Bugfix v0.7.0, which claimed 100% test coverage on state. --- tests/unit/util/test_state.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index dd23d1a..2443049 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -117,3 +117,12 @@ def test_state_iter(tmpdir): for key in d: state[key] = d[key] assert set([key for key in state]) == set(d) + + +def test_two_instances(tmpdir): + """Test that 2 different intances don't overwrite each other""" + s1 = State(os.path.join(str(tmpdir), 'state.dat')) + s2 = State(os.path.join(str(tmpdir), 'state.dat')) + s1["x"] = "foo" + s2["y"] = "bar" + assert s2["x"] == "foo"

1 0

[sbws/maint-1.1] fix: state: Read file before setting key
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit fe827ac182d7cee0061660d32dd56a7d11af3c65 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 17:10:57 2020 +0000 fix: state: Read file before setting key Otherwise, if other instance of state set a key, it's lost by the current instance. Bugfix v0.7.0. --- sbws/util/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbws/util/state.py b/sbws/util/state.py index 1d33c3d..f460413 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -97,6 +97,9 @@ class State: raise TypeError( 'May only store value with type in %s, not %s' % (State._ALLOWED_TYPES, type(value))) + # NOTE: important, read the file before setting the key, + # otherwise if other instances are creating other keys, they're lost. + self._state = self._read() self._state.__setitem__(key, value) self._write()

1 0

[sbws/maint-1.1] fix: state: Let json manage data types
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 26409a3d8187cc65147a25484655a7639bdb705e Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 17:27:59 2020 +0000 fix: state: Let json manage data types Since state uses json and json will raise an error when it can't decode/encode some datatype. --- sbws/util/state.py | 34 +++++++--------------------------- tests/unit/util/test_state.py | 43 ------------------------------------------- 2 files changed, 7 insertions(+), 70 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index f460413..86cc59b 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -4,16 +4,12 @@ import json class State: - ''' - State allows one to atomically access and update a simple state file on - disk across threads and across processes. + """ + `json` wrapper to read a json file every time it gets a key and to write + to the file every time a key is set. - To put it blunty, due to limited developer time and his inability to - quickly find a way to safely access and update more complex data types - (namely, collections like list, set, and dict), you may only store simple - types of data as enumerated in _ALLOWED_TYPES. Keys must be strings. - - Data is stored as JSON on disk in the provided file file. + Every time a key is got or set, the file is locked, to atomically access + and update the file across threads and across processes. >>> state = State('foo.state') >>> # state == {} @@ -40,8 +36,8 @@ class State: >>> # We can do many of the same things with a State object as with a dict >>> for key in state: print(key) >>> # Prints 'linux', 'age', and 'name' - ''' - _ALLOWED_TYPES = (int, float, str, bool, type(None)) + + """ def __init__(self, fname): self._fname = fname @@ -68,35 +64,19 @@ class State: Implements a dictionary ``get`` method reading and locking a json file. """ - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() return self._state.get(key, d) def __getitem__(self, key): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() return self._state.__getitem__(key) def __delitem__(self, key): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() self._state.__delitem__(key) self._write() def __setitem__(self, key, value): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) - if type(value) not in State._ALLOWED_TYPES: - raise TypeError( - 'May only store value with type in %s, not %s' % - (State._ALLOWED_TYPES, type(value))) # NOTE: important, read the file before setting the key, # otherwise if other instances are creating other keys, they're lost. self._state = self._read() diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index 2443049..a57768d 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -11,26 +11,6 @@ def test_state_set_allowed_key_types(tmpdir): assert state[key] == 4 -def test_state_set_bad_key_types(tmpdir): - state = State(os.path.join(str(tmpdir), 'statefoo')) - attempt_keys = (15983, None, True, -1.2, [], {}, set()) - for key in attempt_keys: - try: - state[key] = 4 - except TypeError: - pass - else: - assert None, 'Should not have been able to use %s %s as a key' %\ - (key, type(key)) - try: - state[key] - except TypeError: - pass - else: - assert None, '%s %s is not a valid key type, so should have got '\ - 'TypeError when giving it' % (key, type(key)) - - def test_state_set_allowed_value_types(tmpdir): state = State(os.path.join(str(tmpdir), 'statefoo')) attempt_vals = (15983, None, True, -1.2, 'loooooool') @@ -39,19 +19,6 @@ def test_state_set_allowed_value_types(tmpdir): assert state['foo'] == val -def test_state_set_bad_value_types(tmpdir): - state = State(os.path.join(str(tmpdir), 'statefoo')) - attempt_vals = ([], {}, set()) - for val in attempt_vals: - try: - state['foo'] = val - except TypeError: - pass - else: - assert None, 'Should not have been able to use %s %s as a value' %\ - (val, type(val)) - - def test_state_del(tmpdir): state = State(os.path.join(str(tmpdir), 'statefoo')) d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} @@ -65,16 +32,6 @@ def test_state_del(tmpdir): for key in d: assert d[key] == state[key] - attempt_keys = (15983, None, True, -1.2, [], {}, set()) - for key in attempt_keys: - try: - del state[key] - except TypeError: - pass - else: - assert None, 'Should not have been allowed to delete %s %s '\ - 'because it is not a valid key type' % (key, type(key)) - d['e'] = 5 state['e'] = 5 d['e'] = 5.5

1 0

[sbws/maint-1.1] chg: timestamps: Add module to manage datetime sequences
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 003337ee7d241e18637ad5d67099d06cb0b4e9e8 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 18:27:44 2020 +0000 chg: timestamps: Add module to manage datetime sequences --- sbws/util/timestamps.py | 89 ++++++++++++++++++++++++++++++++++++++ tests/unit/util/test_timestamps.py | 44 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/sbws/util/timestamps.py b/sbws/util/timestamps.py new file mode 100644 index 0000000..393b30b --- /dev/null +++ b/sbws/util/timestamps.py @@ -0,0 +1,89 @@ +"""Util classes to manipulate sequences of datetime timestamps. + +Optionally update also a state file. + +""" +# Workarounds to store datetimes for objects because they are not compossed +# by other objects nor stored in a database with a creation datetime. +import collections +from datetime import datetime, timedelta +import logging + +from sbws.util.timestamp import is_old + +log = logging.getLogger(__name__) + + +class DateTimeSeq(collections.deque): + """Store and manage a datetime sequence and optionally a state file.""" + + def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): + self._maxlen = maxlen + self._items = collections.deque(iterable, maxlen) + self._state = state + self._state_key = state_key + + def _remove_old(self): + self._items = collections.deque( + filter(lambda x: not is_old(x), self._items), maxlen=self._maxlen + ) + + def update(self, dt=None): + self._remove_old() + self._items.append(dt or datetime.utcnow().replace(microsecond=0)) + if self._state is not None and self._state_key: + self._state[self._state_key] = list(self._items) + return list(self._items) + + def last(self): + if len(self._items) > 0: + return self._items[-1] + return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) + + def list(self): + return list(self._items) + + def __len__(self): + return len(self._items) + + +class DateTimeIntSeq(collections.deque): + """ + Store and manage a sequence of lists composed of a datetime and an int. + + Optionally store and manage an state file. + """ + + def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): + self._maxlen = maxlen + self._items = collections.deque(iterable, maxlen) + self._state = state + self._state_key = state_key + + def _remove_old(self): + self._items = collections.deque( + filter(lambda x: not is_old(x[0]), self._items), + maxlen=self._maxlen, + ) + + def update(self, dt=None, number=0): + self._remove_old() + # Because json serializes tuples to lists, use list instead of tuple + # to facilitate comparisons. + self._items.append( + [dt or datetime.utcnow().replace(microsecond=0), number] + ) + if self._state is not None and self._state_key: + self._state[self._state_key] = list(self._items) + return list(self._items) + + def last(self): + if len(self._items) > 0: + return self._items[-1] + return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) + + def list(self): + return list(self._items) + + def __len__(self): + return sum(map(lambda x: x[1], self._items)) diff --git a/tests/unit/util/test_timestamps.py b/tests/unit/util/test_timestamps.py new file mode 100644 index 0000000..aef0501 --- /dev/null +++ b/tests/unit/util/test_timestamps.py @@ -0,0 +1,44 @@ +"""timestamps.py unit tests.""" + +from datetime import datetime, timedelta + +from sbws.util.state import State +from sbws.util.timestamps import ( + DateTimeSeq, + DateTimeIntSeq, +) + + +def test_update_datetime_seq(conf): + now = datetime.utcnow().replace(microsecond=0) + state = State(conf["paths"]["state_fpath"]) + # Create a list of 6 datetimes that started 6 days in the past. + dts = [now - timedelta(days=x) for x in range(6, 0, -1)] + dt_seq = DateTimeSeq( + dts, state=state, state_key="recent_measurement_attempt" + ) + new_dts = dt_seq.update() + # The updated list will not contain the 2 first (left) datetimes and it + # will have one last timestamp (right). + assert new_dts[:-1] == dts[2:] + assert 5 == state.count("recent_measurement_attempt") + assert 5 == len(dt_seq) + + +def test_update_datetime_int_seq(conf): + now = datetime.utcnow().replace(microsecond=0) + state = State(conf["paths"]["state_fpath"]) + # Create a list of 6 datetimes that started 6 days in the past. + dts = [[now - timedelta(days=x), 2] for x in range(6, 0, -1)] + dt_seq = DateTimeIntSeq( + dts, state=state, state_key="recent_measurement_attempt" + ) + new_dts = dt_seq.update() + # The updated list will not contain the 2 first (left) tuples and it + # will have one last tuple (right). + # The last tuple have 0 as the integer, instead of 2, so the count will be + # 2 * 4 = 8 + assert new_dts[:-1] == dts[2:] + assert 8 == state.count("recent_measurement_attempt") + # And `len` should return the same. + assert 8 == len(dt_seq)

1 0

[sbws/maint-1.1] chg: state: Add method to count list values
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 9acad379acffa9c167529e7e12a2c445208cd60d Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 18:24:50 2020 +0000 chg: state: Add method to count list values --- sbws/util/state.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sbws/util/state.py b/sbws/util/state.py index 86cc59b..50e6a66 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -90,3 +90,18 @@ class State: def __contains__(self, item): self._state = self._read() return self._state.__contains__(item) + + def count(self, k): + """ + Returns the length if the key value is a list + or the sum of number if the key value is a list of list + or the key value + or None if the state doesn't have the key. + """ + if self.get(k): + if isinstance(self._state[k], list): + if isinstance(self._state[k][0], list): + return sum(map(lambda x: x[1], self._state[k])) + return len(self._state[k]) + return self.get(k) + return None

1 0

[sbws/maint-1.1] chg: state: Encode/decode datetimes
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit af849a06a7183b06aa1ab7ed478ed862c86b6566 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 13:01:50 2020 +0000 chg: state: Encode/decode datetimes --- sbws/util/state.py | 6 ++++-- tests/unit/util/test_state.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index 50e6a66..b6614a7 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -2,6 +2,8 @@ from sbws.util.filelock import FileLock import os import json +from .json import CustomDecoder, CustomEncoder + class State: """ @@ -48,12 +50,12 @@ class State: return {} with FileLock(self._fname): with open(self._fname, 'rt') as fd: - return json.load(fd) + return json.load(fd, cls=CustomDecoder) def _write(self): with FileLock(self._fname): with open(self._fname, 'wt') as fd: - return json.dump(self._state, fd, indent=4) + return json.dump(self._state, fd, indent=4, cls=CustomEncoder) def __len__(self): self._state = self._read() diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index a57768d..d08a4a9 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -83,3 +83,11 @@ def test_two_instances(tmpdir): s1["x"] = "foo" s2["y"] = "bar" assert s2["x"] == "foo" + + +def test_datetime_values(tmpdir): + import datetime + state = State(os.path.join(str(tmpdir), 'state.dat')) + now = datetime.datetime.utcnow().replace(microsecond=0) + state["datetimes"] = now + assert now == state["datetimes"]

1 0

[sbws/maint-1.1] chg: v3bwfile: Convert datetime to str
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 65589118cd18500722acee1fd26e9b2a76083033 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 08:04:48 2020 +0000 chg: v3bwfile: Convert datetime to str --- sbws/lib/v3bwfile.py | 6 ++++-- tests/unit/lib/test_v3bwfile.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 2fd7ce4..80f1e34 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -18,7 +18,8 @@ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, - now_unixts, isostr_to_dt_obj) + now_unixts, isostr_to_dt_obj, + dt_obj_to_isodt_str) from sbws.util.state import State log = logging.getLogger(__name__) @@ -448,7 +449,8 @@ class V3BWHeader(object): ''' state = State(state_fpath) if 'scanner_started' in state: - return state['scanner_started'] + # From v1.1.0-dev `state` is capable of converting strs to datetime + return dt_obj_to_isodt_str(state['scanner_started']) else: return None diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 566cc8d..8fe2dab 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -516,3 +516,16 @@ def test_set_under_min_report(mock_consensus, conf, datadir): assert bwl.vote == 0 assert bwl.under_min_report == 1 assert bwl.bw != 1 + + +def test_generator_started(root_data_path, datadir): + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + # The method is correct + assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file( + state_fpath + ) + # `results` does not matter here, using them to not have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + # And the header is correct + assert "2019-03-25T13:03:06" == header.generator_started

1 0

[sbws/maint-1.1] chg: resultdump: Use custom json encoder/decoder
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 8e5da39c171bc84c5ae37786d7dad198957a0d11 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 13:11:46 2020 +0000 chg: resultdump: Use custom json encoder/decoder --- sbws/lib/resultdump.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index f7ec55b..02952f9 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -12,6 +12,7 @@ from datetime import timedelta from enum import Enum from sbws.globals import RESULT_VERSION, fail_hard from sbws.util.filelock import DirectoryLock +from sbws.util.json import CustomEncoder, CustomDecoder from sbws.lib.relaylist import Relay from .. import settings @@ -45,7 +46,9 @@ def load_result_file(fname, success_only=False): for line in fd: num_total += 1 try: - r = Result.from_dict(json.loads(line.strip())) + r = Result.from_dict( + json.loads(line.strip(), cls=CustomDecoder) + ) except json.decoder.JSONDecodeError: log.warning('Could not decode result %s', line.strip()) r = None @@ -416,7 +419,7 @@ class Result: 'Unknown result type {}'.format(d['type'])) def __str__(self): - return json.dumps(self.to_dict()) + return json.dumps(self.to_dict(), cls=CustomEncoder) class ResultError(Result):

1 0

[sbws/maint-1.1] chg: json: Create custom JSON encoder/decoder
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit f19738f1c24316399c0928f62130ec133cee5db0 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 16:52:21 2020 +0000 chg: json: Create custom JSON encoder/decoder to be able to serialize/deserailize datetime in the state file. --- sbws/util/json.py | 44 ++++++++++++++++++++++++++++++++++++ tests/unit/util/test_json.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/sbws/util/json.py b/sbws/util/json.py new file mode 100644 index 0000000..f4f8130 --- /dev/null +++ b/sbws/util/json.py @@ -0,0 +1,44 @@ +"""JSON custom serializers and deserializers.""" +import datetime +import json + +from .timestamps import DateTimeSeq, DateTimeIntSeq + + +class CustomEncoder(json.JSONEncoder): + """JSONEncoder that serializes datetime to ISO 8601 string.""" + + def default(self, obj): + if isinstance(obj, DateTimeSeq) or isinstance(obj, DateTimeIntSeq): + return [self.default(i) for i in obj.list()] + if isinstance(obj, datetime.datetime): + return obj.replace(microsecond=0).isoformat() + else: + return super().default(obj) + + +class CustomDecoder(json.JSONDecoder): + """JSONDecoder that deserializes ISO 8601 string to datetime.""" + + def decode(self, s, **kwargs): + decoded = super().decode(s, **kwargs) + return self.process(decoded) + + def process(self, obj): + if isinstance(obj, list) and obj: + return [self.process(item) for item in obj] + if isinstance(obj, dict): + return {key: self.process(value) for key, value in obj.items()} + if isinstance(obj, str): + try: + return datetime.datetime.strptime(obj, "%Y-%m-%dT%H:%M:%S") + except ValueError: + try: + datetime.datetime.strptime( + obj, "%Y-%m-%dT%H:%M:%S.%f" + ).replace(microsecond=0) + except ValueError: + pass + except TypeError: + pass + return obj diff --git a/tests/unit/util/test_json.py b/tests/unit/util/test_json.py new file mode 100644 index 0000000..05e0bf3 --- /dev/null +++ b/tests/unit/util/test_json.py @@ -0,0 +1,54 @@ +"""json.py unit tests.""" +import json + +from sbws.util.json import CustomDecoder, CustomEncoder + +STATE = """{ + "min_perc_reached": null, + "recent_consensus_count": [ + "2020-03-04T10:00:00", + "2020-03-05T10:00:00", + "2020-03-06T10:00:00" + ], + "recent_measurement_attempt": [ + [ + "2020-03-04T10:00:00", + 2 + ], + [ + "2020-03-05T10:00:00", + 2 + ], + [ + "2020-03-06T10:00:00", + 2 + ] + ], + "recent_priority_list": [ + "2020-03-04T10:00:00", + "2020-03-05T10:00:00", + "2020-03-06T10:00:00" + ], + "recent_priority_relay": [ + [ + "2020-03-04T10:00:00", + 2 + ], + [ + "2020-03-05T10:00:00", + 2 + ], + [ + "2020-03-06T10:00:00", + 2 + ] + ], + "scanner_started": "2020-03-14T16:15:22", + "uuid": "x" +}""" + + +def test_decode_encode_roundtrip(): + d = json.loads(STATE, cls=CustomDecoder) + s = json.dumps(d, cls=CustomEncoder, indent=4, sort_keys=True) + assert s == STATE

1 0