April 2020 - tor-commits - lists.torproject.org

[tor-browser-build/master] Bug 33877: Disable Samples and Regression tests For Libevent Build
by boklm＠torproject.org 14 Apr '20

14 Apr '20

commit f4792f507d57583fce4f52afcb927076762653a1 Author: sisbell <shane.isbell(a)gmail.com> Date: Sun Mar 1 16:08:28 2020 -0800 Bug 33877: Disable Samples and Regression tests For Libevent Build --- projects/libevent/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/libevent/build b/projects/libevent/build index 1d620d9..fe15e8e 100644 --- a/projects/libevent/build +++ b/projects/libevent/build @@ -11,7 +11,7 @@ cd /var/tmp/build/[% project %]-[% c('version') %] export MACOSX_DEPLOYMENT_TARGET=[% c("var/macosx_deployment_target") %] [% END -%] ./autogen.sh -./configure --disable-static [% c("var/configure_opt") %] --prefix=$distdir +./configure --disable-static [% c("var/configure_opt") %] --disable-libevent-regress --disable-samples --prefix=$distdir [% IF c("var/osx") -%] export LD_PRELOAD=[% c("var/faketime_path") %] export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"

1 0

[tor-browser-build/master] Bug 28765: LibEvent Build for Android
by boklm＠torproject.org 14 Apr '20

14 Apr '20

commit 06bb0f2d4c91d7ac60ae180297d3c0707a004199 Author: sisbell <shane.isbell(a)gmail.com> Date: Sun Mar 1 16:08:28 2020 -0800 Bug 28765: LibEvent Build for Android --- projects/libevent/build | 2 +- projects/libevent/config | 4 ++++ rbm.conf | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/projects/libevent/build b/projects/libevent/build index fe15e8e..9c4f190 100644 --- a/projects/libevent/build +++ b/projects/libevent/build @@ -11,7 +11,7 @@ cd /var/tmp/build/[% project %]-[% c('version') %] export MACOSX_DEPLOYMENT_TARGET=[% c("var/macosx_deployment_target") %] [% END -%] ./autogen.sh -./configure --disable-static [% c("var/configure_opt") %] --disable-libevent-regress --disable-samples --prefix=$distdir +./configure [% c("var/configure_opt") %] --disable-libevent-regress --disable-samples --prefix=$distdir [% IF c("var/osx") -%] export LD_PRELOAD=[% c("var/faketime_path") %] export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" diff --git a/projects/libevent/config b/projects/libevent/config index fa01c1c..2e5dee6 100644 --- a/projects/libevent/config +++ b/projects/libevent/config @@ -9,12 +9,16 @@ filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/buil var: container: use_container: 1 + configure_opt_project: --disable-static targets: osx-x86_64: var: arch_deps: - faketime + android: + var: + configure_opt_project: --disable-shared input_files: - project: container-image diff --git a/rbm.conf b/rbm.conf index a988c89..cd17da9 100644 --- a/rbm.conf +++ b/rbm.conf @@ -289,12 +289,12 @@ targets: var: linux-i686: 1 osname: linux-i686 - configure_opt_i686: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32' - configure_opt: '[% c("var/configure_opt_i686") %]' + configure_opt: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 [% c("var/configure_opt_project") %]' linux: var: linux: 1 compiler: gcc + configure_opt: '[% c("var/configure_opt_project") %]' # We only build snowflake on the alpha and nightly # channels for now. snowflake: '[% c("var/alpha") || c("var/nightly") %]' @@ -343,7 +343,7 @@ targets: container: suite: stretch arch: amd64 - configure_opt: '--host=[% c("arch") %]-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]"' + configure_opt: '--host=[% c("arch") %]-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]" [% c("var/configure_opt_project") %]' CFLAGS: '-fstack-protector-strong -fno-strict-overflow -Wno-missing-field-initializers -Wformat -Wformat-security [% c("var/flag_mwindows") %]' LDFLAGS: '-Wl,--dynamicbase -Wl,--nxcompat -Wl,--enable-reloc-section -Wl,--no-insert-timestamp -lssp -L$gcclibs [% c("var/flag_HEASLR") %] [% c("var/flag_mwindows") %]' flag_mwindows: '-mwindows' @@ -372,7 +372,7 @@ targets: suite: stretch arch: amd64 compiler: 'macosx-toolchain' - configure_opt: '--host=x86_64-apple-darwin11 CC="x86_64-apple-darwin11-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin11-clang++ [% c("var/FLAGS") %]"' + configure_opt: '--host=x86_64-apple-darwin11 CC="x86_64-apple-darwin11-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin11-clang++ [% c("var/FLAGS") %]" [% c("var/configure_opt_project") %]' FLAGS: "-target x86_64-apple-darwin11 -B $cctoolsdir -isysroot $sysrootdir" LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie" macosx_deployment_target: '10.9'

1 0

[sbws/master] chg: state: Add method to count list values
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 9acad379acffa9c167529e7e12a2c445208cd60d Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 18:24:50 2020 +0000 chg: state: Add method to count list values --- sbws/util/state.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sbws/util/state.py b/sbws/util/state.py index 86cc59b..50e6a66 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -90,3 +90,18 @@ class State: def __contains__(self, item): self._state = self._read() return self._state.__contains__(item) + + def count(self, k): + """ + Returns the length if the key value is a list + or the sum of number if the key value is a list of list + or the key value + or None if the state doesn't have the key. + """ + if self.get(k): + if isinstance(self._state[k], list): + if isinstance(self._state[k][0], list): + return sum(map(lambda x: x[1], self._state[k])) + return len(self._state[k]) + return self.get(k) + return None

1 0

[sbws/master] fix: state: Let json manage data types
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 26409a3d8187cc65147a25484655a7639bdb705e Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 17:27:59 2020 +0000 fix: state: Let json manage data types Since state uses json and json will raise an error when it can't decode/encode some datatype. --- sbws/util/state.py | 34 +++++++--------------------------- tests/unit/util/test_state.py | 43 ------------------------------------------- 2 files changed, 7 insertions(+), 70 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index f460413..86cc59b 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -4,16 +4,12 @@ import json class State: - ''' - State allows one to atomically access and update a simple state file on - disk across threads and across processes. + """ + `json` wrapper to read a json file every time it gets a key and to write + to the file every time a key is set. - To put it blunty, due to limited developer time and his inability to - quickly find a way to safely access and update more complex data types - (namely, collections like list, set, and dict), you may only store simple - types of data as enumerated in _ALLOWED_TYPES. Keys must be strings. - - Data is stored as JSON on disk in the provided file file. + Every time a key is got or set, the file is locked, to atomically access + and update the file across threads and across processes. >>> state = State('foo.state') >>> # state == {} @@ -40,8 +36,8 @@ class State: >>> # We can do many of the same things with a State object as with a dict >>> for key in state: print(key) >>> # Prints 'linux', 'age', and 'name' - ''' - _ALLOWED_TYPES = (int, float, str, bool, type(None)) + + """ def __init__(self, fname): self._fname = fname @@ -68,35 +64,19 @@ class State: Implements a dictionary ``get`` method reading and locking a json file. """ - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() return self._state.get(key, d) def __getitem__(self, key): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() return self._state.__getitem__(key) def __delitem__(self, key): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) self._state = self._read() self._state.__delitem__(key) self._write() def __setitem__(self, key, value): - if not isinstance(key, str): - raise TypeError( - 'Keys must be strings. %s is a %s' % (key, type(key))) - if type(value) not in State._ALLOWED_TYPES: - raise TypeError( - 'May only store value with type in %s, not %s' % - (State._ALLOWED_TYPES, type(value))) # NOTE: important, read the file before setting the key, # otherwise if other instances are creating other keys, they're lost. self._state = self._read() diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index 2443049..a57768d 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -11,26 +11,6 @@ def test_state_set_allowed_key_types(tmpdir): assert state[key] == 4 -def test_state_set_bad_key_types(tmpdir): - state = State(os.path.join(str(tmpdir), 'statefoo')) - attempt_keys = (15983, None, True, -1.2, [], {}, set()) - for key in attempt_keys: - try: - state[key] = 4 - except TypeError: - pass - else: - assert None, 'Should not have been able to use %s %s as a key' %\ - (key, type(key)) - try: - state[key] - except TypeError: - pass - else: - assert None, '%s %s is not a valid key type, so should have got '\ - 'TypeError when giving it' % (key, type(key)) - - def test_state_set_allowed_value_types(tmpdir): state = State(os.path.join(str(tmpdir), 'statefoo')) attempt_vals = (15983, None, True, -1.2, 'loooooool') @@ -39,19 +19,6 @@ def test_state_set_allowed_value_types(tmpdir): assert state['foo'] == val -def test_state_set_bad_value_types(tmpdir): - state = State(os.path.join(str(tmpdir), 'statefoo')) - attempt_vals = ([], {}, set()) - for val in attempt_vals: - try: - state['foo'] = val - except TypeError: - pass - else: - assert None, 'Should not have been able to use %s %s as a value' %\ - (val, type(val)) - - def test_state_del(tmpdir): state = State(os.path.join(str(tmpdir), 'statefoo')) d = {'a': 1, 'b': 2, 'c': 3, 'd': 4} @@ -65,16 +32,6 @@ def test_state_del(tmpdir): for key in d: assert d[key] == state[key] - attempt_keys = (15983, None, True, -1.2, [], {}, set()) - for key in attempt_keys: - try: - del state[key] - except TypeError: - pass - else: - assert None, 'Should not have been allowed to delete %s %s '\ - 'because it is not a valid key type' % (key, type(key)) - d['e'] = 5 state['e'] = 5 d['e'] = 5.5

1 0

[sbws/master] chg: timestamps: Add module to manage datetime sequences
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 003337ee7d241e18637ad5d67099d06cb0b4e9e8 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 18:27:44 2020 +0000 chg: timestamps: Add module to manage datetime sequences --- sbws/util/timestamps.py | 89 ++++++++++++++++++++++++++++++++++++++ tests/unit/util/test_timestamps.py | 44 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/sbws/util/timestamps.py b/sbws/util/timestamps.py new file mode 100644 index 0000000..393b30b --- /dev/null +++ b/sbws/util/timestamps.py @@ -0,0 +1,89 @@ +"""Util classes to manipulate sequences of datetime timestamps. + +Optionally update also a state file. + +""" +# Workarounds to store datetimes for objects because they are not compossed +# by other objects nor stored in a database with a creation datetime. +import collections +from datetime import datetime, timedelta +import logging + +from sbws.util.timestamp import is_old + +log = logging.getLogger(__name__) + + +class DateTimeSeq(collections.deque): + """Store and manage a datetime sequence and optionally a state file.""" + + def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): + self._maxlen = maxlen + self._items = collections.deque(iterable, maxlen) + self._state = state + self._state_key = state_key + + def _remove_old(self): + self._items = collections.deque( + filter(lambda x: not is_old(x), self._items), maxlen=self._maxlen + ) + + def update(self, dt=None): + self._remove_old() + self._items.append(dt or datetime.utcnow().replace(microsecond=0)) + if self._state is not None and self._state_key: + self._state[self._state_key] = list(self._items) + return list(self._items) + + def last(self): + if len(self._items) > 0: + return self._items[-1] + return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) + + def list(self): + return list(self._items) + + def __len__(self): + return len(self._items) + + +class DateTimeIntSeq(collections.deque): + """ + Store and manage a sequence of lists composed of a datetime and an int. + + Optionally store and manage an state file. + """ + + def __init__(self, iterable=[], maxlen=None, state=None, state_key=None): + self._maxlen = maxlen + self._items = collections.deque(iterable, maxlen) + self._state = state + self._state_key = state_key + + def _remove_old(self): + self._items = collections.deque( + filter(lambda x: not is_old(x[0]), self._items), + maxlen=self._maxlen, + ) + + def update(self, dt=None, number=0): + self._remove_old() + # Because json serializes tuples to lists, use list instead of tuple + # to facilitate comparisons. + self._items.append( + [dt or datetime.utcnow().replace(microsecond=0), number] + ) + if self._state is not None and self._state_key: + self._state[self._state_key] = list(self._items) + return list(self._items) + + def last(self): + if len(self._items) > 0: + return self._items[-1] + return datetime.utcnow().replace(microsecond=0) - timedelta(hour=1) + + def list(self): + return list(self._items) + + def __len__(self): + return sum(map(lambda x: x[1], self._items)) diff --git a/tests/unit/util/test_timestamps.py b/tests/unit/util/test_timestamps.py new file mode 100644 index 0000000..aef0501 --- /dev/null +++ b/tests/unit/util/test_timestamps.py @@ -0,0 +1,44 @@ +"""timestamps.py unit tests.""" + +from datetime import datetime, timedelta + +from sbws.util.state import State +from sbws.util.timestamps import ( + DateTimeSeq, + DateTimeIntSeq, +) + + +def test_update_datetime_seq(conf): + now = datetime.utcnow().replace(microsecond=0) + state = State(conf["paths"]["state_fpath"]) + # Create a list of 6 datetimes that started 6 days in the past. + dts = [now - timedelta(days=x) for x in range(6, 0, -1)] + dt_seq = DateTimeSeq( + dts, state=state, state_key="recent_measurement_attempt" + ) + new_dts = dt_seq.update() + # The updated list will not contain the 2 first (left) datetimes and it + # will have one last timestamp (right). + assert new_dts[:-1] == dts[2:] + assert 5 == state.count("recent_measurement_attempt") + assert 5 == len(dt_seq) + + +def test_update_datetime_int_seq(conf): + now = datetime.utcnow().replace(microsecond=0) + state = State(conf["paths"]["state_fpath"]) + # Create a list of 6 datetimes that started 6 days in the past. + dts = [[now - timedelta(days=x), 2] for x in range(6, 0, -1)] + dt_seq = DateTimeIntSeq( + dts, state=state, state_key="recent_measurement_attempt" + ) + new_dts = dt_seq.update() + # The updated list will not contain the 2 first (left) tuples and it + # will have one last tuple (right). + # The last tuple have 0 as the integer, instead of 2, so the count will be + # 2 * 4 = 8 + assert new_dts[:-1] == dts[2:] + assert 8 == state.count("recent_measurement_attempt") + # And `len` should return the same. + assert 8 == len(dt_seq)

1 0

[sbws/master] chg: json: Create custom JSON encoder/decoder
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit f19738f1c24316399c0928f62130ec133cee5db0 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 14 16:52:21 2020 +0000 chg: json: Create custom JSON encoder/decoder to be able to serialize/deserailize datetime in the state file. --- sbws/util/json.py | 44 ++++++++++++++++++++++++++++++++++++ tests/unit/util/test_json.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/sbws/util/json.py b/sbws/util/json.py new file mode 100644 index 0000000..f4f8130 --- /dev/null +++ b/sbws/util/json.py @@ -0,0 +1,44 @@ +"""JSON custom serializers and deserializers.""" +import datetime +import json + +from .timestamps import DateTimeSeq, DateTimeIntSeq + + +class CustomEncoder(json.JSONEncoder): + """JSONEncoder that serializes datetime to ISO 8601 string.""" + + def default(self, obj): + if isinstance(obj, DateTimeSeq) or isinstance(obj, DateTimeIntSeq): + return [self.default(i) for i in obj.list()] + if isinstance(obj, datetime.datetime): + return obj.replace(microsecond=0).isoformat() + else: + return super().default(obj) + + +class CustomDecoder(json.JSONDecoder): + """JSONDecoder that deserializes ISO 8601 string to datetime.""" + + def decode(self, s, **kwargs): + decoded = super().decode(s, **kwargs) + return self.process(decoded) + + def process(self, obj): + if isinstance(obj, list) and obj: + return [self.process(item) for item in obj] + if isinstance(obj, dict): + return {key: self.process(value) for key, value in obj.items()} + if isinstance(obj, str): + try: + return datetime.datetime.strptime(obj, "%Y-%m-%dT%H:%M:%S") + except ValueError: + try: + datetime.datetime.strptime( + obj, "%Y-%m-%dT%H:%M:%S.%f" + ).replace(microsecond=0) + except ValueError: + pass + except TypeError: + pass + return obj diff --git a/tests/unit/util/test_json.py b/tests/unit/util/test_json.py new file mode 100644 index 0000000..05e0bf3 --- /dev/null +++ b/tests/unit/util/test_json.py @@ -0,0 +1,54 @@ +"""json.py unit tests.""" +import json + +from sbws.util.json import CustomDecoder, CustomEncoder + +STATE = """{ + "min_perc_reached": null, + "recent_consensus_count": [ + "2020-03-04T10:00:00", + "2020-03-05T10:00:00", + "2020-03-06T10:00:00" + ], + "recent_measurement_attempt": [ + [ + "2020-03-04T10:00:00", + 2 + ], + [ + "2020-03-05T10:00:00", + 2 + ], + [ + "2020-03-06T10:00:00", + 2 + ] + ], + "recent_priority_list": [ + "2020-03-04T10:00:00", + "2020-03-05T10:00:00", + "2020-03-06T10:00:00" + ], + "recent_priority_relay": [ + [ + "2020-03-04T10:00:00", + 2 + ], + [ + "2020-03-05T10:00:00", + 2 + ], + [ + "2020-03-06T10:00:00", + 2 + ] + ], + "scanner_started": "2020-03-14T16:15:22", + "uuid": "x" +}""" + + +def test_decode_encode_roundtrip(): + d = json.loads(STATE, cls=CustomDecoder) + s = json.dumps(d, cls=CustomEncoder, indent=4, sort_keys=True) + assert s == STATE

1 0

[sbws/master] chg: state: Encode/decode datetimes
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit af849a06a7183b06aa1ab7ed478ed862c86b6566 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 13:01:50 2020 +0000 chg: state: Encode/decode datetimes --- sbws/util/state.py | 6 ++++-- tests/unit/util/test_state.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index 50e6a66..b6614a7 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -2,6 +2,8 @@ from sbws.util.filelock import FileLock import os import json +from .json import CustomDecoder, CustomEncoder + class State: """ @@ -48,12 +50,12 @@ class State: return {} with FileLock(self._fname): with open(self._fname, 'rt') as fd: - return json.load(fd) + return json.load(fd, cls=CustomDecoder) def _write(self): with FileLock(self._fname): with open(self._fname, 'wt') as fd: - return json.dump(self._state, fd, indent=4) + return json.dump(self._state, fd, indent=4, cls=CustomEncoder) def __len__(self): self._state = self._read() diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index a57768d..d08a4a9 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -83,3 +83,11 @@ def test_two_instances(tmpdir): s1["x"] = "foo" s2["y"] = "bar" assert s2["x"] == "foo" + + +def test_datetime_values(tmpdir): + import datetime + state = State(os.path.join(str(tmpdir), 'state.dat')) + now = datetime.datetime.utcnow().replace(microsecond=0) + state["datetimes"] = now + assert now == state["datetimes"]

1 0

[sbws/master] chg: resultdump: Use custom json encoder/decoder
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 8e5da39c171bc84c5ae37786d7dad198957a0d11 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 13:11:46 2020 +0000 chg: resultdump: Use custom json encoder/decoder --- sbws/lib/resultdump.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index f7ec55b..02952f9 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -12,6 +12,7 @@ from datetime import timedelta from enum import Enum from sbws.globals import RESULT_VERSION, fail_hard from sbws.util.filelock import DirectoryLock +from sbws.util.json import CustomEncoder, CustomDecoder from sbws.lib.relaylist import Relay from .. import settings @@ -45,7 +46,9 @@ def load_result_file(fname, success_only=False): for line in fd: num_total += 1 try: - r = Result.from_dict(json.loads(line.strip())) + r = Result.from_dict( + json.loads(line.strip(), cls=CustomDecoder) + ) except json.decoder.JSONDecodeError: log.warning('Could not decode result %s', line.strip()) r = None @@ -416,7 +419,7 @@ class Result: 'Unknown result type {}'.format(d['type'])) def __str__(self): - return json.dumps(self.to_dict()) + return json.dumps(self.to_dict(), cls=CustomEncoder) class ResultError(Result):

1 0

[sbws/master] chg: v3bwfile: Convert datetime to str
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit 65589118cd18500722acee1fd26e9b2a76083033 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 22 08:04:48 2020 +0000 chg: v3bwfile: Convert datetime to str --- sbws/lib/v3bwfile.py | 6 ++++-- tests/unit/lib/test_v3bwfile.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 2fd7ce4..80f1e34 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -18,7 +18,8 @@ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, - now_unixts, isostr_to_dt_obj) + now_unixts, isostr_to_dt_obj, + dt_obj_to_isodt_str) from sbws.util.state import State log = logging.getLogger(__name__) @@ -448,7 +449,8 @@ class V3BWHeader(object): ''' state = State(state_fpath) if 'scanner_started' in state: - return state['scanner_started'] + # From v1.1.0-dev `state` is capable of converting strs to datetime + return dt_obj_to_isodt_str(state['scanner_started']) else: return None diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 566cc8d..8fe2dab 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -516,3 +516,16 @@ def test_set_under_min_report(mock_consensus, conf, datadir): assert bwl.vote == 0 assert bwl.under_min_report == 1 assert bwl.bw != 1 + + +def test_generator_started(root_data_path, datadir): + state_fpath = os.path.join(root_data_path, '.sbws/state.dat') + # The method is correct + assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file( + state_fpath + ) + # `results` does not matter here, using them to not have an empty list. + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BWHeader.from_results(results, '', '', state_fpath) + # And the header is correct + assert "2019-03-25T13:03:06" == header.generator_started

1 0

[sbws/master] chg: resultdump: Remove `_count` from attributes
by juga＠torproject.org 14 Apr '20

14 Apr '20

commit b1c0f0d0a5d6e1648b8ef024880853b0d9d43d00 Author: juga0 <juga(a)riseup.net> Date: Sat Mar 21 14:59:11 2020 +0000 chg: resultdump: Remove `_count` from attributes Tests wont' pass with this commit, they'll be fixed in the next commits --- sbws/lib/resultdump.py | 113 ++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 02952f9..a7c971f 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -225,9 +225,9 @@ class Result: consensus_bandwidth_is_unmeasured=None, # Counters to be stored by relay and not per measurement, # since the measurements might fail. - relay_in_recent_consensus_count=None, - relay_recent_measurement_attempt_count=None, - relay_recent_priority_list_count=None): + relay_in_recent_consensus=None, + relay_recent_measurement_attempt=None, + relay_recent_priority_list=None): """ Initializes a ``Result.Relay``. @@ -247,15 +247,14 @@ class Result: self.consensus_bandwidth = consensus_bandwidth self.consensus_bandwidth_is_unmeasured = \ consensus_bandwidth_is_unmeasured - self.relay_in_recent_consensus_count = \ - relay_in_recent_consensus_count - self.relay_recent_measurement_attempt_count = \ - relay_recent_measurement_attempt_count - self.relay_recent_priority_list_count = \ - relay_recent_priority_list_count - - def __init__(self, relay, circ, dest_url, scanner_nick, t=None, - relay_in_recent_consensus_count=None): + self.relay_in_recent_consensus = \ + relay_in_recent_consensus + self.relay_recent_measurement_attempt = \ + relay_recent_measurement_attempt + self.relay_recent_priority_list = \ + relay_recent_priority_list + + def __init__(self, relay, circ, dest_url, scanner_nick, t=None): """ Initilizes the measurement and the relay with all the relay attributes. """ @@ -267,9 +266,9 @@ class Result: relay.observed_bandwidth, relay.consensus_bandwidth, relay.consensus_bandwidth_is_unmeasured, - relay.relay_in_recent_consensus_count, - relay.relay_recent_measurement_attempt_count, - relay.relay_recent_priority_list_count + relay.relay_in_recent_consensus, + relay.relay_recent_measurement_attempt, + relay.relay_recent_priority_list ) self._circ = circ self._dest_url = dest_url @@ -317,27 +316,27 @@ class Result: return self._relay.master_key_ed25519 @property - def relay_in_recent_consensus_count(self): + def relay_in_recent_consensus(self): """Number of times the relay was in a consensus.""" - return self._relay.relay_in_recent_consensus_count + return self._relay.relay_in_recent_consensus @property - def relay_recent_measurement_attempt_count(self): + def relay_recent_measurement_attempt(self): """Returns the relay recent measurements attemps. It is initialized in :class:`~sbws.lib.relaylist.Relay` and incremented in :func:`~sbws.core.scanner.main_loop`. """ - return self._relay.relay_recent_measurement_attempt_count + return self._relay.relay_recent_measurement_attempt @property - def relay_recent_priority_list_count(self): + def relay_recent_priority_list(self): """Returns the relay recent "prioritization"s to be measured. It is initialized in :class:`~sbws.lib.relaylist.Relay` and incremented in :func:`~sbws.core.scanner.main_loop`. """ - return self._relay.relay_recent_priority_list_count + return self._relay.relay_recent_priority_list @property def circ(self): @@ -371,12 +370,12 @@ class Result: 'type': self.type, 'scanner': self.scanner, 'version': self.version, - 'relay_in_recent_consensus_count': - self.relay_in_recent_consensus_count, - 'relay_recent_measurement_attempt_count': - self.relay_recent_measurement_attempt_count, - 'relay_recent_priority_list_count': - self.relay_recent_priority_list_count, + 'relay_in_recent_consensus': + self.relay_in_recent_consensus, + 'relay_recent_measurement_attempt': + self.relay_recent_measurement_attempt, + 'relay_recent_priority_list': + self.relay_recent_priority_list, } @staticmethod @@ -459,12 +458,12 @@ class ResultError(Result): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None), # noqa - relay_recent_measurement_attempt_count= # noqa - d.get('relay_recent_measurement_attempt_count', None), # noqa - relay_recent_priority_list_count= # noqa - d.get('relay_recent_priority_list_count', None), # noqa + relay_in_recent_consensus= # noqa + d.get('relay_in_recent_consensus', None), # noqa + relay_recent_measurement_attempt= # noqa + d.get('relay_recent_measurement_attempt', None), # noqa + relay_recent_priority_list= # noqa + d.get('relay_recent_priority_list', None), # noqa ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -507,12 +506,12 @@ class ResultErrorCircuit(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None), # noqa - relay_recent_measurement_attempt_count= # noqa - d.get('relay_recent_measurement_attempt_count', None), # noqa - relay_recent_priority_list_count= # noqa - d.get('relay_recent_priority_list_count', None), # noqa + relay_in_recent_consensus= # noqa + d.get('relay_in_recent_consensus', None), # noqa + relay_recent_measurement_attempt= # noqa + d.get('relay_recent_measurement_attempt', None), # noqa + relay_recent_priority_list= # noqa + d.get('relay_recent_priority_list', None), # noqa ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -537,12 +536,12 @@ class ResultErrorStream(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None), # noqa - relay_recent_measurement_attempt_count= # noqa - d.get('relay_recent_measurement_attempt_count', None), # noqa - relay_recent_priority_list_count= # noqa - d.get('relay_recent_priority_list_count', None), # noqa + relay_in_recent_consensus= # noqa + d.get('relay_in_recent_consensus', None), # noqa + relay_recent_measurement_attempt= # noqa + d.get('relay_recent_measurement_attempt', None), # noqa + relay_recent_priority_list= # noqa + d.get('relay_recent_priority_list', None), # noqa ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -657,12 +656,12 @@ class ResultErrorAuth(ResultError): Result.Relay( d['fingerprint'], d['nickname'], d['address'], d['master_key_ed25519'], - relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None), # noqa - relay_recent_measurement_attempt_count= # noqa - d.get('relay_recent_measurement_attempt_count', None), # noqa - relay_recent_priority_list_count= # noqa - d.get('relay_recent_priority_list_count', None), # noqa + relay_in_recent_consensus= # noqa + d.get('relay_in_recent_consensus', None), # noqa + relay_recent_measurement_attempt= # noqa + d.get('relay_recent_measurement_attempt', None), # noqa + relay_recent_priority_list= # noqa + d.get('relay_recent_priority_list', None), # noqa ), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -701,12 +700,12 @@ class ResultSuccess(Result): d.get('relay_burst_bandwidth'), d['relay_observed_bandwidth'], d.get('consensus_bandwidth'), d.get('consensus_bandwidth_is_unmeasured'), - relay_in_recent_consensus_count= # noqa - d.get('relay_in_recent_consensus_count', None), # noqa - relay_recent_measurement_attempt_count= # noqa - d.get('relay_recent_measurement_attempt_count', None), # noqa - relay_recent_priority_list_count= # noqa - d.get('relay_recent_priority_list_count', None), # noqa + relay_in_recent_consensus= # noqa + d.get('relay_in_recent_consensus', None), # noqa + relay_recent_measurement_attempt= # noqa + d.get('relay_recent_measurement_attempt', None), # noqa + relay_recent_priority_list= # noqa + d.get('relay_recent_priority_list', None), # noqa ), d['circ'], d['dest_url'], d['scanner'], t=d['time'])

1 0