March 2020 - tor-commits - lists.torproject.org

[sbws/master] fix: tests: Add relaylist test
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 6c8fc408ac3e648d3da1da1bb60ebb6026f9dd4e Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 10:17:21 2020 +0000 fix: tests: Add relaylist test Tests don't pass in this commit, they're fixed in the next commits. --- sbws/util/timestamp.py | 2 +- setup.py | 2 +- tests/unit/lib/test_relaylist.py | 51 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/sbws/util/timestamp.py b/sbws/util/timestamp.py index 075e82b..2b056f2 100644 --- a/sbws/util/timestamp.py +++ b/sbws/util/timestamp.py @@ -86,7 +86,7 @@ def is_old(timestamp, measurements_period=MEASUREMENTS_PERIOD): # This will raise an exception if the string is not correctly # formatted. timestamp = isostr_to_dt_obj(timestamp) - else: + elif isinstance(timestamp, int) or isinstance(timestamp, float): # This will raise an exception if the type is not int or float or # is not actually a timestamp timestamp = unixts_to_dt_obj(timestamp) diff --git a/setup.py b/setup.py index f567327..f328d7a 100755 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ setup( extras_require={ # vulture: find unused code 'dev': ['flake8', 'vulture'], - 'test': ['tox', 'pytest', 'coverage'], + 'test': ['tox', 'pytest', 'coverage', 'freezegun'], # recommonmark: to make sphinx render markdown 'doc': ['sphinx', 'recommonmark', 'pylint'], }, diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 3dec382..9c3b4d6 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta # freezegun is able to mock any datetime object, it also allows comparations. from freezegun import freeze_time -from sbws.lib.relaylist import remove_old_consensus_timestamps +from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps def test_remove_old_consensus_timestamps(): @@ -21,3 +21,52 @@ def test_remove_old_consensus_timestamps(): ) assert len(new_timestamps) == len(timestamps) - 1 assert days_ago not in new_timestamps + + +def test_init_relays( + args, conf, controller, controller_1h_later, controller_5days_later +): + """ + Test `init_relays` when creating the RelayList the first time and when a + new consensus is received. + Test that the number of consesus timesamps and relays is correct. + """ + # There is no need to mock datetime to update the consensus, since the + # actual date will be always later. + # But it's needed to have the correct list of timestamps both for RelayList + # and Relay. + with freeze_time("2020-02-29 10:00:00"): + relay_list = RelayList(args, conf, controller=controller) + assert len(relay_list._consensus_timestamps) == 1 + # The actual number of relays in the consensus + assert len(relay_list._relays) == 6433 + fps = {r.fingerprint for r in relay_list._relays} + + # One hour later there is a new consensus + relay_list._controller = controller_1h_later + with freeze_time("2020-02-29 11:00:00"): + # Call relays update the list of relays. + relay_list.relays + assert len(relay_list._consensus_timestamps) == 2 + # Check that the number of relays is now the previous one plus the relays + # that are in the new consensus that there were not in the previous one. + fps_1h_later = {r.fingerprint for r in relay_list._relays} + added_fps = fps_1h_later.difference(fps) + assert 6505 == 6433 + len(added_fps) + + # Five days later plus 1 second. + # The first consensus timestamp will get removed. + relay_list._controller = controller_5days_later + with freeze_time("2020-03-05 10:00:01"): + relay_list.relays + assert len(relay_list._consensus_timestamps) == 2 + fps_5days_later = {r.fingerprint for r in relay_list._relays} + # The number of added relays will be the number of relays in this + # consensus that were not in the other 2 conensuses + added_fps = fps_5days_later.difference(fps_1h_later) + # The number of removed relays that are in this consensus, plus the added + # ones that were not in the first consensus (because it has been removed). + removed_fps = fps.difference(fps_5days_later) + # The number of relays will be the number of relays in the cosensus plus + # the added ones minus the removed ones. + assert 6925 == 6505 + len(added_fps) - len(removed_fps) \ No newline at end of file

1 0

[sbws/master] fix: tests: Test the number of consensus in Relay
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit bbb59b1eb70ff54b77747831dd629cc222240f77 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 18:08:33 2020 +0000 fix: tests: Test the number of consensus in Relay This test does not pass in this commit, but in the next bugfix. --- tests/unit/lib/test_relaylist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py index 9c3b4d6..c4c6e71 100644 --- a/tests/unit/lib/test_relaylist.py +++ b/tests/unit/lib/test_relaylist.py @@ -38,6 +38,7 @@ def test_init_relays( with freeze_time("2020-02-29 10:00:00"): relay_list = RelayList(args, conf, controller=controller) assert len(relay_list._consensus_timestamps) == 1 + assert len(relay_list._relays[0]._consensus_timestamps) == 1 # The actual number of relays in the consensus assert len(relay_list._relays) == 6433 fps = {r.fingerprint for r in relay_list._relays} @@ -48,6 +49,7 @@ def test_init_relays( # Call relays update the list of relays. relay_list.relays assert len(relay_list._consensus_timestamps) == 2 + assert len(relay_list._relays[0]._consensus_timestamps) == 2 # Check that the number of relays is now the previous one plus the relays # that are in the new consensus that there were not in the previous one. fps_1h_later = {r.fingerprint for r in relay_list._relays} @@ -60,6 +62,7 @@ def test_init_relays( with freeze_time("2020-03-05 10:00:01"): relay_list.relays assert len(relay_list._consensus_timestamps) == 2 + assert len(relay_list._relays[0]._consensus_timestamps) == 2 fps_5days_later = {r.fingerprint for r in relay_list._relays} # The number of added relays will be the number of relays in this # consensus that were not in the other 2 conensuses

1 0

[sbws/master] fix: relaylist: Use seconds removing consensuses
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 77832ace14276b7064e7da8099e4efad8a542145 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 10:23:20 2020 +0000 fix: relaylist: Use seconds removing consensuses by default days is passed to timedelta, what was making the oldest date be thousands of days in the past. Bugfix 1.1.0. --- sbws/lib/relaylist.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 64a53d9..722896e 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -24,7 +24,7 @@ def remove_old_consensus_timestamps( :param int measurements_period: :returns list: a new list of ``consensus_timestamps`` """ - oldest_date = datetime.utcnow() - timedelta(measurements_period) + oldest_date = datetime.utcnow() - timedelta(seconds=measurements_period) new_consensus_timestamps = \ [t for t in consensus_timestamps if t >= oldest_date] return new_consensus_timestamps @@ -483,11 +483,12 @@ class RelayList: r = Relay(ns.fingerprint, c, ns=ns, timestamp=timestamp) new_relays.append(r) + days = self._measurements_period / (60 * 60 * 24) log.debug("Previous number of relays being measured %d", len(self._relays)) log.debug("Number of relays not in the in the consensus in the last " "%d days: %d.", - self._measurements_period, num_old_relays) + days, num_old_relays) log.debug("Number of relays to measure with the current consensus: " "%d", len(new_relays)) return new_relays

1 0

[sbws/master] fix: relaylist: Use is_old fn removing consensus
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 2763a868b88ef02f28f8d2056f12c9a67e54ecd7 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 10:36:30 2020 +0000 fix: relaylist: Use is_old fn removing consensus since the logic is the same and the there were two bugfixes on the same logic. --- sbws/lib/relaylist.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 722896e..aaec6ea 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -24,9 +24,11 @@ def remove_old_consensus_timestamps( :param int measurements_period: :returns list: a new list of ``consensus_timestamps`` """ - oldest_date = datetime.utcnow() - timedelta(seconds=measurements_period) - new_consensus_timestamps = \ - [t for t in consensus_timestamps if t >= oldest_date] + new_consensus_timestamps = [ + t + for t in consensus_timestamps + if not timestamp.is_old(t, measurements_period) + ] return new_consensus_timestamps

1 0

[sbws/master] fix: relaylist: Update relay status before consensus
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 51eb50046b2b9b8a5eba15f3c0fd4c93c97e1e77 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 18:11:51 2020 +0000 fix: relaylist: Update relay status before consensus Update relay status before updating the consensus timestamps Timestamps that are not old yet were getting removed because the document.valid_after timestamp was still the one from the previous consensus. Closes #33570. --- sbws/lib/relaylist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index aaec6ea..9ffff49 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -456,9 +456,9 @@ class RelayList: # If a relay in the previous consensus and is in the current # one, update its timestamp, router status and descriptor. fp = r.fingerprint - r.update_consensus_timestamps(timestamp) # new_relays_dict[fp] is the router status. r.update_router_status(new_relays_dict[fp]) + r.update_consensus_timestamps(timestamp) try: descriptor = c.get_server_descriptor(fp, default=None) except (DescriptorUnavailable, ControllerError) as e:

1 0

[sbws/master] fix: timestamp: Old timestamps are minor than older
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit d7b5d3deef1c303461f623d32eaf76782f2910eb Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 18:33:47 2020 +0000 fix: timestamp: Old timestamps are minor than older Old timestamps are minor than the older date, not major. --- sbws/util/timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/util/timestamp.py b/sbws/util/timestamp.py index 4c3fb34..ee0755a 100644 --- a/sbws/util/timestamp.py +++ b/sbws/util/timestamp.py @@ -91,4 +91,4 @@ def is_old(timestamp, measurements_period=MEASUREMENTS_PERIOD): # is not actually a timestamp timestamp = unixts_to_dt_obj(timestamp) oldest_date = datetime.utcnow() - timedelta(measurements_period) - return timestamp > oldest_date + return timestamp < oldest_date

1 0

[sbws/master] fix: relaylist: Stop passing argument to self.is_old
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 380a80624ebd1dda9066cd5a4d8dd7c26c14e196 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 18:28:48 2020 +0000 fix: relaylist: Stop passing argument to self.is_old --- sbws/lib/relaylist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 65ec526..64a53d9 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -471,7 +471,7 @@ class RelayList: # If the relay is not in the current consensus but is not "old" # yet, add it to the new list of relays too, though its timestamp, # router status and descriptor can't be updated. - elif not r.is_old(self._measurements_period): + elif not r.is_old(): new_relays.append(r) # Otherwise, don't add it to the new list of relays. # For debugging, count the old relays that will be discarded.

1 0

[sbws/master] fix: tests: Add test for remove old consensus ts
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 6aaa438e4c6e623afda7d0251b43cb48a8bc7a58 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 8 16:36:44 2020 +0000 fix: tests: Add test for remove old consensus ts Tests don't pass in this commit, it's fixed in the next commits. --- tests/unit/lib/test_relaylist.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py new file mode 100644 index 0000000..3dec382 --- /dev/null +++ b/tests/unit/lib/test_relaylist.py @@ -0,0 +1,23 @@ +"""relaylist.py unit tests.""" +from datetime import datetime, timedelta + +# When datetime is imported as a class (`from datetime import datetime`) it can +# not be mocked because it is a built-in type. It can only be mocked when +# imported as module. +# freezegun is able to mock any datetime object, it also allows comparations. +from freezegun import freeze_time + +from sbws.lib.relaylist import remove_old_consensus_timestamps + + +def test_remove_old_consensus_timestamps(): + days_ago = datetime(2020, 3, 1) + timestamps = [days_ago] + [ + days_ago + timedelta(days=x) for x in range(1, 5) + ] + with freeze_time(days_ago + timedelta(days=5, seconds=1)): + new_timestamps = remove_old_consensus_timestamps( + timestamps, 5 * 24 * 60 * 60 + ) + assert len(new_timestamps) == len(timestamps) - 1 + assert days_ago not in new_timestamps

1 0

[sbws/master] fix: tests: Add test timestamp.is_old
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit feae0a57e7ec14b1904d56d2f6081605a996f086 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 18:36:42 2020 +0000 fix: tests: Add test timestamp.is_old The tests don't pass in this commit, it's fixed in the next ones. --- tests/unit/util/test_timestamp.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/util/test_timestamp.py b/tests/unit/util/test_timestamp.py index 8ebc2f6..37cc9ea 100644 --- a/tests/unit/util/test_timestamp.py +++ b/tests/unit/util/test_timestamp.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- """Test timestamp conversion util functions""" -from datetime import datetime, timezone - +from datetime import datetime, timezone, timedelta from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, - unixts_to_isodt_str, unixts_to_str) + unixts_to_isodt_str, unixts_to_str, is_old) isodt_str = '2018-05-23T12:55:04' @@ -25,3 +24,14 @@ def test_unixts_to_isodt_str(): def test_unixts_to_str(): assert str(unixts) == unixts_to_str(unixts) + + +def test_is_old(): + # Since this timestamp is generated a few microseconds before checking + # the oldest timestamp, it will be old. + old_timestamp = datetime.utcnow() - timedelta(days=5) + assert is_old(old_timestamp) + # A recent timestamp should be at least 1 second newer that the oldest + recent_timestamp = datetime.utcnow() - timedelta(days=5) \ + + timedelta(seconds=1) + assert not is_old(recent_timestamp)

1 0

[sbws/master] fix: timestamp: measurements period is in seconds
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 3ec1a8e6658c3766a3dad15000416569428fa17b Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 18:31:41 2020 +0000 fix: timestamp: measurements period is in seconds by default days is passed to timedelta, what was making the oldest date be thousands of days in the past. --- sbws/util/timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/util/timestamp.py b/sbws/util/timestamp.py index ee0755a..075e82b 100644 --- a/sbws/util/timestamp.py +++ b/sbws/util/timestamp.py @@ -90,5 +90,5 @@ def is_old(timestamp, measurements_period=MEASUREMENTS_PERIOD): # This will raise an exception if the type is not int or float or # is not actually a timestamp timestamp = unixts_to_dt_obj(timestamp) - oldest_date = datetime.utcnow() - timedelta(measurements_period) + oldest_date = datetime.utcnow() - timedelta(seconds=measurements_period) return timestamp < oldest_date

1 0