commit f014084108eafd6478fbe5e2e0afc6855435c87f Author: juga0 <juga@riseup.net> Date: Mon Nov 23 20:02:26 2020 +0000 chg: scanner: Stop storing recent_measurement_attempt because it stores a timestamp for each attempt, which makes state.dat grow thousand of lines (json). Closes #40023, #40020 --- sbws/core/scanner.py | 3 ++- tests/integration/test_files.py | 7 +++++-- tests/unit/lib/test_heartbeat.py | 2 ++ tests/unit/lib/test_v3bwfile.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index e5d5e90..8b6a6a4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -523,7 +523,8 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump, # Don't start measuring a relay if sbws is stopping. if settings.end_event.is_set(): break - relay_list.increment_recent_measurement_attempt() + # 40023, disable to decrease state.dat json lines + # relay_list.increment_recent_measurement_attempt() target.increment_relay_recent_measurement_attempt() num_relays += 1 # callback and callback_err must be non-blocking diff --git a/tests/integration/test_files.py b/tests/integration/test_files.py index aa7973d..c44340c 100644 --- a/tests/integration/test_files.py +++ b/tests/integration/test_files.py @@ -32,7 +32,9 @@ def test_state(conf): assert 1 == state.count("recent_consensus") assert 1 == state.count("recent_priority_list") assert 15 == state.count("recent_priority_relay") - assert 15 == state.count("recent_measurement_attempt") + # Because of 40023, it's 0. Change to 15 if we store it again at some point + # assert 15 == state.count("recent_measurement_attempt") + assert not state.count("recent_measurement_attempt") def test_v3bwfile(conf): @@ -42,7 +44,8 @@ def test_v3bwfile(conf): assert "1" == bwfile.header.recent_consensus_count assert "1" == bwfile.header.recent_priority_list_count assert "15" == bwfile.header.recent_priority_relay_count - assert "15" == bwfile.header.recent_measurement_attempt_count + # Because of 40023, there is not this header + # assert "15" == bwfile.header.recent_measurement_attempt_count for bwline in bwfile.bw_lines: assert 1 == bwline.relay_in_recent_consensus_count assert 1 == bwline.relay_recent_priority_list_count diff --git a/tests/unit/lib/test_heartbeat.py b/tests/unit/lib/test_heartbeat.py index b2ad634..8a2b8ea 100644 --- a/tests/unit/lib/test_heartbeat.py +++ b/tests/unit/lib/test_heartbeat.py @@ -1,10 +1,12 @@ """Unit tests for heartbeat""" import logging +import pytest from sbws.lib import heartbeat from sbws.util.state import State +@pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled") def test_total_measured_percent(conf, caplog): state = State(conf["paths"]["state_fname"]) state["recent_priority_list"] = [1, 2, 3] diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 07f59d7..d1448d1 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -4,6 +4,7 @@ import json import logging import math import os.path +import pytest from unittest import mock from sbws import __version__ as version @@ -588,6 +589,7 @@ def test_recent_consensus_count(root_data_path, datadir): assert "1" == header.recent_consensus_count +@pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled") def test_recent_measurement_attempt_count(root_data_path, datadir): state_fpath = os.path.join(root_data_path, '.sbws/state.dat') assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file(