[tor-commits] [sbws/master] chg: scanner: Stop storing recent_measurement_attempt

[juga at torproject.org]

commit f014084108eafd6478fbe5e2e0afc6855435c87f
Author: juga0 <juga at riseup.net>
Date:   Mon Nov 23 20:02:26 2020 +0000

    chg: scanner: Stop storing recent_measurement_attempt
    
    because it stores a timestamp for each attempt, which makes state.dat
    grow thousand of lines (json).
    
    Closes #40023, #40020
---
 sbws/core/scanner.py             | 3 ++-
 tests/integration/test_files.py  | 7 +++++--
 tests/unit/lib/test_heartbeat.py | 2 ++
 tests/unit/lib/test_v3bwfile.py  | 2 ++
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py
index e5d5e90..8b6a6a4 100644
--- a/sbws/core/scanner.py
+++ b/sbws/core/scanner.py
@@ -523,7 +523,8 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump,
             # Don't start measuring a relay if sbws is stopping.
             if settings.end_event.is_set():
                 break
-            relay_list.increment_recent_measurement_attempt()
+            # 40023, disable to decrease state.dat json lines
+            # relay_list.increment_recent_measurement_attempt()
             target.increment_relay_recent_measurement_attempt()
             num_relays += 1
             # callback and callback_err must be non-blocking
diff --git a/tests/integration/test_files.py b/tests/integration/test_files.py
index aa7973d..c44340c 100644
--- a/tests/integration/test_files.py
+++ b/tests/integration/test_files.py
@@ -32,7 +32,9 @@ def test_state(conf):
     assert 1 == state.count("recent_consensus")
     assert 1 == state.count("recent_priority_list")
     assert 15 == state.count("recent_priority_relay")
-    assert 15 == state.count("recent_measurement_attempt")
+    # Because of 40023, it's 0. Change to 15 if we store it again at some point
+    # assert 15 == state.count("recent_measurement_attempt")
+    assert not state.count("recent_measurement_attempt")
 
 
 def test_v3bwfile(conf):
@@ -42,7 +44,8 @@ def test_v3bwfile(conf):
     assert "1" == bwfile.header.recent_consensus_count
     assert "1" == bwfile.header.recent_priority_list_count
     assert "15" == bwfile.header.recent_priority_relay_count
-    assert "15" == bwfile.header.recent_measurement_attempt_count
+    # Because of 40023, there is not this header
+    # assert "15" == bwfile.header.recent_measurement_attempt_count
     for bwline in bwfile.bw_lines:
         assert 1 == bwline.relay_in_recent_consensus_count
         assert 1 == bwline.relay_recent_priority_list_count
diff --git a/tests/unit/lib/test_heartbeat.py b/tests/unit/lib/test_heartbeat.py
index b2ad634..8a2b8ea 100644
--- a/tests/unit/lib/test_heartbeat.py
+++ b/tests/unit/lib/test_heartbeat.py
@@ -1,10 +1,12 @@
 """Unit tests for heartbeat"""
 import logging
+import pytest
 
 from sbws.lib import heartbeat
 from sbws.util.state import State
 
 
+ at pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled")
 def test_total_measured_percent(conf, caplog):
     state = State(conf["paths"]["state_fname"])
     state["recent_priority_list"] = [1, 2, 3]
diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py
index 07f59d7..d1448d1 100644
--- a/tests/unit/lib/test_v3bwfile.py
+++ b/tests/unit/lib/test_v3bwfile.py
@@ -4,6 +4,7 @@ import json
 import logging
 import math
 import os.path
+import pytest
 from unittest import mock
 
 from sbws import __version__ as version
@@ -588,6 +589,7 @@ def test_recent_consensus_count(root_data_path, datadir):
     assert "1" == header.recent_consensus_count
 
 
+ at pytest.mark.skip(reason="increment_recent_measurement_attempt() disabled")
 def test_recent_measurement_attempt_count(root_data_path, datadir):
     state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
     assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file(