[tor-commits] [sbws/master] fixup! scanner: log backtrace when not progressing

[juga at torproject.org]

commit 5dae9b99d2cda373b5f05139faf1b7a8710fc5d8
Author: juga0 <juga at riseup.net>
Date:   Wed Jan 23 10:22:17 2019 +0000

    fixup! scanner: log backtrace when not progressing
---
 sbws/core/scanner.py | 29 ++++++++++++++++++-----------
 sbws/globals.py      |  1 +
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py
index 9c7dff5..67fa30b 100644
--- a/sbws/core/scanner.py
+++ b/sbws/core/scanner.py
@@ -12,7 +12,7 @@ from ..lib.relayprioritizer import RelayPrioritizer
 from ..lib.destination import DestinationList
 from ..util.timestamp import now_isodt_str
 from ..util.state import State
-from sbws.globals import fail_hard
+from sbws.globals import fail_hard, TIMEOUT_MEASUREMENTS
 import sbws.util.stem as stem_utils
 import sbws.util.requests as requests_utils
 from argparse import ArgumentDefaultsHelpFormatter
@@ -31,17 +31,24 @@ log = logging.getLogger(__name__)
 
 
 def dumpstacks():
-    import pdb
     import traceback
-    log.warning("sbws stop measuring relays, probably because of a bug."
-                "Please, open a ticket in trac.torproject.org with this"
-                "backtrace.")
+    log.critical("sbws stop measuring relays, probably because of a bug."
+                 "Please, open a ticket in trac.torproject.org with this"
+                 "backtrace.")
     thread_id2name = dict([(t.ident, t.name) for t in threading.enumerate()])
     for thread_id, stack in sys._current_frames().items():
         log.critical("Thread: %s(%d)",
                      thread_id2name.get(thread_id, ""), thread_id)
         log.critical(traceback.print_stack(stack))
-    pdb.set_trace()
+    # If logging level is less than DEBUG (more verbose), start pdb so that
+    # developers can debug the issue.
+    if log.getEffectiveLevel() < logging.DEBUG:
+        import pdb
+        pdb.set_trace()
+    # Otherwise exit.
+    else:
+        # Change to stop threads when #28869 is merged
+        sys.exit(1)
 
 
 def timed_recv_from_server(session, dest, byte_range):
@@ -376,15 +383,15 @@ def run_speedtest(args, conf):
             while len(pending_results) >= max_pending_results:
                 time.sleep(5)
                 pending_results = [r for r in pending_results if not r.ready()]
-        counter = 0
-        # give it 3min, otherwise there's a bug or deadlock
-        while len(pending_results) > 0 and counter <= 36:
-            counter += 1
+        time_waiting = 0
+        while (len(pending_results) > 0
+               and time_waiting <= TIMEOUT_MEASUREMENTS):
             log.debug("Number of pending measurement threads %s after "
                       "a prioritization loop.", len(pending_results))
             time.sleep(5)
+            time_waiting += 5
             pending_results = [r for r in pending_results if not r.ready()]
-        if counter > 36:
+        if time_waiting > TIMEOUT_MEASUREMENTS:
             dumpstacks()
         loop_tstop = time.time()
         loop_tdelta = (loop_tstop - loop_tstart) / 60
diff --git a/sbws/globals.py b/sbws/globals.py
index 2277850..50caaf7 100644
--- a/sbws/globals.py
+++ b/sbws/globals.py
@@ -34,6 +34,7 @@ SUPERVISED_USER_CONFIG_PATH = "/etc/sbws/sbws.ini"
 SUPERVISED_RUN_DPATH = "/run/sbws/tor"
 
 SOCKET_TIMEOUT = 60  # seconds
+TIMEOUT_MEASUREMENTS = 60 * 3  # 3 minutes
 
 SBWS_SCALE_CONSTANT = 7500
 TORFLOW_SCALING = 1