[tor-commits] [ooni-probe/master] Add _failing_ scheduler test showing one of #612 deadlock reasons

[art at torproject.org]

commit c35b28365412b1a854865fad7b1ce500928c5093
Author: Leonid Evdokimov <leon at darkk.net.ru>
Date:   Fri Oct 7 23:13:13 2016 +0300

    Add _failing_ scheduler test showing one of #612 deadlock reasons
    
    If the lock has too many waiters then some stack overflow raises
    RuntimeError and scheduler fails altogether.
---
 ooni/tests/test_scheduler.py | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py
index a9501c0..c8eaf55 100644
--- a/ooni/tests/test_scheduler.py
+++ b/ooni/tests/test_scheduler.py
@@ -1,9 +1,10 @@
 import os
+import sys
 import shutil
 import random
 import tempfile
 
-from twisted.internet import defer
+from twisted.internet import reactor, task, defer
 from twisted.trial import unittest
 
 from ooni.agent.scheduler import ScheduledTask, DidNotRun, FileSystemlockAndMutex
@@ -51,8 +52,40 @@ class TestScheduler(unittest.TestCase):
             self.assertEqual(len(in_file.readlines()), 1)
 
         self.assertEqual(dummy_st.should_run, False)
+        self.assertFalse(os.path.islink(os.path.join(scheduler_directory, identifier + '.lock')))
         shutil.rmtree(scheduler_directory)
 
+    def test_thundering_herd(self):
+        lockno = int(os.getenv('TTH_LOCKNO', str(sys.getrecursionlimit() + 16)))
+        scheduler_directory = tempfile.mkdtemp()
+        counter = os.path.join(scheduler_directory, 'counter')
+        class DummyST(ScheduledTask):
+            @defer.inlineCallbacks
+            def task(subself):
+                self.assertTrue(os.path.islink(os.path.join(scheduler_directory, subself.identifier + '.lock')))
+                with open(counter, 'w+') as fd:
+                    data = fd.read()
+                    fd.seek(0)
+                    if not data:
+                        fd.write('1')
+                        yield task.deferLater(reactor, 1, lambda: 42)
+                    else:
+                        self.assertTrue(False) # should be unreachable due to schedule
+        identifier = "dummy"
+        dummy_st = DummyST(schedule='@daily', identifier=identifier, scheduler_directory=scheduler_directory)
+        dl = defer.DeferredList([dummy_st.run() for i in xrange(lockno)], consumeErrors=True)
+        @dl.addBoth
+        def so_what(results):
+            self.assertFalse(os.path.islink(os.path.join(scheduler_directory, identifier + '.lock')))
+            self.assertEqual(results[0], (True, None)) # do not expect to get `42` here
+            for okflag, ex in results[1:]:
+                self.assertEqual(okflag, False)
+                ex.trap(DidNotRun)
+            self.assertEqual(dummy_st.should_run, False)
+            with open(counter, 'r') as fd:
+                self.assertEqual(fd.read(), '1')
+            shutil.rmtree(scheduler_directory)
+        return dl
 
     @defer.inlineCallbacks
     def test_filesystem_lock_and_mutex(self):