October 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix one of the reasons of stuck scheduler lockfile for RunDeck task, see #612
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 877a2873a39641d2f0b6e56d8f40ead91c426e03 Author: Leonid Evdokimov <leon(a)darkk.net.ru> Date: Fri Sep 30 15:02:30 2016 +0300 Fix one of the reasons of stuck scheduler lockfile for RunDeck task, see #612 --- ooni/nettest.py | 18 +++++++++++++----- ooni/nettests/blocking/web_connectivity.py | 5 ++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index d6784f4..7a2afc8 100644 --- a/ooni/nettest.… [View More]py +++ b/ooni/nettest.py @@ -6,7 +6,7 @@ import sys from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.trial.runner import filenameToModule -from twisted.python import usage, reflect +from twisted.python import failure, usage, reflect from ooni import __version__ as ooniprobe_version, errors from ooni import otime @@ -578,6 +578,9 @@ class NetTest(object): """ This is a generator that yields measurements and registers the callbacks for when a measurement is successful or has failed. + + FIXME: If this generator throws exception TaskManager scheduler is + irreversibly damaged. """ for test_class, test_methods in self.testCases: @@ -591,11 +594,16 @@ class NetTest(object): test_instance._setUp() test_instance.summary = self.summary for method in test_methods: + try: + measurement = self.makeMeasurement( + test_instance, + method, + test_input) + except Exception: + log.exception(failure.Failure()) + log.err('Failed to run %s %s %s' % (test_instance, method, test_input)) + continue # it's better to skip single measurement... log.debug("Running %s %s" % (test_instance, method)) - measurement = self.makeMeasurement( - test_instance, - method, - test_input) measurements.append(measurement.done) self.state.taskCreated() yield measurement diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index d8d539f..1d1b742 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -29,6 +29,9 @@ from ooni.common.http_utils import REQUEST_HEADERS class InvalidControlResponse(Exception): pass +class AbsentHostname(Exception): + pass + class UsageOptions(usage.Options): optParameters = [ ['url', 'u', None, 'Specify a single URL to test'], @@ -175,7 +178,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.hostname = urlparse(self.input).netloc if not self.hostname: - raise Exception("Invalid input") + raise AbsentHostname('No hostname', self.input) self.control = { 'tcp_connect': {}, [View Less]

1 0

[ooni-probe/master] The correct name for cloudfront based backends is "cloudfront" (#620)
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 73b934ca05330d7c746e904f3f486e81b2ffcf12 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Oct 11 13:05:28 2016 +0200 The correct name for cloudfront based backends is "cloudfront" (#620) --- ooni/ui/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index e7a0d88..c16173b 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -210,13 +210,13 @@ def initializeOoniprobe(global_options): include_country … [View More]

1 0

[ooni-probe/master] The https backend bouncer was still the testing one (#627)
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 9799f1d2e602cf29478c922447643b178ed47557 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 12 16:13:30 2016 +0200 The https backend bouncer was still the testing one (#627) :8 --- ooni/constants.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ooni/constants.py b/ooni/constants.py index 38f85b4..6efa52e 100644 --- a/ooni/constants.py +++ b/ooni/constants.py @@ -1,7 +1,5 @@ CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion'… [View More]

1 0

[ooni-probe/master] Replace 32 bit GPG keyids with full ids, see https://evil32.com/ for a reason
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 7e3401223fb7f838e576c8ba0b9f299477000c38 Author: Leonid Evdokimov <leon(a)darkk.net.ru> Date: Thu Sep 29 12:39:48 2016 +0300 Replace 32 bit GPG keyids with full ids, see https://evil32.com/ for a reason Remember that old gnupg version did not use full id even when it was given full id: https://bugs.gnupg.org/gnupg/issue1579 --- Vagrantfile | 4 ++-- scripts/setup-dependencies.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --… [View More]

1 0

[ooni-probe/master] Comment update
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 9c08df08a1a682f6a93826c3ca73045dfd2d5b32 Author: Leonid Evdokimov <leon(a)darkk.net.ru> Date: Tue Oct 11 18:01:21 2016 +0300 Comment update --- ooni/agent/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 969602a..3e93d25 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -31,8 +31,8 @@ class FileSystemlockAndMutex(object): concurrently the mutex lock will … [View More]

1 0

[ooni-probe/master] Merge pull request #608 from TheTorProject/fix/606
by art＠torproject.org 14 Oct '16

14 Oct '16

commit ae465de52e18e084d465de3290c5888fd1a59e0a Merge: 4d79ff9 7f078c6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Sep 28 21:02:35 2016 +0200 Merge pull request #608 from TheTorProject/fix/606 On first run of the UpdateInputsAndResources task fetch the resources… ooni/__init__.py | 2 +- ooni/agent/scheduler.py | 11 +++++++++++ ooni/deck/store.py | 5 ++++- ooni/geoip.py | 1 + 4 files changed, 17 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Fix #612 deadlock confirmed by TestScheduler.test_thundering_herd
by art＠torproject.org 14 Oct '16

14 Oct '16

commit 15bfb0312030a79c1985564dcbf7ee18101388b4 Author: Leonid Evdokimov <leon(a)darkk.net.ru> Date: Mon Oct 10 23:06:50 2016 +0300 Fix #612 deadlock confirmed by TestScheduler.test_thundering_herd The trivial fix makes sense as current SchedulerService ticks every 30 seconds and imprecise scheduling is acceptable. --- ooni/agent/scheduler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index f6b52b7..… [View More]

1 0

[ooni-probe/master] Add some more debug level messages
by art＠torproject.org 14 Oct '16

14 Oct '16

commit ea311bbc6fd19700e6179e128a8e9abfd5fc84e2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 22 16:07:52 2016 +0200 Add some more debug level messages --- ooni/agent/scheduler.py | 3 +++ ooni/deck/store.py | 4 +++- ooni/geoip.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index cc71ee4..f6b52b7 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -123,6 +123,… [View More]

1 0

[ooni-probe/master] Add microsecond-precise logging to have more info to debug timing issues
by art＠torproject.org 14 Oct '16

14 Oct '16

commit e00072738e08dd0fccd09b8b48a7ed38b7a094ae Author: Leonid Evdokimov <leon(a)darkk.net.ru> Date: Thu Sep 29 13:58:03 2016 +0300 Add microsecond-precise logging to have more info to debug timing issues --- ooni/utils/log.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index f20fdce..b4136be 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -3,6 +3,7 @@ import sys import … [View More]errno import codecs import logging +from datetime import datetime from twisted.python import log as tw_log from twisted.python.logfile import DailyLogFile @@ -100,6 +101,27 @@ class StdoutStderrObserver(LogLevelObserver): self.write(text + "\n") self.flush() +class MsecLogObserver(tw_log.FileLogObserver): + def formatTime(self, when): + """ + Code from Twisted==16.4.1 modified to log microseconds. Although this + logging subsystem is legacy: http://twistedmatrix.com/trac/ticket/7596 + Also, `timeFormat` is not used as `%z` is broken. + """ + tzOffset = -self.getTimezoneOffset(when) + when = datetime.utcfromtimestamp(when + tzOffset) + tzHour = abs(int(tzOffset / 60 / 60)) + tzMin = abs(int(tzOffset / 60 % 60)) + if tzOffset < 0: + tzSign = '-' + else: + tzSign = '+' + return '%d-%02d-%02d %02d:%02d:%02d,%06d%s%02d%02d' % ( + when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.microsecond, + tzSign, tzHour, tzMin) + class OONILogger(object): def msg(self, msg, *arg, **kw): text = log_encode(msg) @@ -152,18 +174,18 @@ class OONILogger(object): daily_logfile = DailyLogFile(log_filename, log_folder) - tw_log.msg("Starting %s on %s (%s UTC)" % (application_name, - otime.prettyDateNow(), - otime.prettyDateNowUTC())) - - self.fileObserver = LogLevelObserver(daily_logfile, - log_level=file_log_level) + self.fileObserver = MsecLogObserver(LogLevelObserver(daily_logfile, + log_level=file_log_level)) self.stdoutObserver = StdoutStderrObserver(sys.stdout, log_level=stdout_log_level) tw_log.startLoggingWithObserver(self.fileObserver.emit) tw_log.addObserver(self.stdoutObserver.emit) + tw_log.msg("Starting %s on %s (%s UTC)" % (application_name, + otime.prettyDateNow(), + otime.prettyDateNowUTC())) + def stop(self): self.stdoutObserver.stop() self.fileObserver.stop() [View Less]

1 0

[ooni-probe/master] Change to scheduler and deck_store to better support unittesting
by art＠torproject.org 14 Oct '16

14 Oct '16

commit c0b9626242ba7627286da6512a8522f519d76556 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Oct 11 18:01:58 2016 +0200 Change to scheduler and deck_store to better support unittesting --- ooni/agent/scheduler.py | 11 ++++++++--- ooni/deck/store.py | 7 ++++--- ooni/utils/__init__.py | 11 ++++++++++- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 3e93d25..bda8510 100644 --- a/ooni/agent/… [View More]scheduler.py +++ b/ooni/agent/scheduler.py @@ -5,7 +5,7 @@ from hashlib import md5 from datetime import datetime from twisted.application import service -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.internet.task import LoopingCall from twisted.python.filepath import FilePath @@ -71,7 +71,9 @@ class ScheduledTask(object): identifier = None def __init__(self, schedule=None, identifier=None, - scheduler_directory=config.scheduler_directory): + scheduler_directory=None): + if scheduler_directory is None: + scheduler_directory = config.scheduler_directory if schedule is not None: self.schedule = schedule if identifier is not None: @@ -346,11 +348,14 @@ class SchedulerService(service.MultiService): """ This service is responsible for running the periodic tasks. """ - def __init__(self, director, interval=30): + def __init__(self, director, interval=30, _reactor=reactor): service.MultiService.__init__(self) self.director = director self.interval = interval + self._looping_call = LoopingCall(self._should_run) + self._looping_call.clock = _reactor + self._scheduled_tasks = [] def schedule(self, task): diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 1e4785e..8fdb856 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -130,9 +130,10 @@ class InputStore(object): class DeckStore(object): - def __init__(self): - self.enabled_directory = FilePath(config.decks_enabled_directory) - self.available_directory = FilePath(config.decks_available_directory) + def __init__(self, enabled_directory=config.decks_enabled_directory, + available_directory=config.decks_available_directory): + self.enabled_directory = FilePath(enabled_directory) + self.available_directory = FilePath(available_directory) self._cache = {} self._cache_stale = True diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 99f1985..def533a 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -6,7 +6,7 @@ import errno import gzip import os -from datetime import datetime +from datetime import datetime, timedelta from zipfile import ZipFile from twisted.python.filepath import FilePath @@ -91,6 +91,15 @@ def randomStr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length)) +def randomDate(start, end): + """ + From: http://stackoverflow.com/a/553448 + """ + delta = end - start + int_delta = (delta.days * 24 * 60 * 60) + random_second = random.randrange(int_delta) + return start + timedelta(seconds=random_second) + LONG_DATE = "%Y-%m-%d %H:%M:%S" SHORT_DATE = "%Y%m%dT%H%M%SZ" [View Less]

1 0