April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Add support for setting the url and expected-body in tests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c17f564789b85bc9baa9dc1c64e77d8ff070aebd Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 15:06:59 2016 +0100 Add support for setting the url and expected-body in tests * When these are changed from their default values a flag is set to false --- docs/source/tests/lantern.rst | 6 ++++++ docs/source/tests/psiphon.rst | 7 +++++++ ooni/nettests/third_party/lantern.py | 30 ++++++++++++++++++++++++++---- ooni/nettests/third_party/psiphon.py | 29 +++++++++++++++++++---------- ooni/utils/net.py | 3 +++ 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 9f3af2f..9f3ee09 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -31,6 +31,11 @@ How to run the test `ooniprobe nettests/third_party/lantern.py` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe nettests/third_party/lantern.py --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + Sample report ============= @@ -55,6 +60,7 @@ test_version: 0.0.1 --- body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." bootstrapped: true +default_configuration: true input: null lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} ``` diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index 3934bcb..3499501 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -35,6 +35,12 @@ To test Psiphon when it is installed in a different path other than the user hom `ooniprobe third_party/psiphon -p <path to Psiphon repository>` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe third_party/psiphon --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + + How to install Psiphon =================== @@ -80,6 +86,7 @@ Sample report agent: agent input: null psiphon_installed: true + default_configuration: true requests: - request: body: null diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 187e210..da19566 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,11 +1,22 @@ import os +import distutils.spawn + from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.web.client import ProxyAgent, readBody +from twisted.python import usage + from ooni.templates.process import ProcessTest, ProcessDirector -from ooni.utils import log +from ooni.utils import log, net from ooni.errors import handleAllFailures -import distutils.spawn + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over lantern (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] + ] class LanternTest(ProcessTest): """ @@ -26,10 +37,21 @@ class LanternTest(ProcessTest): timeout = 120 def setUp(self): + self.report['body'] = None + self.report['failure'] = None + self.report['success'] = None + self.report['default_configuration'] = True + self.command = [distutils.spawn.find_executable("lantern"), "--headless"] self.bootstrapped = defer.Deferred() self.exited = False - self.url = 'http://www.google.com/humans.txt' + + self.url = self.localOptions['url'] + if self.url != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False def stop(self, reason=None): if not self.exited: @@ -49,7 +71,7 @@ class LanternTest(ProcessTest): def test_lantern_circumvent(self): def addResultToReport(result): self.report['body'] = result - if result.startswith('Google is built by a large'): + if result.startswith(self.localOptions['expected-body']): log.msg("Got the HTTP response body I expected!") self.report['success'] = True else: diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 575002b..2b18b18 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -6,17 +6,19 @@ from twisted.internet import defer, reactor from twisted.internet.error import ProcessExitedAlready from twisted.python import usage -from ooni.utils import log +from ooni.utils import log, net from ooni.templates import process, httpt class UsageOptions(usage.Options): - log.debug("UsageOptions") optParameters = [ - ['psiphonpath', 'p', None, 'Specify psiphon python client path.'] + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over psiphon (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] ] - class PsiphonTest(httpt.HTTPTest, process.ProcessTest): """ @@ -43,8 +45,19 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): def setUp(self): log.debug('PsiphonTest.setUp') + self.report['bootstrapped_success'] = None + self.report['request_success'] = None + self.report['psiphon_found'] = None + self.report['default_configuration'] = True + self.bootstrapped = defer.Deferred() - self.url = 'http://www.google.com/humans.txt' + self.url = self.localOptions['url'] + + if self.localOptions['url'] != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -83,10 +96,6 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') self.createCommand() - - self.report['bootstrapped_success'] = None - self.report['request_success'] = None - self.report['psiphon_found'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') self.report['psiphon_found'] = False @@ -121,7 +130,7 @@ connect(False) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - if res.body.startswith('Google is built by a large'): + if res.body.startswith(self.localOptions['expected-body']): self.report['request_success'] = True else: self.report['request_success'] = False diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 96d62a1..1f87101 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -46,6 +46,9 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} +# This is used as a default for checking if we get the expected result when +# fetching URLs over some proxy. +GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large') class StringProducer(object): implements(IBodyProducer)

1 0

[ooni-probe/master] Update user documentation for psiphon and lantern
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c6964f5fb8ddcc3cec5d3eeebb28a07bd9ab0c7b Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 17:53:39 2016 +0100 Update user documentation for psiphon and lantern --- docs/source/tests/lantern.rst | 17 +++++++++-------- docs/source/tests/psiphon.rst | 17 ++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 0903c3a..9f3af2f 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -17,22 +17,25 @@ Description =========== This test launches Lantern in --headless mode, and parses output to determine -if it has bootstrapped. After bootstrap, it fetches the URL supplied by the ---url option using Lanterns http proxy interface listening on 127.0.0.1.8787. +if it has bootstrapped. After bootstrap, it fetches a URL using Lanterns HTTP +proxy interface listening on 127.0.0.1.8787 and checks to see if the response +body matches the expected result. +As a URL for testing we use http://www.google.com/humans.txt and look for the +string "Google is built by a large" in the response body. The specific string used to determine bootstrap from Lantern output in version -"2.0.10" is "client (http) proxy at" from standard output. +"2.0.10" is "Successfully dialed via" from standard output. How to run the test =================== -`ooniprobe nettests/third_party/lantern.py -u http://<url>` +`ooniprobe nettests/third_party/lantern.py` Sample report ============= From running: -`ooniprobe nettests/third_party/lantern.py -u http://www.google.com` +`ooniprobe nettests/third_party/lantern.py` :: @@ -50,9 +53,7 @@ test_name: lantern_circumvention_tool_test test_version: 0.0.1 ... --- -body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ - >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n\ - <A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" +body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." bootstrapped: true input: null lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index b05f1f0..3934bcb 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -16,7 +16,12 @@ Details Description =========== -This test first check that the Psiphon path exists, then launches Psiphon and parses output to determine if it has bootstrapped. After bootstrap, it fetches google.com (or other URL specified by the --url argument) using Psiphons SOCKS proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy argument). +This test first check that the Psiphon path exists, then launches Psiphon and +parses output to determine if it has bootstrapped. After bootstrap, it fetches +`http://www.google.com/humans.txt` using Psiphons SOCKS +proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy +argument). +It will then check to see if the response body contains the string: "Google is built by a large" The specific string used to determine bootstrap from Psiphon output in version "0.0.1" is "Press Ctrl-C to terminate." from standard output. @@ -26,10 +31,6 @@ How to run the test `ooniprobe third_party/psiphon` -To test Psiphon with a different URL run: - -`ooniprobe third_party/psiphon -u http://<url>/` - To test Psiphon when it is installed in a different path other than the user home: `ooniprobe third_party/psiphon -p <path to Psiphon repository>` @@ -49,7 +50,7 @@ To run Psiphon manually, it must be run inside of the proper directory: Sample report ============= -`ooniprobe third_party/psiphon -u http://google.com` +`ooniprobe third_party/psiphon` :: @@ -87,9 +88,7 @@ Sample report tor: {is_tor: false} url: http://google.com response: - body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ - >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has\ - \ moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" + body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." code: 301 headers: - - Content-Length

1 0

[ooni-probe/master] Refactor lantern test to use the ProcessTest template
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c1ff7516d35b7f76fb6638ae2bce04db5c4b3864 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 17:55:16 2016 +0100 Refactor lantern test to use the ProcessTest template * Improve blocking detection --- ooni/nettests/third_party/lantern.py | 98 ++++++++++++++---------------------- 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 1f1b3b0..187e210 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,47 +1,13 @@ +import os from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.python import usage from twisted.web.client import ProxyAgent, readBody from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log from ooni.errors import handleAllFailures -import os.path -from os import getenv - -class UsageOptions(usage.Options): - optParameters = [ - ['url', 'u', None, 'Specify a single URL to test.'],] - -class LanternProcessDirector(ProcessDirector): - """ - This Process Director monitors Lantern during its - bootstrap and fires a callback if bootstrap is - successful or an errback if it fails to bootstrap - before timing out. - """ - - def __init__(self, d, timeout=None): - self.d = d - self.stderr = "" - self.stdout = "" - self.finished = None - self.timeout = timeout - self.stdin = None - self.timer = None - self.exit_reason = None - self.bootstrapped = defer.Deferred() - - def outReceived(self, data): - self.stdout += data - # output received, see if we have bootstrapped - if not self.bootstrapped.called and "Connected to proxy on localhost" in self.stdout: - log.debug("Bootstrap Detected") - self.cancelTimer() - self.bootstrapped.callback("bootstrapped") - +import distutils.spawn class LanternTest(ProcessTest): - """ This class tests Lantern (https://getlantern.org) @@ -54,47 +20,59 @@ class LanternTest(ProcessTest): """ name = "Lantern Circumvention Tool Test" - description = "Bootstraps Lantern and does a HTTP GET for the specified URL" + description = "Bootstraps Lantern, connects to a URL and verifies if it contains the expected input" author = "Aaron Gibson" - version = "0.0.1" - timeout = 20 - usageOptions = UsageOptions - requiredOptions = ['url'] + version = "0.1.0" + timeout = 120 def setUp(self): - self.command = ["lantern", "--headless"] - self.d = defer.Deferred() - self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) - self.d.addCallback(self.processEnded, self.command) - if self.localOptions['url']: - self.url = self.localOptions['url'] - - def runLantern(self): - paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), self.command[0]) for x in getenv('PATH').split(':')]) - log.debug("Spawning Lantern") - reactor.spawnProcess(self.processDirector, paths[0], self.command) + self.command = [distutils.spawn.find_executable("lantern"), "--headless"] + self.bootstrapped = defer.Deferred() + self.exited = False + self.url = 'http://www.google.com/humans.txt' + + def stop(self, reason=None): + if not self.exited: + self.processDirector.close() + self.processDirector.transport.signalProcess('TERM') + self.exited = True + + def handleRead(self, stdout=None, stderr=None): + """ + This is called with each chunk of data from stdout and stderr. + """ + if not self.bootstrapped.called and "Successfully dialed via" in self.processDirector.stdout: + log.msg("Lantern connection successful") + self.processDirector.cancelTimer() + self.bootstrapped.callback("bootstrapped") def test_lantern_circumvent(self): - proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) - agent = ProxyAgent(proxyEndpoint, reactor) - def addResultToReport(result): self.report['body'] = result - self.report['success'] = True + if result.startswith('Google is built by a large'): + log.msg("Got the HTTP response body I expected!") + self.report['success'] = True + else: + self.report['success'] = False def addFailureToReport(failure): + log.err("Failed to connect to lantern") + log.failure(failure) self.report['failure'] = handleAllFailures(failure) self.report['success'] = False def doRequest(noreason): - log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url) + proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) + agent = ProxyAgent(proxyEndpoint, reactor) + log.msg("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url) request = agent.request("GET", self.url) request.addCallback(readBody) request.addCallback(addResultToReport) request.addCallback(self.processDirector.close) return request - self.processDirector.bootstrapped.addCallback(doRequest) - self.processDirector.bootstrapped.addErrback(addFailureToReport) - self.runLantern() + self.bootstrapped.addCallback(doRequest) + self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(self.stop) + self.d = self.run(self.command, env=os.environ, usePTY=1) return self.d

1 0

[ooni-probe/master] Set the measurement_start_time closer to the test method call to improve accuracy
by art＠torproject.org 29 Apr '16

29 Apr '16

commit cb718ade5beaf546bb87375f4a79bf16bd15956a Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 19:44:49 2016 +0100 Set the measurement_start_time closer to the test method call to improve accuracy --- ooni/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index b5a15e0..381902a 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -116,8 +116,6 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'measurement_start_time' not in self.testInstance.report.keys(): - self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() self.testInstance.setUp() @@ -141,6 +139,8 @@ class Measurement(TaskWithTimeout): pass def run(self): + if 'measurement_start_time' not in self.testInstance.report.keys(): + self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() return self.netTestMethod()

1 0

[ooni-probe/master] hotfix: remove unneeded calls for check_for_censorship in http_host test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 7405cacf858399ac7a18bc228cf8662c62a347cf Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 21:00:27 2016 +0100 hotfix: remove unneeded calls for check_for_censorship in http_host test --- ooni/nettests/manipulation/http_host.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index d0387cb..cab4d47 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -68,14 +68,12 @@ class HTTPHost(httpt.HTTPTest): if not body.startswith("{"): log.msg("This does not appear to be JSON") self.report['transparent_http_proxy'] = True - self.check_for_censorship(body) return try: content = json.loads(body) except: log.msg("The json does not parse, this is not what we expected") self.report['transparent_http_proxy'] = True - self.check_for_censorship(body) return # We base the determination of the presence of a transparent HTTP

1 0

[ooni-probe/master] Re-create the command inside the test method.
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d368c895b9fda39d3422ea78e567876b1b1294dc Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 14:41:45 2016 +0100 Re-create the command inside the test method. If it's not done inside the test method when it fails and is re-scheduled it will not re-run the setUp and hence it will have already been deleted. --- ooni/nettests/third_party/psiphon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 7c1462b..575002b 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -58,6 +58,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): getenv('HOME'), 'psiphon-circumvention-system/pyclient/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) + def createCommand(self): # psi_client.py can not be run directly because the paths in the # code are relative, so it'll fail to execute from this test x = """ @@ -68,7 +69,7 @@ connect(False) f.write(x) f.close() self.command = [sys.executable, f.name] - log.debug('command: %s' % ''.join(self.command)) + log.debug('command: %s' % ' '.join(self.command)) def handleRead(self, stdout, stderr): if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: @@ -81,6 +82,7 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') + self.createCommand() self.report['bootstrapped_success'] = None self.report['request_success'] = None

1 0

[ooni-probe/master] Implement measurement_start_time and test_start_time in client
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 58db73a6d8c7d5296262cfa8ea2ce6e987ef724e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 15:49:08 2016 +0100 Implement measurement_start_time and test_start_time in client --- ooni/nettest.py | 11 ++++++----- ooni/otime.py | 3 +++ ooni/reporter.py | 4 +++- ooni/tasks.py | 5 ++--- ooni/tests/test_director.py | 6 ++---- ooni/tests/test_reporter.py | 2 +- ooni/tests/test_utils.py | 10 +++++----- ooni/utils/__init__.py | 6 +++--- 8 files changed, 25 insertions(+), 22 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9bdc6d9..9c5e171 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -197,8 +197,7 @@ class NetTestLoader(object): return input_files - @property - def testDetails(self): + def setTestDetails(self): from ooni import __version__ as software_version input_file_hashes = [] @@ -206,8 +205,8 @@ class NetTestLoader(object): input_file_hashes.append(input_file['hash']) options = sanitize_options(self.options) - test_details = { - 'start_time': otime.epochToUTC(time.time()), + self.testDetails = { + 'test_start_time': otime.UTCTimestampNow(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], 'probe_ip': config.probe_ip.geodata['ip'], @@ -223,7 +222,6 @@ class NetTestLoader(object): 'annotations': self.annotations, 'data_format_version': '0.2.0' } - return test_details def _parseNetTestOptions(self, klass): """ @@ -351,6 +349,9 @@ class NetTestLoader(object): if options: klass.localOptions = options + # XXX this class all needs to be refactored and this is kind of a + # hack. + self.setTestDetails() test_instance = klass() if test_instance.requiresRoot and not hasRawSocketPermission(): diff --git a/ooni/otime.py b/ooni/otime.py index 1406fbf..37ada02 100644 --- a/ooni/otime.py +++ b/ooni/otime.py @@ -85,3 +85,6 @@ def epochToTimestamp(seconds): def epochToUTC(seconds): return float(datetime.utcfromtimestamp(seconds).strftime("%s")) + +def UTCTimestampNow(): + return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") diff --git a/ooni/reporter.py b/ooni/reporter.py index d9d147c..d772ae9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -267,6 +267,7 @@ class OONIBReporter(OReporter): 'input': entry.testInstance.report.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.testInstance.report.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.testInstance.report.pop('test_runtime', None), 'test_keys': entry.testInstance.report } @@ -275,6 +276,7 @@ class OONIBReporter(OReporter): 'input': entry.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } @@ -352,7 +354,7 @@ class OONIBReporter(OReporter): 'probe_cc': self.testDetails['probe_cc'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], - 'start_time': self.testDetails['start_time'], + 'test_start_time': self.testDetails['test_start_time'], 'input_hashes': self.testDetails['input_hashes'], 'data_format_version': self.testDetails['data_format_version'], 'format': 'json' diff --git a/ooni/tasks.py b/ooni/tasks.py index 3f331a2..b5a15e0 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -116,9 +116,8 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'test_start_time' not in self.testInstance.report.keys(): - start_time = otime.epochToUTC(self.testInstance._start_time) - self.testInstance.report['test_start_time'] = start_time + if 'measurement_start_time' not in self.testInstance.report.keys(): + self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() self.testInstance.setUp() diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 38e8481..61d504c 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -1,5 +1,3 @@ -import time - from mock import patch, MagicMock from ooni.settings import config @@ -60,7 +58,7 @@ class TestStartSniffing(unittest.TestCase): self.director = Director() self.testDetails = { 'test_name': 'foo', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' } # Each NetTestCase has a name attribute @@ -85,7 +83,7 @@ class TestStartSniffing(unittest.TestCase): self.testDetails = { 'test_name': 'bar', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' } with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: sniffer = mock_scapy_sniffer.return_value diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 4b99584..5adb4af 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -26,7 +26,7 @@ test_details = { 'input_hashes': [], 'probe_asn': 'AS0', 'probe_cc': 'ZZ', - 'start_time': time.time(), + 'test_start_time': '2016-01-01 22:33:11', 'data_format_version': '0.2.0' } diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index e2e448a..855eb19 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -8,7 +8,7 @@ class TestUtils(unittest.TestCase): def setUp(self): self.test_details = { 'test_name': 'foo', - 'start_time': 441763200 + 'test_start_time': '2016-01-01 01:22:22' } self.extension = 'ext' self.prefix = 'prefix' @@ -46,19 +46,19 @@ class TestUtils(unittest.TestCase): def test_generate_filename(self): filename = generate_filename(self.test_details) - self.assertEqual(filename, 'foo-1984-01-01T000000Z') + self.assertEqual(filename, 'foo-2016-01-01T012222Z') def test_generate_filename_with_extension(self): filename = generate_filename(self.test_details, extension=self.extension) - self.assertEqual(filename, 'foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext') def test_generate_filename_with_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z') def test_generate_filename_with_extension_and_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext') def test_generate_filename_with_filename(self): filename = generate_filename(self.test_details, filename=self.filename) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5e9616b..78a1c7b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -3,12 +3,12 @@ import string import random import glob import os +from datetime import datetime import gzip from base64 import b64encode from zipfile import ZipFile -from ooni import otime from ooni import errors @@ -122,8 +122,8 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): extension. """ if filename is None: - test_name, start_time = testDetails['test_name'], testDetails['start_time'] - start_time = otime.epochToTimestamp(start_time) + test_name, start_time = testDetails['test_name'], testDetails['test_start_time'] + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H%M%SZ") suffix = "%s-%s" % (test_name, start_time) basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix final_filename = '%s.%s' % (basename, extension) if extension is not None else basename

1 0

[ooni-probe/master] Fix bug in parsing of options for Lantern
by art＠torproject.org 29 Apr '16

29 Apr '16

commit bcc2be41fb5b3a0ce45e5dce81718f677fbf0af5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 20:10:12 2016 +0100 Fix bug in parsing of options for Lantern * Add check to assert if lantern is installed in test requirements --- ooni/nettests/third_party/lantern.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index da19566..5018ccb 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -10,6 +10,9 @@ from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log, net from ooni.errors import handleAllFailures +class LanternNotInstalled(Exception): + pass + class UsageOptions(usage.Options): optParameters = [ ['url', 'u', net.GOOGLE_HUMANS[0], @@ -35,6 +38,11 @@ class LanternTest(ProcessTest): author = "Aaron Gibson" version = "0.1.0" timeout = 120 + usageOptions = UsageOptions + + def requirements(self): + if not distutils.spawn.find_executable("lantern"): + raise LanternNotInstalled('lantern is not installed') def setUp(self): self.report['body'] = None

1 0

[ooni-probe/master] Remove dead/unnecessary code from otime
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 7528c07975edbd45776086ccaf329e0aed923d35 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 22 17:33:12 2016 +0100 Remove dead/unnecessary code from otime * Minor naming changes to functions --- ooni/nettest.py | 2 +- ooni/otime.py | 80 +++--------------------------------------------- ooni/tasks.py | 2 +- ooni/tests/test_otime.py | 14 --------- ooni/utils/log.py | 2 +- 5 files changed, 8 insertions(+), 92 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9c5e171..6cd9f57 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -206,7 +206,7 @@ class NetTestLoader(object): options = sanitize_options(self.options) self.testDetails = { - 'test_start_time': otime.UTCTimestampNow(), + 'test_start_time': otime.timestampNowLongUTC(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], 'probe_ip': config.probe_ip.geodata['ip'], diff --git a/ooni/otime.py b/ooni/otime.py index 37ada02..ffa95d0 100644 --- a/ooni/otime.py +++ b/ooni/otime.py @@ -1,17 +1,4 @@ -from datetime import datetime, timedelta, tzinfo - -class UTC(tzinfo): - """UTC""" - ZERO = timedelta(0) - - def utcoffset(self, dt): - return self.ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return self.ZERO +from datetime import datetime def prettyDateNow(): """ @@ -19,72 +6,15 @@ def prettyDateNow(): """ return datetime.now().ctime() -def utcPrettyDateNow(): +def prettyDateNowUTC(): """ Returns a good looking string for utc time. """ return datetime.utcnow().ctime() -class InvalidTimestampFormat(Exception): - pass - -def fromTimestamp(s): - """ - Converts a string that is output from the timestamp function back to a - datetime object - - Args: - s (str): a ISO8601 formatted string. - ex. 1912-06-23T101234Z" - - Note: we currently only support parsing strings that are generated from the - timestamp function and have no intention in supporting the full standard. - """ - try: - date_part, time_part = s.split('T') - hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6] - year, month, day = date_part.split('-') - except: - raise InvalidTimestampFormat(s) - - return datetime(int(year), int(month), int(day), int(hours), int(minutes), - int(seconds)) - -def timestamp(t=None): +def timestampNowLongUTC(): """ - The timestamp for ooni reports follows ISO 8601 in - UTC time format. - We do not inlcude ':' and include seconds. - - Example: - - if the current date is "10:12:34 AM, June 23 1912" (datetime(1912, 6, - 23, 10, 12, 34)) - - the timestamp will be: - - "1912-06-23T101234Z" - - Args: - t (datetime): a datetime object representing the - time to be represented (*MUST* be expressed - in UTC). - - If not specified will default to the current time - in UTC. + Returns a timestamp in the format of %Y-%m-%d %H:%M:%S in Universal Time + Coordinates. """ - if not t: - t = datetime.utcnow() - ISO8601 = "%Y-%m-%dT%H%M%SZ" - return t.strftime(ISO8601) - - -def epochToTimestamp(seconds): - return timestamp(datetime.fromtimestamp(seconds, UTC())) - - -def epochToUTC(seconds): - return float(datetime.utcfromtimestamp(seconds).strftime("%s")) - -def UTCTimestampNow(): return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") diff --git a/ooni/tasks.py b/ooni/tasks.py index 381902a..ac128d6 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -140,7 +140,7 @@ class Measurement(TaskWithTimeout): def run(self): if 'measurement_start_time' not in self.testInstance.report.keys(): - self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() + self.testInstance.report['measurement_start_time'] = otime.timestampNowLongUTC() return self.netTestMethod() diff --git a/ooni/tests/test_otime.py b/ooni/tests/test_otime.py deleted file mode 100644 index f3997eb..0000000 --- a/ooni/tests/test_otime.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest -from datetime import datetime -from ooni import otime - -test_date = datetime(2002, 6, 26, 22, 45, 49) - - -class TestOtime(unittest.TestCase): - def test_timestamp(self): - self.assertEqual(otime.timestamp(test_date), "2002-06-26T224549Z") - - def test_fromTimestamp(self): - time_stamp = otime.timestamp(test_date) - self.assertEqual(test_date, otime.fromTimestamp(time_stamp)) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index beba78f..d3036f0 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -64,7 +64,7 @@ class OONILogger(object): txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), - otime.utcPrettyDateNow())) + otime.prettyDateNowUTC())) self.fileObserver = txlog.FileLogObserver(daily_logfile) self.stdoutObserver = LogWithNoPrefix(sys.stdout)

1 0

[ooni-probe/master] Implement unittest to test for bug spotted by @bassosimone
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 48435739f1bdd6631065a8ef41e5c51605955b2a Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Mar 21 16:54:18 2016 +0100 Implement unittest to test for bug spotted by @bassosimone * Fix bug in serializing of JSON --- ooni/reporter.py | 2 +- ooni/tests/test_reporter.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index d772ae9..2ee8468 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -276,7 +276,7 @@ class OONIBReporter(OReporter): 'input': entry.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.pop('test_start_time', None), - 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), + 'measurement_start_time': entry.pop('measurement_start_time', None), 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 5adb4af..94192ee 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -32,7 +32,8 @@ test_details = { oonib_new_report_message = { 'report_id': "20140129T202038Z_AS0_" + "A" * 50, - 'backend_version': "1.0" + 'backend_version': "1.0", + 'supported_formats': ["yaml", "json"] } oonib_generic_error_message = {

1 0