- tor-commits - lists.torproject.org

[ooni-probe/master] Join ooniprobe and ooniprobequeue into one command
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 1192ca0d0fd7c7b4aa5cb593fe711eb175d5c880 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 23:33:14 2016 +0200 Join ooniprobe and ooniprobequeue into one command --- bin/ooniprobe | 19 +++++++++++++++++-- bin/ooniprobequeue | 16 ---------------- ooni/oonicli.py | 21 +++++++-------------- ooni/tests/test_oonicli.py | 5 +++-- setup.py | 3 +-- 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index 4832d91..ad97553 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,9 +1,24 @@ #!/usr/bin/env python +import sys + from twisted.internet import reactor -from ooni.oonicli import runWithDirector -d = runWithDirector() +from ooni.oonicli import setupGlobalOptions +from ooni.oonicli import runWithDaemonDirector, runWithDirector + +exit_code=0 + +global_options = setupGlobalOptions(logging=True, start_tor=True, + check_incoherences=True) +if global_options['queue']: + d = runWithDaemonDirector(global_options) +else: + d = runWithDirector(global_options) @d.addBoth def cb(result): + global exit_code + if result is not None: + exit_code=1 reactor.stop() reactor.run() +sys.exit(exit_code) diff --git a/bin/ooniprobequeue b/bin/ooniprobequeue deleted file mode 100755 index 88359e0..0000000 --- a/bin/ooniprobequeue +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import sys -from twisted.internet import reactor - -code=0 - -from ooni.oonicli import runWithDaemonDirector -d = runWithDaemonDirector() -(a)d.addBoth -def cb(result): - global code - if result is not None: - code=1 - reactor.stop() -reactor.run() -sys.exit(code) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 09936b6..1ffe467 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -205,6 +205,7 @@ def setupGlobalOptions(logging, start_tor, check_incoherences): log.err("Insufficient Privileges to capture packets." " See ooniprobe.conf privacy.includepcap") sys.exit(2) + global_options['check_incoherences'] = check_incoherences return global_options def setupAnnotations(global_options): @@ -294,14 +295,13 @@ def createDeck(global_options, url=None): return deck -def runTestWithDirector(director, global_options, url=None, - start_tor=True, check_incoherences=True): +def runTestWithDirector(director, global_options, url=None, start_tor=True): deck = createDeck(global_options, url=url) start_tor |= deck.requiresTor d = director.start(start_tor=start_tor, - check_incoherences=check_incoherences) + check_incoherences=global_options['check_incoherences']) def setup_nettest(_): try: @@ -341,13 +341,11 @@ def runTestWithDirector(director, global_options, url=None, d.addErrback(director_startup_other_failures) return d -def runWithDirector(logging=True, start_tor=True, check_incoherences=True): +def runWithDirector(global_options): """ Instance the director, parse command line options and start an ooniprobe test! """ - global_options = setupGlobalOptions(logging, start_tor, check_incoherences) - from ooni.director import Director director = Director() if global_options['list']: @@ -398,14 +396,13 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): return runTestWithDirector(director=director, start_tor=start_tor, - global_options=global_options, - check_incoherences=check_incoherences) + global_options=global_options) # this variant version of runWithDirector splits the process in two, # allowing a single director instance to be reused with multiple decks. -def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True): +def runWithDaemonDirector(global_options): """ Instance the director, parse command line options and start an ooniprobe test! @@ -421,9 +418,6 @@ def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True) print "Install with \"pip install pika\"." sys.exit(7) - - global_options = setupGlobalOptions(logging, start_tor, check_incoherences) - director = Director() if global_options.get('annotations') is not None: @@ -464,8 +458,7 @@ def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True) d = runTestWithDirector(director=director, start_tor=start_tor, global_options=global_options, - url=data['url'].encode('utf8'), - check_incoherences=check_incoherences) + url=data['url'].encode('utf8')) # When the test has been completed, go back to waiting for a message. d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1) except exceptions.AMQPError,v: diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 172b9cd..f5c8545 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -7,7 +7,7 @@ from twisted.internet import defer from ooni.tests import is_internet_connected from ooni.tests.bases import ConfigTestCase from ooni.settings import config -from ooni.oonicli import runWithDirector +from ooni.oonicli import runWithDirector, setupGlobalOptions from ooni.utils.net import hasRawSocketPermission @@ -89,7 +89,8 @@ class TestRunDirector(ConfigTestCase): sys.argv.extend(ooni_args) sys.argv.extend(['-n', '-o', output_file, test_name]) sys.argv.extend(nettest_args) - yield runWithDirector(False, False, False) + global_options = setupGlobalOptions(False, False, False) + yield runWithDirector(global_options) with open(output_file) as f: entries = yaml.safe_load_all(f) header = entries.next() diff --git a/setup.py b/setup.py index 1fd9a7b..a70df0c 100644 --- a/setup.py +++ b/setup.py @@ -204,8 +204,7 @@ setup( packages=packages, include_package_data=True, scripts=["bin/oonideckgen", "bin/ooniprobe", - "bin/oonireport", "bin/ooniresources", - "bin/ooniprobequeue"], + "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links, install_requires=install_requires, zip_safe=False,

1 0

[ooni-probe/master] Merge branch 'feature/fix_cct_tests' into release/1.4.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 397ed5506d97a17cb4d05852af06ebf3674c638b Merge: b2dbd38 bcc2be4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 23:17:01 2016 +0200 Merge branch 'feature/fix_cct_tests' into release/1.4.0 * feature/fix_cct_tests: Fix bug in parsing of options for Lantern Add support for setting the url and expected-body in tests Re-create the command inside the test method. Update user documentation for psiphon and lantern When GeoIP lookup is disabled it's not possible to perform sanitisations Always use the same URL when testing psiphon Add support for stripping potential inclusion of probe IPs in reports Refactor lantern test to use the ProcessTest template Add test for Vanilla Tor docs/source/tests/lantern.rst | 23 ++++--- docs/source/tests/psiphon.rst | 24 ++++--- ooni/nettests/blocking/vanilla_tor.py | 105 +++++++++++++++++++++++++++++ ooni/nettests/third_party/lantern.py | 122 ++++++++++++++++++---------------- ooni/nettests/third_party/psiphon.py | 50 ++++++++------ ooni/templates/httpt.py | 8 ++- ooni/templates/process.py | 9 ++- ooni/utils/net.py | 3 + 8 files changed, 245 insertions(+), 99 deletions(-) diff --cc ooni/nettests/third_party/psiphon.py index 2633f60,2b18b18..d4eb18f --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@@ -30,11 -31,11 +31,11 @@@ class PsiphonTest(httpt.HTTPTest, proc """ name = "Psiphon Test" - description = "Bootstraps Psiphon and \ - does a HTTP GET for the specified URL" + description = ("Bootstraps Psiphon and" + "does a HTTP GET for the specified URL") author = "juga" - version = "0.0.1" - timeout = 20 + version = "0.1.0" + timeout = 120 usageOptions = UsageOptions def _setUp(self): diff --cc ooni/templates/httpt.py index edab3fa,12995bb..fd6c8ab --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@@ -163,22 -131,21 +163,26 @@@ class HTTPTest(NetTestCase) 'url': request['url'], 'method': request['method'], 'tor': request['tor'] -- } ++ }, ++ 'response': None } if response: + if self.localOptions.get('withoutbody', 0) is 0: + response_body = _representBody(response_body) + else: + response_body = '' + # Attempt to redact the IP address of the probe from the responses + if config.privacy.includeip is False and config.probe_ip.address is not None: + response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") - request_response['response'] = { - 'headers': list(response.headers.getAllRawHeaders()), - 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', + session['response'] = { + 'headers': _representHeaders(response.headers), + 'body': response_body, 'code': response.code - } + } + session['failure'] = None if failure_string: - request_response['failure'] = failure_string - - self.report['requests'].append(request_response) + session['failure'] = failure_string + self.report['requests'].append(session) def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body")

1 0

[ooni-probe/master] Update references to ooni logo
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 44028962566098be17af0ecb5ccb2c3fa3858fba Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 19:53:43 2016 +0200 Update references to ooni logo * Use the png instead of the svg for the mascot image --- README.rst | 2 +- setup.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b42d755..ef9e0ba 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ ooniprobe: a network interference detection tool ___________________________________________________________________________ -.. image:: https://ooni.torproject.org/images/ooni-header-mascot.svg +.. image:: https://ooni.torproject.org/images/ooni-header-mascot.png :target: https:://ooni.torproject.org/ OONI, the Open Observatory of Network Interference, is a global observation diff --git a/setup.py b/setup.py index a70df0c..da0b967 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ ooniprobe: a network interference detection tool ___________________________________________________________________________ -.. image:: https://ooni.torproject.org/theme/img/ooni-logo.png +.. image:: https://ooni.torproject.org/images/ooni-header-mascot.png :target: https:://ooni.torproject.org/ OONI, the Open Observatory of Network Interference, is a global observation @@ -53,6 +53,13 @@ To install ooniprobe you will need the following dependencies: * libffi-dev * libdumbnet-dev + +On debian based systems this can generally be done by running: + +.. code:: bash + + sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip + When you got them run: .. code:: bash

1 0

[ooni-probe/master] Always output the same deck filename
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 97cd370e7349e6f74471f8ebb6beedb900197551 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 12:14:18 2016 +0200 Always output the same deck filename --- README.rst | 2 +- ooni/deckgen/cli.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index f137b88..a6dcb2c 100644 --- a/README.rst +++ b/README.rst @@ -188,7 +188,7 @@ you can do so (be sure to have updated the input resources first) by running:: oonideckgen --country-code IT --output ~/ -You will now have in your home a folder called `deck-YOUR_COUNTRY_CODE`, containing the ooni +You will now have in your home a folder called `deck-it`, containing the ooni deck (ends with .deck) and the inputs. Note: that you should not move the `deck-*` directory once it has been generated as the paths to the inputs referenced by the test in the deck are diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index c2ebfd8..2e0e976 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -111,8 +111,7 @@ def generate_deck(options): if config.advanced.debug: deck.pprint() deck_filename = os.path.join(options['output'], - "%s-%s-user.deck" % (__version__, - options['country-code'])) + "default-user.deck") deck.write_to_file(deck_filename) print "Deck written to %s" % deck_filename print "Run ooniprobe like so:"

1 0

[ooni-probe/master] Use the process template in the netalyzr test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 51c9b63c9cbf4499f43ec497e883afbfee8388c8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 12:07:02 2016 +0200 Use the process template in the netalyzr test * Clean up text of psiphon test --- ooni/nettests/third_party/netalyzr.py | 77 +++++++++++++++++++---------------- ooni/nettests/third_party/psiphon.py | 2 +- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/ooni/nettests/third_party/netalyzr.py b/ooni/nettests/third_party/netalyzr.py index 3154af7..ee5c477 100644 --- a/ooni/nettests/third_party/netalyzr.py +++ b/ooni/nettests/third_party/netalyzr.py @@ -6,55 +6,62 @@ # Arturo "hellais" Filastò <art(a)fuffa.org> # :licence: see LICENSE -from ooni import nettest -from ooni.utils import log import time import os +import distutils.spawn + +from twisted.python import usage from twisted.internet import reactor, threads -class NetalyzrWrapperTest(nettest.NetTestCase): - name = "NetalyzrWrapper" - requiresRoot = False - requiresTor = False +from ooni.templates import process +from ooni.utils import log - def setUp(self): - cwd = os.path.abspath(os.path.join(os.path.abspath(__file__), '..')) +class JavaNotInstalled(Exception): + pass - # XXX set the output directory to something more uniform - outputdir = os.path.join(cwd, '..', '..') +class CouldNotFindNetalyzrCli(Exception): + pass - program_path = os.path.join(cwd, 'NetalyzrCLI.jar') - program = "java -jar %s -d" % program_path +class UsageOptions(usage.Options): + optParameters = [ + ['clipath', 'p', None, 'Specify the path to NetalyzrCLI.jar (can be ' + 'downloaded from ' + 'http://netalyzr.icsi.berkeley.edu/NetalyzrCLI.jar)'] + ] - test_token = time.asctime(time.gmtime()).replace(" ", "_").strip() +class NetalyzrWrapperTest(process.ProcessTest): + name = "NetalyzrWrapper" + description = "A wrapper around the Netalyzr java command line client" + author = "Jacob Appelbaum <jacob(a)appelbaum.net>" + + requiredOptions = ['clipath'] - self.output_file = os.path.join(outputdir, - "NetalyzrCLI_" + test_token + ".out") - self.output_file.strip() - self.run_me = program + " 2>&1 >> " + self.output_file + usageOptions = UsageOptions + requiresRoot = False + requiresTor = False - def blocking_call(self): - try: - result = threads.blockingCallFromThread(reactor, os.system, self.run_me) - except: - log.debug("Netalyzr had an error, please see the log file: %s" % self.output_file) - finally: - self.clean_up() + timeout = 300 + + def requirements(self): + if not distutils.spawn.find_executable("java"): + raise JavaNotInstalled("Java is not installed.") + + def setUp(self): + if not os.path.exists(self.localOptions['clipath']): + raise CouldNotFindNetalyzrCli("Could not find NetalyzrCLI.jar at {}".format(self.localOptions['clipath'])) - def clean_up(self): - self.report['netalyzr_report'] = self.output_file - log.debug("finished running NetalzrWrapper") - log.debug("Please check %s for Netalyzr output" % self.output_file) + self.command = [ + distutils.spawn.find_executable("java"), + "-jar", + "{}".format(self.localOptions['clipath']), + "-d" + ] def test_run_netalyzr(self): """ This test simply wraps netalyzr and runs it from command line """ log.msg("Running NetalyzrWrapper (this will take some time, be patient)") - log.debug("with command '%s'" % self.run_me) - # XXX we probably want to use a processprotocol here to obtain the - # stdout from Netalyzr. This would allows us to visualize progress - # (currently there is no progress because the stdout of os.system is - # trapped by twisted) and to include the link to the netalyzr report - # directly in the OONI report, perhaps even downloading it. - reactor.callInThread(self.blocking_call) + log.debug("with command '%s'" % self.command) + self.d = self.run(self.command, env=os.environ, usePTY=1) + return self.d diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index d4eb18f..9e50832 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -31,7 +31,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): """ name = "Psiphon Test" - description = ("Bootstraps Psiphon and" + description = ("Bootstraps Psiphon and " "does a HTTP GET for the specified URL") author = "juga" version = "0.1.0"

1 0

[ooni-probe/master] Add codename for ooniprobe release
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 28ba3237b782a95202008ba1aea98bc608295401 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 19:46:46 2016 +0200 Add codename for ooniprobe release --- ChangeLog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 82d04cd..471e282 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -4,6 +4,8 @@ Changelog v1.4.0 (Wed, 27 Apr 2016) ------------------------- +codename: Under the Sea + * Support for reporting using JSON * Support for running ooniprobe with a message queue providing URLs to test

1 0

[ooni-probe/master] Use a correct humans.txt in the example for lantern
by art＠torproject.org 29 Apr '16

29 Apr '16

commit f058349ee84de84c6ce491e02d5e022858693796 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 23:19:51 2016 +0200 Use a correct humans.txt in the example for lantern --- docs/source/tests/lantern.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 9f3ee09..7be4d49 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -34,7 +34,7 @@ How to run the test For advanced usages you may also configure a different URL and expected body for the response with the `--url` and `--expected-body` command line options. -`ooniprobe nettests/third_party/lantern.py --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` +`ooniprobe nettests/third_party/lantern.py --url http://humanstxt.org/humans.txt --expected-body '/* TEAM */'` Sample report =============

1 0

[ooni-probe/master] Improve the installation instructions for ooniprobe
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 652fe233c4b74011841420d947485be6779c0733 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 13:21:15 2016 +0200 Improve the installation instructions for ooniprobe --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index a6dcb2c..b42d755 100644 --- a/README.rst +++ b/README.rst @@ -107,10 +107,19 @@ Make sure you have installed the following dependencies: * libffi-dev * tor (>=0.2.5.1 to run all the tor related tests) +On debian based systems this can generally be done by running:: + + sudo apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip + Then you should be able to install ooniprobe by running:: + sudo pip install ooniprobe + +or as a user to install inside a virtualenv:: + pip install ooniprobe + Other platforms (with Vagrant) ------------------------------

1 0

[ooni-probe/master] Bump to version 1.4.0
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 943dbec8d344be7b02fb7f6e8ae6f9e0fc6fcbe1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Apr 27 19:48:05 2016 +0200 Bump to version 1.4.0 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index a9c6112..5ccd2da 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.3.2" +__version__ = "1.4.0" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'report', 'reporter',

1 0

[ooni-probe/master] Bump to version 1.4.2
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 4ddac3ecf1aa4e797214cbbcbb6a7930adc0c0f8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 29 11:28:22 2016 +0200 Bump to version 1.4.2 --- ChangeLog.rst | 8 ++++++++ ooni/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index c24d46d..56dd061 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,14 @@ Changelog ========= +v1.4.2 (Fri, 29 Apr 2016) +------------------------- + +* Hotfix for bug in serialising binary response bodies + +* Use the most recent scapy version + + v1.4.1 (Wed, 27 Apr 2016) ------------------------- diff --git a/ooni/__init__.py b/ooni/__init__.py index cbf84b7..8e05068 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.4.1" +__version__ = "1.4.2" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'report', 'reporter',

1 0