July 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Bump the minimum required pyOpenSSL version
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 475f7c911f5947f95e8842d849d0fb87db46967f Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jun 21 16:31:31 2016 +0200 Bump the minimum required pyOpenSSL version * Using older versions of pyOpenSSL fails to verify certificates generated with letsencrypt. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7107d9e..dc184c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ setuptools>=11.3 PyYAML>=3.10 Twisted>=13.2.0 ipaddr>=2.1.10 -pyOpenSSL>=0.13 +pyOpenSSL>=0.15.1 geoip txtorcon>=0.7 txsocksx>=0.0.2

1 0

[ooni-probe/master] Update changelog
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 72508964f3357e18296054fe02f3a57be52752a2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Jul 10 22:20:50 2016 +0200 Update changelog --- ChangeLog.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index f995fc2..b21065b 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,32 @@ Changelog ========= +v1.6.0 (Sun, 10 Jul 2016) +------------------------- +codename: Shells and Seaweed + +* Add support for cloudfrontend and HTTPS collector +https://github.com/TheTorProject/ooni-probe/issues/530 + +* Add bisection logic to inputProcessor (big thanks to @seamustuohy for the + patch): + https://github.com/TheTorProject/ooni-probe/issues/503 + +* Add bridge failover support: + https://github.com/TheTorProject/ooni-probe/issues/538 + +* Make it possible to run tests without specifying the test type + https://github.com/TheTorProject/ooni-probe/issues/483 + +Bug fixes: + +* Silently ignores '--pcapfile' flag (thanks to @willscott for the patch!): + https://github.com/TheTorProject/ooni-probe/issues/521 + +* The options specified on the command line should have priority over the deck + options. + https://github.com/TheTorProject/ooni-probe/issues/529 + v1.5.1 (Fri, 3 Jun 2016) ------------------------- codename: The Big Wave

1 0

[ooni-probe/master] Add preferred-backend command line option
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 45dc9638d321fac3e27e4dbf4aef69e1a62886d0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jun 29 19:16:46 2016 +0200 Add preferred-backend command line option --- ooni/oonicli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c0ee5cd..e9038a4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -57,6 +57,11 @@ class Options(usage.Options): "directory."], ["annotations", "a", None, "Annotate the report with a key:value[, " "key:value] format."], + ["preferred-backend", "p", None, "Set the preferred backend to use " + "when submitting results and/or " + "communicating with test helpers. " + "Can be either onion, " + "https or cloudfront"], ["queue", "Q", None, "AMQP Queue URL amqp://user:pass@host:port/vhost/queue"] ] @@ -385,6 +390,9 @@ def runWithDirector(global_options): if global_options.get('annotations') is not None: global_options['annotations'] = setupAnnotations(global_options) + if global_options.get('preferred-backend') is not None: + config.advanced.preferred_backend = global_options['preferred-backend'] + if global_options['no-collector']: log.msg("Not reporting using a collector") global_options['collector'] = None

1 0

[ooni-probe/master] Call setupBaseAddress after setting the settings
by art＠torproject.org 10 Jul '16

10 Jul '16

commit bec21d9ab32e5dbbbde7927dc1bb572742ebb24c Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jun 21 18:00:41 2016 +0200 Call setupBaseAddress after setting the settings This makes it possible to re-instantiate a BackendClient from the settings --- ooni/backend_client.py | 2 +- ooni/tests/test_deck.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index 83869cc..506aba4 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -42,12 +42,12 @@ class OONIBClient(object): self.backend_type = guess_backend_type(self.base_address) self.backend_type = self.backend_type.encode('ascii') - self._setupBaseAddress() self.settings = { 'type': self.backend_type, 'address': self.base_address, 'front': self.front } + self._setupBaseAddress() def _setupBaseAddress(self): parsed_address = urlparse(self.base_address) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index f299e70..ba87ec8 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -185,7 +185,7 @@ class TestDeck(BaseTestCase, ConfigTestCase): yield deck.lookupCollectorAndTestHelpers() self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], - 'http://thirteenchars123.onion') + 'httpo://thirteenchars123.onion') self.assertEqual(deck.netTestLoaders[0].localOptions['backend'], '127.0.0.1') @@ -237,7 +237,7 @@ class TestDeck(BaseTestCase, ConfigTestCase): ) self.assertEqual( deck.netTestLoaders[0].collector.settings['front'], - 'https://front.cloudfront.net' + 'front.cloudfront.net' ) self.assertEqual(

1 0

[ooni-probe/master] Update to version 1.6.0
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 0248c55416ca9dd0491901804bfcf5f6cfe8d7e8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jun 30 12:38:31 2016 +0200 Update to version 1.6.0 * Set the correct addresses for the canonical https and cloudfronted bouncer * Update contact information --- ooni/__init__.py | 4 ++-- ooni/constants.py | 6 ++---- setup.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 314534e..fdf96e5 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,10 +1,10 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.6.0rc1" +__version__ = "1.6.0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ -__resources_version__ = "1.5.1" +__resources_version__ = "1.6.0" __all__ = [ 'common', diff --git a/ooni/constants.py b/ooni/constants.py index 8752bab..6efa52e 100644 --- a/ooni/constants.py +++ b/ooni/constants.py @@ -1,9 +1,7 @@ CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion' -# XXX Change these two to the production ones once the release is made. -# CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io' -CANONICAL_BOUNCER_HTTPS = 'https://bouncer.test.ooni.io' +CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io' CANONICAL_BOUNCER_CLOUDFRONT = ( - 'https://d47sqwfs8741k.cloudfront.net/', + 'https://d3kr4emv7f56qa.cloudfront.net/', 'a0.awsstatic.com' ) diff --git a/setup.py b/setup.py index 1a8138c..ff1b6c2 100644 --- a/setup.py +++ b/setup.py @@ -336,7 +336,7 @@ setup( name="ooniprobe", version=__version__, author=__author__, - author_email="ooni-dev(a)lists.torproject.org", + author_email="contact(a)openobservatory.org", description="Network measurement tool for" "identifying traffic manipulation and blocking.", long_description=__doc__,

1 0

[ooni-probe/master] Change the default measurement concurrency to 4 and reporting to 7.
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 71e2b411377319f88b5c19f25e0339158dfcc677 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 8 17:09:11 2016 +0200 Change the default measurement concurrency to 4 and reporting to 7. Rewrite the configuration for lepidopter installations of ooniprobe. * Clean up the data/ tree --- MANIFEST.in | 2 ++ data/Makefile | 10 ---------- data/inputs/.gitignore | 2 ++ data/inputs/Makefile | 13 ------------- data/inputs/README | 47 ---------------------------------------------- data/ooniprobe.conf.sample | 10 +++++----- data/resources/README | 3 ++- setup.py | 22 ++++++++++++++++++++-- 8 files changed, 31 insertions(+), 78 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 920f933..e2903fb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,4 +6,6 @@ include data/ooniprobe.1 include data/oonireport.1 include data/ooniresources.1 include data/ooniprobe.conf.sample +include data/configs/lepidopter-ooniprobe.conf +include data/configs/lepidopter-oonireport.conf include ooni/settings.ini diff --git a/data/Makefile b/data/Makefile deleted file mode 100644 index 969cc65..0000000 --- a/data/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -geoip: - wget -N http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz - gunzip -f GeoIPASNum.dat.gz - wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat… - gunzip -f GeoIP.dat.gz - wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz - gunzip -f GeoLiteCity.dat.gz -clean: - rm -f *.dat - diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/inputs/Makefile b/data/inputs/Makefile deleted file mode 100644 index ff74cf6..0000000 --- a/data/inputs/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# This file will fetch a set of inputs that are of use to ooni-probe -all: whatheaders lists - -lists: - curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz - tar xzf input-pack.tar.gz - rm input-pack.tar.gz - -whatheaders: - wget http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip - unzip whatheaders-latest.xml.zip - mv whatheaders*.xml whatheaders.xml - rm whatheaders-latest.xml.zip diff --git a/data/inputs/README b/data/inputs/README deleted file mode 100644 index 9a55f67..0000000 --- a/data/inputs/README +++ /dev/null @@ -1,47 +0,0 @@ -In here you will find some very simple input lists that are useful for testing -the correct functionality of the various OONIProbe tests. - -# DNS Consistency - -./ooniprobe-dev -o dns_tamper_test.yamloo data/nettests/blocking/dns_consistency.py -t -example_inputs/dns_tamper_test_resolvers.txt -f example_inputs/dns_tamper_file.txt - -less dns_tamper_test.yamloo - -# Captive Portal - -./ooniprobe-dev -o captive_portal_test.yamloo data/nettests/core/captiveportal.py - -less captive_portal_test.yamloo - -# HTTP Host - -./ooniprobe-dev -o http_host.yamloo data/nettests/manipulation/http_host.py -b http://ooni.nu/test -f example_inputs/http_host_file.txt - -less http_host.yamloo - -# Keyword filtering - -./ooniprobe-dev -o keyword_filtering.yamloo data/nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt - -less keyword_filtering.yamloo - -# URL List - -./ooniprobe-dev -o url_lists.yamloo data/nettests/core/url_list.py -f test_inputs/url_lists_file.txt - -less url_lists.yamloo - -# Squid transparent proxy - -./ooniprobe-dev -o squid.yamloo data/nettests/core/squid.py - -less squid.yamloo - -# HTTP Requests - -Not Implemented - -# Traceroute - -Not Implemented diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index e310ddc..cde4c00 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -33,17 +33,17 @@ advanced: # explicitly set the control port and SOCKS port start_tor: true # After how many seconds we should give up on a particular measurement - measurement_timeout: 60 + measurement_timeout: 120 # After how many retries we should give up on a measurement measurement_retries: 2 # How many measurements to perform concurrently - measurement_concurrency: 10 + measurement_concurrency: 4 # After how may seconds we should give up reporting - reporting_timeout: 80 + reporting_timeout: 360 # After how many retries to give up on reporting - reporting_retries: 3 + reporting_retries: 5 # How many reports to perform concurrently - reporting_concurrency: 15 + reporting_concurrency: 7 oonid_api_port: 8042 report_log_file: null inputs_dir: null diff --git a/data/resources/README b/data/resources/README index 5157b33..fb768f4 100644 --- a/data/resources/README +++ b/data/resources/README @@ -1 +1,2 @@ -In here shall be stored all the resources needed for using ooniresources and oonideckgen. +In here shall be stored all the resources downloaded via ooniresources to be +used by oonideckgen. diff --git a/setup.py b/setup.py index 1138ee3..1a8138c 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,8 @@ To setup a daily cronjob run this: Have fun! """ +from __future__ import print_function + from ooni import __version__, __author__ import os import sys @@ -106,13 +108,18 @@ TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" def run_command(args, cwd=None): try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError as e: + except EnvironmentError: return None stdout = p.communicate()[0].strip() if p.returncode != 0: return None return stdout +def is_lepidopter(): + if os.path.exists("/etc/default/lepidopter"): + return True + return False + class OoniInstall(install): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") @@ -175,11 +182,22 @@ class OoniInstall(install): stderr=sys.stderr.fileno()) process.wait() + def update_lepidopter_config(self): + try: + shutil.copyfile("data/configs/lepidopter-ooniprobe.conf", + "/etc/ooniprobe/ooniprobe.conf") + shutil.copyfile("data/configs/lepidopter-oonireport.conf", + "/etc/ooniprobe/oonireport.conf") + except Exception: + print("ERR: Failed to copy configuration files to /etc/ooniprobe/") + def run(self): prefix = os.path.abspath(self.prefix) self.set_data_files(prefix) self.do_egg_install() self.ooniresources() + if is_lepidopter(): + self.update_lepidopter_config() class ExecutableNotFound(Exception): pass @@ -226,7 +244,7 @@ class CreateOoniResources(Command): try: self.find_executables() except ExecutableNotFound as enf: - print("Could not find '{0}'".format(enf.message)) + print("ERR: Could not find '{0}'".format(enf.message)) return tmp_dir = tempfile.mkdtemp()

1 0

[ooni-probe/master] Avoid clash with -p printdeck
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 81ec18e582cc9e4960055bd94ef0577fb7ae2d12 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jun 30 10:31:35 2016 +0200 Avoid clash with -p printdeck --- ooni/oonicli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index e9038a4..fb3754f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -57,7 +57,7 @@ class Options(usage.Options): "directory."], ["annotations", "a", None, "Annotate the report with a key:value[, " "key:value] format."], - ["preferred-backend", "p", None, "Set the preferred backend to use " + ["preferred-backend", "P", None, "Set the preferred backend to use " "when submitting results and/or " "communicating with test helpers. " "Can be either onion, "

1 0

[ooni-probe/master] Add command for generating the ooni-resources file
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 17da512784a1373a2b9258b5c4c01b0548121418 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jun 30 12:37:50 2016 +0200 Add command for generating the ooni-resources file --- setup.py | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 3bfc5cd..1138ee3 100644 --- a/setup.py +++ b/setup.py @@ -89,15 +89,31 @@ Have fun! from ooni import __version__, __author__ import os import sys +import shutil import tempfile +import subprocess from ConfigParser import SafeConfigParser from os.path import join as pj -from setuptools import setup -from setuptools.command.install import install as _st_install +from setuptools import setup, Command +from setuptools.command.install import install from distutils.spawn import find_executable -class install(_st_install): +GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" +GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.da…" +TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" + +def run_command(args, cwd=None): + try: + p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) + except EnvironmentError as e: + return None + stdout = p.communicate()[0].strip() + if p.returncode != 0: + return None + return stdout + +class OoniInstall(install): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") o = open(config_file, "w+") @@ -154,9 +170,9 @@ class install(_st_install): def ooniresources(self): ooniresources = find_executable("ooniresources") - from subprocess import Popen - process = Popen([ooniresources], - stdout=sys.stdout.fileno(), stderr=sys.stderr.fileno()) + process = subprocess.Popen([ooniresources], + stdout=sys.stdout.fileno(), + stderr=sys.stderr.fileno()) process.wait() def run(self): @@ -165,6 +181,107 @@ class install(_st_install): self.do_egg_install() self.ooniresources() +class ExecutableNotFound(Exception): + pass + +class CreateOoniResources(Command): + description = ("Create ooni-resources.tar.gz containing test-lists and " + "GeoIP data files") + user_options = [] + + def initialize_options(self): + pass + def finalize_options(self): + pass + def download(self, url, directory, filename): + dst_path = pj(directory, filename) + args = [ + self.wget, + "-O", + dst_path, + url + ] + out = run_command(args) + if out is None: + raise Exception("Failed to download {0}".format(url)) + return dst_path + + def find_executables(self): + self.wget = find_executable("wget") + if not self.wget: + raise ExecutableNotFound("wget") + self.tar = find_executable("tar") + if not self.tar: + raise ExecutableNotFound("tar") + self.unzip = find_executable("unzip") + if not self.unzip: + raise ExecutableNotFound("unzip") + self.gunzip = find_executable("gunzip") + if not self.gunzip: + raise ExecutableNotFound("gunzip") + + def run(self): + dst_path = "dist/ooni-resources.tar.gz" + + try: + self.find_executables() + except ExecutableNotFound as enf: + print("Could not find '{0}'".format(enf.message)) + return + + tmp_dir = tempfile.mkdtemp() + pkg_dir = tempfile.mkdtemp() + + os.mkdir(pj(pkg_dir, "resources")) + os.mkdir(pj(pkg_dir, "GeoIP")) + + try: + geoip_asn_path = self.download(GEOIP_ASN_URL, tmp_dir, "GeoIPASNum.dat.gz") + except Exception as exc: + print(exc.message) + return + try: + geoip_path = self.download(GEOIP_URL, tmp_dir, "GeoIP.dat.gz") + except Exception as exc: + print(exc.message) + return + try: + test_lists_path = self.download(TEST_LISTS_URL, tmp_dir, "master.zip") + except Exception as exc: + print(exc.message) + return + + run_command([self.gunzip, geoip_asn_path]) + run_command([self.gunzip, geoip_path]) + run_command([self.unzip, "-d", tmp_dir, test_lists_path]) + + shutil.move(pj(tmp_dir, "GeoIP.dat"), + pj(pkg_dir, "GeoIP", "GeoIP.dat")) + shutil.move(pj(tmp_dir, "GeoIPASNum.dat"), + pj(pkg_dir, "GeoIP", "GeoIPASNum.dat")) + shutil.move(pj(tmp_dir, "test-lists-master", "lists"), + pj(pkg_dir, "resources", "citizenlab-test-lists")) + # Don't include services and official lists + shutil.rmtree( + pj(pkg_dir, + "resources", + "citizenlab-test-lists", + "services"), + ignore_errors=True) + shutil.rmtree( + pj(pkg_dir, + "resources", + "citizenlab-test-lists", + "official"), + ignore_errors=True) + run_command([self.tar, "cvzf", dst_path, "-C", pkg_dir, "."]) + + # Cleanup + shutil.rmtree(pkg_dir, ignore_errors=True) + shutil.rmtree(tmp_dir, ignore_errors=True) + print("Written ooniresources to {0}".format(dst_path)) + + install_requires = [] dependency_links = [] data_files = [] @@ -216,7 +333,10 @@ setup( dependency_links=dependency_links, install_requires=install_requires, zip_safe=False, - cmdclass={"install": install}, + cmdclass={ + "install": OoniInstall, + "create_ooniresources": CreateOoniResources + }, classifiers=( "Development Status :: 5 - Production/Stable", "Environment :: Console",

1 0

[ooni-probe/master] Add more unittests for backend_client
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 377be01de996948e5b8b40f57406c522b2e297c6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jun 28 13:52:07 2016 +0200 Add more unittests for backend_client --- ooni/tests/test_backend_client.py | 217 ++++++++++++++++++++++++++++++++++++++ ooni/tests/test_director.py | 1 - ooni/tests/test_oonibclient.py | 144 ------------------------- 3 files changed, 217 insertions(+), 145 deletions(-) diff --git a/ooni/tests/test_backend_client.py b/ooni/tests/test_backend_client.py new file mode 100644 index 0000000..212abb6 --- /dev/null +++ b/ooni/tests/test_backend_client.py @@ -0,0 +1,217 @@ +import os +import shutil +import socket + +from twisted.internet import defer +from twisted.web import error + +from ooni import errors as e +from ooni.settings import config +from ooni.backend_client import CollectorClient, BouncerClient +from ooni.backend_client import WebConnectivityClient +from ooni.tests.bases import ConfigTestCase + +from mock import MagicMock + +input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' +deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' + + +class TestEnd2EndBackendClient(ConfigTestCase): + def setUp(self): + super(TestEnd2EndBackendClient, self).setUp() + host = '127.0.0.1' + port = 8889 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((host, port)) + s.shutdown(2) + + data_dir = '/tmp/testooni' + config.advanced.data_dir = data_dir + + if os.path.exists(data_dir): + shutil.rmtree(data_dir) + os.mkdir(data_dir) + os.mkdir(os.path.join(data_dir, 'inputs')) + os.mkdir(os.path.join(data_dir, 'decks')) + except Exception: + self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") + self.collector_client = CollectorClient('http://' + host + ':' + str(port)) + + @defer.inlineCallbacks + def test_query(self): + res = yield self.collector_client.queryBackend('GET', '/policy/input') + self.assertTrue(isinstance(res, list)) + + @defer.inlineCallbacks + def test_get_input_list(self): + input_list = yield self.collector_client.getInputList() + self.assertTrue(isinstance(input_list, list)) + + @defer.inlineCallbacks + def test_get_input_descriptor(self): + input_descriptor = yield self.collector_client.getInput(input_id) + for key in ['name', 'description', + 'version', 'author', 'date', 'id']: + self.assertTrue(hasattr(input_descriptor, key)) + + @defer.inlineCallbacks + def test_download_input(self): + yield self.collector_client.downloadInput(input_id) + + @defer.inlineCallbacks + def test_get_deck_list(self): + deck_list = yield self.collector_client.getDeckList() + self.assertTrue(isinstance(deck_list, list)) + + @defer.inlineCallbacks + def test_get_deck_descriptor(self): + deck_descriptor = yield self.collector_client.getDeck(deck_id) + for key in ['name', 'description', + 'version', 'author', 'date', 'id']: + self.assertTrue(hasattr(deck_descriptor, key)) + + @defer.inlineCallbacks + def test_download_deck(self): + yield self.collector_client.downloadDeck(deck_id) + + def test_lookup_invalid_helpers(self): + bouncer_client = BouncerClient('http://127.0.0.1:8888') + return self.failUnlessFailure( + bouncer_client.lookupTestHelpers([ + 'sdadsadsa', 'dns' + ]), e.CouldNotFindTestHelper) + + @defer.inlineCallbacks + def test_lookup_no_test_helpers(self): + bouncer_client = BouncerClient('http://127.0.0.1:8888') + required_helpers = [] + helpers = yield bouncer_client.lookupTestHelpers(required_helpers) + self.assertTrue('default' in helpers.keys()) + + @defer.inlineCallbacks + def test_lookup_test_helpers(self): + bouncer_client = BouncerClient('http://127.0.0.1:8888') + required_helpers = [u'http-return-json-headers', u'dns'] + helpers = yield bouncer_client.lookupTestHelpers(required_helpers) + self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) + self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http')) + self.assertTrue(int(helpers['dns']['address'].split('.')[0])) + + @defer.inlineCallbacks + def test_input_descriptor_not_found(self): + yield self.assertFailure(self.collector_client.queryBackend('GET', + '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound) + + @defer.inlineCallbacks + def test_http_errors(self): + yield self.assertFailure(self.collector_client.queryBackend('PUT', + '/policy/input'), error.Error) + + @defer.inlineCallbacks + def test_create_report(self): + res = yield self.collector_client.queryBackend('POST', '/report', { + 'software_name': 'spam', + 'software_version': '2.0', + 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', + 'test_name': 'foobar', + 'test_version': '1.0', + 'input_hashes': [] + }) + assert isinstance(res['report_id'], unicode) + + @defer.inlineCallbacks + def test_report_lifecycle(self): + res = yield self.collector_client.queryBackend('POST', '/report', { + 'software_name': 'spam', + 'software_version': '2.0', + 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', + 'test_name': 'foobar', + 'test_version': '1.0', + 'input_hashes': [] + }) + report_id = str(res['report_id']) + + res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { + 'content': '---\nspam: ham\n...\n' + }) + + res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { + 'content': '---\nspam: ham\n...\n' + }) + + res = yield self.collector_client.queryBackend('POST', '/report/' + report_id + + '/close') + + +class TestBackendClient(ConfigTestCase): + @defer.inlineCallbacks + def test_web_connectivity_client_is_reachable(self): + wcc = WebConnectivityClient( + 'https://web-connectivity.th.ooni.io') + wcc.queryBackend = MagicMock() + wcc.queryBackend.return_value = defer.succeed({"status": "ok"}) + result = yield wcc.isReachable() + self.assertEqual(result, True) + + @defer.inlineCallbacks + def test_web_connectivity_client_is_not_reachable(self): + wcc = WebConnectivityClient( + 'https://web-connectivity.th.ooni.io') + wcc.queryBackend = MagicMock() + wcc.queryBackend.return_value = defer.fail(Exception()) + result = yield wcc.isReachable() + self.assertEqual(result, False) + + + @defer.inlineCallbacks + def test_web_connectivity_client_control(self): + wcc = WebConnectivityClient( + 'https://web-connectivity.th.ooni.io') + wcc.queryBackend = MagicMock() + wcc.queryBackend.return_value = defer.succeed({}) + yield wcc.control("http://example.com/", ["127.0.0.1:8080", + "127.0.0.1:8082"]) + wcc.queryBackend.assert_called_with( + 'POST', '/', + query={ + "http_request": "http://example.com/", + "tcp_connect": ["127.0.0.1:8080", "127.0.0.1:8082"] + }) + + + @defer.inlineCallbacks + def test_bouncer_client_lookup_collector(self): + bcc = BouncerClient('https://bouncer.ooni.io') + bcc.queryBackend = MagicMock() + bcc.queryBackend.return_value = defer.succeed({}) + yield bcc.lookupTestCollector(["foo"]) + bcc.queryBackend.assert_called_with("POST", + "/bouncer/net-tests", + query={'net-tests': ["foo"]}) + + + @defer.inlineCallbacks + def test_bouncer_client_lookup_test_helpers(self): + bcc = BouncerClient('https://bouncer.ooni.io') + bcc.queryBackend = MagicMock() + bcc.queryBackend.return_value = defer.succeed({'spam': 'ham'}) + yield bcc.lookupTestHelpers(["foo"]) + bcc.queryBackend.assert_called_with("POST", + "/bouncer/test-helpers", + query={'test-helpers': ["foo"]}) + + + def test_backend_client_validates_url(self): + raised = False + try: + cc = CollectorClient(settings={ + "type": "onion", "address": "http://invalid.onion" + }) + except Exception as exc: + raised = True + self.assertIsInstance(exc, e.InvalidAddress) + self.assertTrue(raised) diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 0cc740a..18377f5 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -2,7 +2,6 @@ from mock import patch, MagicMock from ooni.settings import config from ooni.director import Director -from ooni.nettest import NetTestLoader from ooni.tests.bases import ConfigTestCase from ooni.nettest import NetTestLoader diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py deleted file mode 100644 index a14a881..0000000 --- a/ooni/tests/test_oonibclient.py +++ /dev/null @@ -1,144 +0,0 @@ -import os -import shutil -import socket - -from twisted.internet import defer -from twisted.web import error - -from ooni import errors as e -from ooni.settings import config -from ooni.backend_client import CollectorClient, BouncerClient -from ooni.tests.bases import ConfigTestCase - -input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' -deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' - - -class TestOONIBClient(ConfigTestCase): - def setUp(self): - super(TestOONIBClient, self).setUp() - host = '127.0.0.1' - port = 8889 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.connect((host, port)) - s.shutdown(2) - - data_dir = '/tmp/testooni' - config.advanced.data_dir = data_dir - - if os.path.exists(data_dir): - shutil.rmtree(data_dir) - os.mkdir(data_dir) - os.mkdir(os.path.join(data_dir, 'inputs')) - os.mkdir(os.path.join(data_dir, 'decks')) - except Exception: - self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") - self.collector_client = CollectorClient('http://' + host + ':' + str(port)) - - @defer.inlineCallbacks - def test_query(self): - res = yield self.collector_client.queryBackend('GET', '/policy/input') - self.assertTrue(isinstance(res, list)) - - @defer.inlineCallbacks - def test_get_input_list(self): - input_list = yield self.collector_client.getInputList() - self.assertTrue(isinstance(input_list, list)) - - @defer.inlineCallbacks - def test_get_input_descriptor(self): - input_descriptor = yield self.collector_client.getInput(input_id) - for key in ['name', 'description', - 'version', 'author', 'date', 'id']: - self.assertTrue(hasattr(input_descriptor, key)) - - @defer.inlineCallbacks - def test_download_input(self): - yield self.collector_client.downloadInput(input_id) - - @defer.inlineCallbacks - def test_get_deck_list(self): - deck_list = yield self.collector_client.getDeckList() - self.assertTrue(isinstance(deck_list, list)) - - @defer.inlineCallbacks - def test_get_deck_descriptor(self): - deck_descriptor = yield self.collector_client.getDeck(deck_id) - for key in ['name', 'description', - 'version', 'author', 'date', 'id']: - self.assertTrue(hasattr(deck_descriptor, key)) - - @defer.inlineCallbacks - def test_download_deck(self): - yield self.collector_client.downloadDeck(deck_id) - - def test_lookup_invalid_helpers(self): - bouncer_client = BouncerClient('http://127.0.0.1:8888') - return self.failUnlessFailure( - bouncer_client.lookupTestHelpers([ - 'sdadsadsa', 'dns' - ]), e.CouldNotFindTestHelper) - - @defer.inlineCallbacks - def test_lookup_no_test_helpers(self): - bouncer_client = BouncerClient('http://127.0.0.1:8888') - required_helpers = [] - helpers = yield bouncer_client.lookupTestHelpers(required_helpers) - self.assertTrue('default' in helpers.keys()) - - @defer.inlineCallbacks - def test_lookup_test_helpers(self): - bouncer_client = BouncerClient('http://127.0.0.1:8888') - required_helpers = [u'http-return-json-headers', u'dns'] - helpers = yield bouncer_client.lookupTestHelpers(required_helpers) - self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) - self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http')) - self.assertTrue(int(helpers['dns']['address'].split('.')[0])) - - @defer.inlineCallbacks - def test_input_descriptor_not_found(self): - yield self.assertFailure(self.collector_client.queryBackend('GET', - '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound) - - @defer.inlineCallbacks - def test_http_errors(self): - yield self.assertFailure(self.collector_client.queryBackend('PUT', - '/policy/input'), error.Error) - - @defer.inlineCallbacks - def test_create_report(self): - res = yield self.collector_client.queryBackend('POST', '/report', { - 'software_name': 'spam', - 'software_version': '2.0', - 'probe_asn': 'AS0', - 'probe_cc': 'ZZ', - 'test_name': 'foobar', - 'test_version': '1.0', - 'input_hashes': [] - }) - assert isinstance(res['report_id'], unicode) - - @defer.inlineCallbacks - def test_report_lifecycle(self): - res = yield self.collector_client.queryBackend('POST', '/report', { - 'software_name': 'spam', - 'software_version': '2.0', - 'probe_asn': 'AS0', - 'probe_cc': 'ZZ', - 'test_name': 'foobar', - 'test_version': '1.0', - 'input_hashes': [] - }) - report_id = str(res['report_id']) - - res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { - 'content': '---\nspam: ham\n...\n' - }) - - res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, { - 'content': '---\nspam: ham\n...\n' - }) - - res = yield self.collector_client.queryBackend('POST', '/report/' + report_id + - '/close')

1 0

[ooni-probe/master] Merge pull request #546 from TheTorProject/feature/cloud-fronted-collectors
by art＠torproject.org 10 Jul '16

10 Jul '16

commit 5f8a23aa32e4f2e393863f86ac9a74f7611acaf8 Merge: d47aaff 81ec18e Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 1 17:26:10 2016 +0200 Merge pull request #546 from TheTorProject/feature/cloud-fronted-collectors Feature/cloud fronted collectors data/ooniprobe.conf.sample | 3 + ooni/__init__.py | 2 +- ooni/backend_client.py | 39 +++---- ooni/constants.py | 7 ++ ooni/deck.py | 112 ++++++++++++-------- ooni/errors.py | 4 + ooni/oonicli.py | 20 +++- ooni/tests/bases.py | 10 +- ooni/tests/mocks.py | 9 ++ ooni/tests/test_backend_client.py | 217 ++++++++++++++++++++++++++++++++++++++ ooni/tests/test_deck.py | 72 +++++++++++-- ooni/tests/test_director.py | 1 - ooni/tests/test_nettest.py | 4 +- ooni/tests/test_oonibclient.py | 144 ------------------------- requirements.txt | 3 +- 15 files changed, 419 insertions(+), 228 deletions(-)

1 0