- tor-commits - lists.torproject.org

[ooni-probe/master] Drop dependencies that are for sub-modules of ooni
by art＠torproject.org 29 Apr '16

29 Apr '16

commit f098d7048e9394d0ad2f624be153b8d802ab4e61 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Apr 8 16:10:48 2016 +0200 Drop dependencies that are for sub-modules of ooni --- requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 28e5f78..fc6f629 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +pyasn1>=0.1.8 PyYAML>=3.10 Twisted>=13.2.0 ipaddr>=2.1.10 @@ -5,13 +6,8 @@ pyOpenSSL>=0.13 geoip txtorcon>=0.7 txsocksx>=0.0.2 -parsley>=1.1 scapy-real>=2.2.0-dev pypcap>=1.1 service-identity pydumbnet -pyasn1>=0.1.8 -pyasn1-modules -characteristic zope.interface -cryptography

1 0

[ooni-probe/master] Add support for downloading resources from mirror on github.
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 6a6e0ac21e9ee20db47df37a4fc9d4fb4e3be794 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 23 15:20:02 2016 +0200 Add support for downloading resources from mirror on github. * Download resources when they are missing from oonideckgen * Remove dependency on GeoLiteCity database (16MB is just too much) --- ooni/deckgen/cli.py | 32 +++++++-- ooni/deckgen/processors/citizenlab_test_lists.py | 6 +- ooni/geoip.py | 11 --- ooni/resources/__init__.py | 59 +--------------- ooni/resources/cli.py | 37 ++++------ ooni/resources/update.py | 86 ++++++++++++++---------- ooni/tests/test_geoip.py | 1 - 7 files changed, 93 insertions(+), 139 deletions(-) diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 1f618b7..b1a6b87 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -13,8 +13,9 @@ from ooni.geoip import ProbeIP from ooni.settings import config from ooni.deckgen import __version__ -from ooni.resources import inputs - +from ooni.deckgen.processors import citizenlab_test_lists +from ooni.deckgen.processors import namebench_dns_servers +from ooni.resources.update import download_resources class Options(usage.Options): synopsis = """%s [options] @@ -70,11 +71,9 @@ class Deck(object): def generate_deck(options): - dns_servers_processor = inputs['namebench-dns-servers.csv']['processor'] - url_lists_processor = inputs['citizenlab-test-lists.zip']['processor'] try: - url_list_country = url_lists_processor.generate_country_input( + url_list_country = citizenlab_test_lists.generate_country_input( options['country-code'], options['output'] ) @@ -84,10 +83,10 @@ def generate_deck(options): print "We will just use the global one." url_list_country = None - url_list_global = url_lists_processor.generate_global_input( + url_list_global = citizenlab_test_lists.generate_global_input( options['output'] ) - dns_servers = dns_servers_processor.generate_country_input( + dns_servers = namebench_dns_servers.generate_country_input( options['country-code'], options['output'] ) @@ -124,6 +123,20 @@ def get_user_country_code(): yield probe_ip.lookup() defer.returnValue(probe_ip.geodata['countrycode']) +def resources_up_to_date(): + if config.get_data_file_path("GeoIP/GeoIP.dat") is None: + return False + + if config.get_data_file_path("resources/" + "namebench-dns-servers.csv") is None: + return False + + if config.get_data_file_path("resources/" + "citizenlab-test-lists/" + "global.csv") is None: + return False + + return True @defer.inlineCallbacks def run(): @@ -135,6 +148,11 @@ def run(): print options sys.exit(1) + if not resources_up_to_date(): + print("Resources for running ooniprobe are not up to date.") + print("Will update them now.") + yield download_resources() + if not options['output']: options['output'] = os.getcwd() diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index 086b550..8a660cf 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -28,10 +28,9 @@ def generate_country_input(country_code, dst): input_list = config.get_data_file_path("resources/" "citizenlab-test-lists/" - "test-lists-master/lists/" + country_code + ".csv") - if not os.path.exists(input_list): + if not input_list: raise Exception("Could not find list for country %s" % country_code) load_input(input_list, filename) @@ -44,12 +43,11 @@ def generate_global_input(dst): input_list = config.get_data_file_path("resources/" "citizenlab-test-lists/" - "test-lists-master/lists/" "global.csv") if not input_list: print("Could not find the global input list") - print("Perhaps you should run ooniresources --update-inputs") + print("Perhaps you should run ooniresources") raise Exception("Could not find the global input list") load_input(input_list, filename) diff --git a/ooni/geoip.py b/ooni/geoip.py index 11800b6..83f3648 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -31,7 +31,6 @@ class GeoIPDataFilesNotFound(Exception): def IPToLocation(ipaddr): from ooni.settings import config - city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat') country_file = config.get_data_file_path('GeoIP/GeoIP.dat') asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat') @@ -51,12 +50,6 @@ def IPToLocation(ipaddr): error() try: - city_dat = GeoIP(city_file) - location['city'] = city_dat.record_by_addr(ipaddr)['city'] - except: - error() - - try: asn_dat = GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] except: @@ -75,10 +68,6 @@ def database_version(): 'GeoIPASNum': { 'sha256': None, 'timestamp': None - }, - 'GeoLiteCity': { - 'sha256': None, - 'timestamp': None } } diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py index 94cd36e..8b7e63b 100644 --- a/ooni/resources/__init__.py +++ b/ooni/resources/__init__.py @@ -1,59 +1,6 @@ -import os - -from ooni.settings import config -from ooni.utils import unzip, gunzip - -from ooni.deckgen.processors import citizenlab_test_lists -from ooni.deckgen.processors import namebench_dns_servers +from ooni import __version__ as ooniprobe_version __version__ = "0.1.0" -if os.access(config.var_lib_path, os.W_OK): - resources_directory = os.path.join(config.var_lib_path, - "resources") - geoip_directory = os.path.join(config.var_lib_path, - "GeoIP") -else: - resources_directory = os.path.join(config.ooni_home, - "resources") - geoip_directory = os.path.join(config.ooni_home, - "GeoIP") - -inputs = { - "namebench-dns-servers.csv": { - "url": "https://namebench.googlecode.com/svn/trunk/config/servers.csv", - "action": None, - "action_args": [], - "processor": namebench_dns_servers, - }, - "citizenlab-test-lists.zip": { - "url": "https://github.com/citizenlab/test-lists/archive/master.zip", - "action": unzip, - "action_args": [resources_directory], - "processor": citizenlab_test_lists - } -} - -geoip = { - "GeoLiteCity.dat.gz": { - "url": "https://geolite.maxmind.com/download/" - "geoip/database/GeoLiteCity.dat.gz", - "action": gunzip, - "action_args": [geoip_directory], - "processor": None - }, - "GeoIPASNum.dat.gz": { - "url": "https://geolite.maxmind.com/download/" - "geoip/database/asnum/GeoIPASNum.dat.gz", - "action": gunzip, - "action_args": [geoip_directory], - "processor": None - }, - "GeoIP.dat.gz": { - "url": "https://geolite.maxmind.com/" - "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", - "action": gunzip, - "action_args": [geoip_directory], - "processor": None - } -} +ooni_resources_url = ("https://github.com/TheTorProject/ooni-probe/releases" + "/download/v{}/ooni-resources.tar.gz").format(ooniprobe_version) diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index cab9700..0d7832a 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -3,18 +3,23 @@ import sys from twisted.internet import defer from twisted.python import usage -from ooni.utils import log - from ooni.resources import __version__ from ooni.resources import update class Options(usage.Options): - synopsis = """%s""" % sys.argv[0] + synopsis = """%s + This is used to update the resources required to run oonideckgen and + ooniprobe. + You just run this script with no arguments and it will update the + resources. + """ % sys.argv[0] optFlags = [ - ["update-inputs", None, "Update the resources needed for inputs."], - ["update-geoip", None, "Update the geoip related resources."] + ["update-inputs", None, "(deprecated) update the resources needed for " + "inputs."], + ["update-geoip", None, "(deprecated) Update the geoip related " + "resources."] ] optParameters = [] @@ -33,23 +38,7 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1) - if not any(options.values()): - print("%s: no command specified" % sys.argv[0]) - print options - sys.exit(1) + if options['update-inputs'] or options['update-geoip']: + print("WARNING: Passing command line arguments is deprecated") - if options['update-inputs']: - print "Downloading inputs" - try: - yield update.download_inputs() - except Exception as exc: - log.err("failed to download geoip files") - log.exception(exc) - - if options['update-geoip']: - print "Downloading geoip files" - try: - yield update.download_geoip() - except Exception as exc: - log.err("failed to download geoip files") - log.exception(exc) + yield update.download_resources() diff --git a/ooni/resources/update.py b/ooni/resources/update.py index 7a949c9..46bbdf8 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -1,43 +1,57 @@ import os +import tarfile +import tempfile +from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage -from ooni.resources import inputs, geoip, resources_directory -from ooni.utils import unzip, gunzip - +from ooni.settings import config +from ooni.resources import ooni_resources_url @defer.inlineCallbacks -def download_resource(resources): - for filename, resource in resources.items(): - print "Downloading %s" % filename - - if resource["action"] in [unzip, gunzip] and resource["action_args"]: - dirname = resource["action_args"][0] - filename = os.path.join(dirname, filename) - else: - filename = os.path.join(resources_directory, filename) - if not os.path.exists(filename): - directory = os.path.dirname(filename) - if not os.path.isdir(directory): - os.makedirs(directory) - f = open(filename, 'w') - f.close() - elif not os.path.isfile(filename): - print "[!] %s must be a file." % filename - defer.returnValue(False) - yield downloadPage(resource['url'], filename) - - if resource['action'] is not None: - yield defer.maybeDeferred(resource['action'], - filename, - *resource['action_args']) - print "%s written." % filename - - -def download_inputs(): - return download_resource(inputs) - - -def download_geoip(): - return download_resource(geoip) +def download_resources(): + if os.access(config.var_lib_path, os.W_OK): + dst_directory = FilePath(config.var_lib_path) + else: + dst_directory = FilePath(config.ooni_home) + + print("Downloading {} to {}".format(ooni_resources_url, + dst_directory.path)) + tmp_download_directory = FilePath(tempfile.mkdtemp()) + tmp_download_filename = tmp_download_directory.temporarySibling() + + + try: + yield downloadPage(ooni_resources_url, tmp_download_filename.path) + ooni_resources_tar_gz = tarfile.open(tmp_download_filename.path) + ooni_resources_tar_gz.extractall(tmp_download_directory.path) + + if not tmp_download_directory.child('GeoIP').exists(): + raise Exception("Could not find GeoIP data files in downloaded " + "tar.") + + if not tmp_download_directory.child('resources').exists(): + raise Exception("Could not find resources data files in " + "downloaded tar.") + + geoip_dir = dst_directory.child('GeoIP') + resources_dir = dst_directory.child('resources') + + if geoip_dir.exists(): + geoip_dir.remove() + tmp_download_directory.child('GeoIP').moveTo(geoip_dir) + + if resources_dir.exists(): + resources_dir.remove() + tmp_download_directory.child('resources').moveTo(resources_dir) + + print("Written GeoIP files to {}".format(geoip_dir.path)) + print("Written resources files to {}".format(resources_dir.path)) + + except Exception as exc: + print("Failed to download resources!") + raise exc + + finally: + tmp_download_directory.remove() diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index 5e8b67b..66ba13e 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -26,7 +26,6 @@ class TestGeoIP(bases.ConfigTestCase): version = geoip.database_version() assert 'GeoIP' in version.keys() assert 'GeoIPASNum' in version.keys() - assert 'GeoLiteCity' in version.keys() assert len(version['GeoIP']['sha256']) == 64 assert isinstance(version['GeoIP']['timestamp'], float)

1 0

[ooni-probe/master] Remove hacks module and patch reduce_ex
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 55e55a55a0365c3e4958482e315add4f0090243a Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 23 19:03:49 2016 +0200 Remove hacks module and patch reduce_ex * Delete benchmarks txt file --- bin/ooniprobe | 7 ------ bin/ooniprobequeue | 5 ----- ooni/kit/benchmarks.txt | 35 ----------------------------- ooni/utils/hacks.py | 60 ------------------------------------------------- 4 files changed, 107 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index 31da086..4832d91 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,13 +1,6 @@ #!/usr/bin/env python -import copy_reg from twisted.internet import reactor -# This is a hack to overcome a bug in python -from ooni.utils.hacks import patched_reduce_ex -copy_reg._reduce_ex = patched_reduce_ex - -# from ooni.oonicli import run -# run() from ooni.oonicli import runWithDirector d = runWithDirector() @d.addBoth diff --git a/bin/ooniprobequeue b/bin/ooniprobequeue index f81f156..88359e0 100755 --- a/bin/ooniprobequeue +++ b/bin/ooniprobequeue @@ -1,12 +1,7 @@ #!/usr/bin/env python import sys -import copy_reg from twisted.internet import reactor -# This is a hack to overcome a bug in python -from ooni.utils.hacks import patched_reduce_ex -copy_reg._reduce_ex = patched_reduce_ex - code=0 from ooni.oonicli import runWithDaemonDirector diff --git a/ooni/kit/benchmarks.txt b/ooni/kit/benchmarks.txt deleted file mode 100644 index c2070e5..0000000 --- a/ooni/kit/benchmarks.txt +++ /dev/null @@ -1,35 +0,0 @@ -Read file B -done in 0.74356508255 -Read file A -done in 0.94336104393 -Computing prob matrix -done in 0.0432229042053 -Computing eigenvalues -done in 0.00188422203064 -Corelation: 0.999999079331 - - -Read file B -done in 0.946599960327 -Read file A -done in 0.909075975418 -Computing prob matrix -done in 0.0541450977325 -Computing eigenvalues -done in 0.00259518623352 -Computing prob matrix B -done in 0.060467004776 -Computing eigen B -done in 0.00102496147156 -Computing correlation -done in 0.000158071517944 -Corelation: 0.999999079331 - -this was with: -683 filea.txt -678 fileb.txt - -diff file* | wc -l -283 - - diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py deleted file mode 100644 index 7ec3d0f..0000000 --- a/ooni/utils/hacks.py +++ /dev/null @@ -1,60 +0,0 @@ -# When some software has issues and we need to fix it in a -# hackish way, we put it in here. This one day will be empty. - -import copy_reg -from twisted.web.client import SchemeNotSupported - -from txsocksx.http import SOCKS5Agent as SOCKS5AgentOriginal - - -def patched_reduce_ex(self, proto): - """ - This is a hack to overcome a bug in one of pythons core functions. It is - located inside of copy_reg and is called _reduce_ex. - - Some background on the issue can be found here: - - http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attr… - http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object - - There was also an open bug on the pyyaml trac repo, but it got closed because - they could not reproduce. - http://pyyaml.org/ticket/190 - - It turned out to be easier to patch the python core library than to monkey - patch yaml. - - XXX see if there is a better way. sigh... - """ - _HEAPTYPE = 1 << 9 - assert proto < 2 - for base in self.__class__.__mro__: - if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: - break - else: - base = object # not really reachable - if base is object: - state = None - elif base is int: - state = None - else: - if base is self.__class__: - raise TypeError("can't pickle %s objects" % base.__name__) - state = base(self) - args = (self.__class__, base, state) - try: - getstate = self.__getstate__ - except AttributeError: - if getattr(self, "__slots__", None): - raise TypeError("a class that defines __slots__ without " - "defining __getstate__ cannot be pickled") - try: - dict = self.__dict__ - except AttributeError: - dict = None - else: - dict = getstate() - if dict: - return copy_reg._reconstructor, args, dict - else: - return copy_reg._reconstructor, args

1 0

[ooni-probe/master] Print a helpful message when run with no arguments (inciting to run oonideckgen)
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 96dad3f578e0797aee4992fc79b25ea8af2b9414 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 23 15:48:09 2016 +0200 Print a helpful message when run with no arguments (inciting to run oonideckgen) * Optimise a bit the importing of modules --- ooni/oonicli.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 63312b5..5bc2124 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -7,20 +7,11 @@ import random import urlparse from twisted.python import usage -from twisted.python.util import spewer -from twisted.internet import defer, reactor, protocol +from twisted.internet import defer from ooni import errors, __version__ - from ooni.settings import config -from ooni.director import Director -from ooni.deck import Deck, nettest_to_path -from ooni.nettest import NetTestLoader - from ooni.utils import log -from ooni.utils.net import hasRawSocketPermission - - class LifetimeExceeded(Exception): pass @@ -30,7 +21,7 @@ class Options(usage.Options): longdesc = ("ooniprobe loads and executes a suite or a set of suites of" " network tests. These are loaded from modules, packages and" - " files listed on the command line") + " files listed on the command line.") optFlags = [["help", "h"], ["resume", "r"], @@ -71,11 +62,21 @@ class Options(usage.Options): self['test'] = None usage.Options.__init__(self) + def getUsage(self, width=None): + return super(Options, self).getUsage(width) + """ +To get started you may want to run: + +$ oonideckgen + +This will tell you how to run ooniprobe :) +""" + def opt_spew(self): """ Print an insanely verbose log of everything that happens. Useful when debugging freezes or locks in complex code. """ + from twisted.python.util import spewer sys.settrace(spewer) def opt_version(self): @@ -186,6 +187,7 @@ def setupGlobalOptions(logging, start_tor, check_incoherences): log.start(global_options['logfile']) if config.privacy.includepcap: + from ooni.utils.net import hasRawSocketPermission if hasRawSocketPermission(): from ooni.utils.txscapy import ScapyFactory config.scapyFactory = ScapyFactory(config.advanced.interface) @@ -223,6 +225,9 @@ def setupCollector(global_options, collector_address): def createDeck(global_options, url=None): + from ooni.nettest import NetTestLoader + from ooni.deck import Deck, nettest_to_path + if url: log.msg("Creating deck for: %s" % (url)) @@ -331,9 +336,9 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): Instance the director, parse command line options and start an ooniprobe test! """ - global_options = setupGlobalOptions(logging, start_tor, check_incoherences) + from ooni.director import Director director = Director() if global_options['list']: print "# Installed nettests" @@ -379,7 +384,8 @@ def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True) Instance the director, parse command line options and start an ooniprobe test! """ - + from twisted.internet import reactor, protocol + from ooni.director import Director try: import pika from pika import exceptions

1 0

[ooni-probe/master] Detect when the control response contains a blockpage from cloudflare
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 210ec234e1d60be752b44e1b2136b69659de6eec Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Apr 24 21:31:52 2016 +0200 Detect when the control response contains a blockpage from cloudflare --- ooni/nettests/blocking/http_requests.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index abc7efc..24d95cc 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -35,10 +35,10 @@ class HTTPRequestsTest(httpt.HTTPTest): lengths match. """ name = "HTTP Requests" - description = "Performs a HTTP GET request over Tor and one over the " \ - "local network and compares the two results." + description = ("Performs a HTTP GET request over Tor and one over the " + "local network and compares the two results.") author = "Arturo Filastò" - version = "0.2.4" + version = "0.2.5" usageOptions = UsageOptions @@ -78,6 +78,7 @@ class HTTPRequestsTest(httpt.HTTPTest): self.report['factor'] = float(self.factor) self.report['headers_diff'] = None self.report['headers_match'] = None + self.report['control_cloudflare'] = None self.headers = {'User-Agent': [random.choice(userAgents)]} @@ -145,6 +146,11 @@ class HTTPRequestsTest(httpt.HTTPTest): if experiment and control: if hasattr(experiment, 'body') and hasattr(control, 'body') \ and experiment.body and control.body: + self.report['control_cloudflare'] = False + if 'Attention Required! | CloudFlare' in control.body: + log.msg("The control body contains a blockpage from " + "cloudflare. This will skew our results.") + self.report['control_cloudflare'] = True self.compare_body_lengths(len(control.body), len(experiment.body)) if hasattr(experiment, 'headers') and hasattr(control, 'headers') \

1 0

[ooni-probe/master] Remove references to the geoip_data_dir since it's not respected
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c79a087b5926e36962290ce1fb76a4301bddd49c Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 23 19:04:14 2016 +0200 Remove references to the geoip_data_dir since it's not respected --- data/ooniprobe.conf.sample | 1 - ooni/deckgen/processors/namebench_dns_servers.py | 3 +-- ooni/geoip.py | 6 +++--- ooni/oonicli.py | 1 + ooni/tests/test_oonicli.py | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 7aaacd7..5f3f17d 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -23,7 +23,6 @@ reports: pcap: null collector: null advanced: - geoip_data_dir: /usr/share/GeoIP debug: false # enable if auto detection fails #tor_binary: /usr/sbin/tor diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py index f35ca93..9855611 100644 --- a/ooni/deckgen/processors/namebench_dns_servers.py +++ b/ooni/deckgen/processors/namebench_dns_servers.py @@ -8,7 +8,6 @@ from ooni.settings import config class GeoIPDB(object): _borg = {} country = None - asn = None def __init__(self): self.__dict__ = self._borg @@ -25,7 +24,7 @@ class GeoIPDB(object): def generate_country_input(country_code, dst): csv_file = config.get_data_file_path("resources/" - "namebench-dns-servers.csv") + "namebench-dns-servers.csv") filename = os.path.join(dst, "dns-server-%s.txt" % country_code) fw = open(filename, "w") diff --git a/ooni/geoip.py b/ooni/geoip.py index 83f3648..6179844 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -37,9 +37,9 @@ def IPToLocation(ipaddr): location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} def error(): - log.err("Could not find GeoIP data file in %s." - "Try running ooniresources --update-geoip or" - " edit your ooniprobe.conf" % config.advanced.geoip_data_dir) + log.err("Could not find GeoIP data file in data directories." + "Try running ooniresources or" + " edit your ooniprobe.conf") try: country_dat = GeoIP(country_file) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5bc2124..8e7d1f1 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -169,6 +169,7 @@ def director_startup_other_failures(failure): def setupGlobalOptions(logging, start_tor, check_incoherences): global_options = parseOptions() + config.global_options = global_options config.set_paths() config.initialize_ooni_home() diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index c57205c..c3b179a 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -39,7 +39,6 @@ reports: pcap: null collector: null advanced: - geoip_data_dir: /usr/share/GeoIP debug: false interface: auto start_tor: false

1 0

[ooni-probe/master] Use debian/jessie instead of precise
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 223ba4803e17b25bc4d1a54bafc5b51896a2c14c Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 15:45:22 2016 +0200 Use debian/jessie instead of precise * Make the setup of ooni-backend conditional and do it in a another machine --- Vagrantfile | 85 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 16a6ba0..e61bf68 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,22 +1,8 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.configure("2") do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - config.vm.box = "precise32" - config.vm.box_url = "http://files.vagrantup.com/precise32.box" - - config.vm.synced_folder ".", "/ooni" - # Place the ooni-backend source code in ../ooni-backend to sync it with the vagrant instance - config.vm.synced_folder "../ooni-backend", "/backend" - -end - -$script = <<SCRIPT - -echo "deb http://deb.torproject.org/torproject.org precise main" >> /etc/apt/sources.list +$setup_ooniprobe = <<SCRIPT +echo "deb http://deb.torproject.org/torproject.org jessie main" >> /etc/apt/sources.list # Install deb.torproject.org key gpg --keyserver keys.gnupg.net --recv 886DDD89 @@ -27,18 +13,69 @@ apt-get update apt-get install -y tor deb.torproject.org-keyring # Setup for sniffer subsystem -apt-get install -y build-essential libdumbnet-dev python-dumbnet python-pypcap libpcap-dev python-dev python-pip libgeoip-dev libffi-dev -cd /ooni +apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip +cd /data/ooni-probe python setup.py install -echo "Login using 'vagrant ssh', and don't forget to run ooniprobe as root." -echo "First run: 'sudo ooniprobe -i /usr/share/ooni/decks/fast.deck'" + +echo "Now:" +echo "1. vagrant ssh probe" +echo "2. oonideckgen" +echo "3. ooniprobe -i deck-*/*.deck" SCRIPT -# TODO: -# Somehow, ooniprobe is not capable to connect to tor by default. My current -# workaround is to kill tor, and set "start_tor: true" in /root/.ooni/ooniprobe.conf +$setup_oonibackend = <<SCRIPT +echo "Installing Tor..." + +echo "deb http://deb.torproject.org/torproject.org jessie main" >> /etc/apt/sources.list + +gpg --keyserver keys.gnupg.net --recv 886DDD89 +gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | apt-key add - + +apt-get update + +apt-get -y install deb.torproject.org-keyring tor tor-geoipdb + +apt-get -y install curl python-pip python-dev libffi-dev + +cd /data/ooni-backend + +echo "Generating SSL keys" + +openssl genrsa -out private.key 4096 +openssl req -new -key private.key -out server.csr -subj '/CN=www.example.com/O=Example/C=AU' + +openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt + +cp oonib.conf.example /etc/oonibackend.conf + +echo "Installing ooni-backend" +python setup.py install + +echo "Now:" +echo "1. vagrant ssh" +echo "2. vi /etc/oonibackend.conf # possibly" +echo "3. cd /data/ooni-backend" +echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" + +SCRIPT Vagrant.configure("2") do |config| - config.vm.provision :shell, :inline => $script + # Use Debian jessie with the vboxfs contrib add-on + config.vm.box = "debian/contrib-jessie64" + + config.vm.define "probe" do |probe| + probe.vm.synced_folder ".", "/data/ooni-probe" + probe.vm.provision :shell, :inline => $setup_ooniprobe + end + + # If we find a ooni-backend directory in ../ooni-backend we configure a + # ooni-backend as well. + if File.directory?("../ooni-backend") + config.vm.define "backend" do |backend| + backend.vm.synced_folder "../ooni-backend", "/data/ooni-backend" + backend.vm.provision :shell, :inline => $setup_oonibackend + end + end + end

1 0

[ooni-probe/master] Improve readability of the ooniprobe -s command
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 33e86a4a4ae96053812685c9f663db261703e449 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Apr 24 19:13:45 2016 +0200 Improve readability of the ooniprobe -s command * Remove various pieces of dead code --- data/ooniprobe.conf.sample | 2 -- ooni/director.py | 2 +- ooni/geoip.py | 20 -------------------- ooni/nettests/third_party/psiphon.py | 4 ++-- ooni/oonicli.py | 30 +++++++++++++++++++++++------- ooni/tests/test_oonicli.py | 1 - ooni/utils/log.py | 2 -- 7 files changed, 26 insertions(+), 35 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 5f3f17d..edc3f86 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -12,8 +12,6 @@ privacy: includeasn: true # Should we include the country as reported by GeoIP in the report? includecountry: true - # Should we include the city as reported by GeoIP in the report? - includecity: false # Should we collect a full packet capture on the client? includepcap: false reports: diff --git a/ooni/director.py b/ooni/director.py index ab8da65..a60b91e 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -61,7 +61,7 @@ class Director(object): _scheduledTests = 0 # Only list NetTests belonging to these categories - categories = ['blocking', 'manipulation'] + categories = ['blocking', 'manipulation', 'third_party'] def __init__(self): self.activeNetTests = [] diff --git a/ooni/geoip.py b/ooni/geoip.py index 6179844..417c7d5 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -171,8 +171,6 @@ class ProbeIP(object): self.geodata['ip'] = self.address if not config.privacy.includeasn: self.geodata['asn'] = 'AS0' - if not config.privacy.includecity: - self.geodata['city'] = None if not config.privacy.includecountry: self.geodata['countrycode'] = 'ZZ' if not config.privacy.includeip: @@ -194,16 +192,6 @@ class ProbeIP(object): log.msg("Unable to lookup the probe IP via Tor.") try: - yield self.askTraceroute() - log.msg("Found your IP via Traceroute %s" % self.address) - self.resolveGeodata() - defer.returnValue(self.address) - except errors.InsufficientPrivileges: - log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") - except: - log.msg("Unable to lookup the probe IP via traceroute") - - try: yield self.askGeoIPService() log.msg("Found your IP via a GeoIP service: %s" % self.address) self.resolveGeodata() @@ -230,14 +218,6 @@ class ProbeIP(object): if not self.address: raise errors.ProbeIPUnknown - def askTraceroute(self): - """ - Perform a UDP traceroute to determine the probes IP address. - """ - if not hasRawSocketPermission(): - raise errors.InsufficientPrivileges - raise NotImplemented - def askTor(self): """ Obtain the probes IP address by asking the Tor Control port via GET INFO diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index bfdec70..2633f60 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -30,8 +30,8 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): """ name = "Psiphon Test" - description = "Bootstraps Psiphon and \ - does a HTTP GET for the specified URL" + description = ("Bootstraps Psiphon and" + "does a HTTP GET for the specified URL") author = "juga" version = "0.0.1" timeout = 20 diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8e7d1f1..3be64a9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -4,6 +4,7 @@ import os import json import yaml import random +import textwrap import urlparse from twisted.python import usage @@ -24,7 +25,6 @@ class Options(usage.Options): " files listed on the command line.") optFlags = [["help", "h"], - ["resume", "r"], ["no-collector", "n", "disable writing to collector"], ["no-yamloo", "N", "disable writing to YAML file"], ["no-geoip", "g"], @@ -342,12 +342,28 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True): from ooni.director import Director director = Director() if global_options['list']: - print "# Installed nettests" - for net_test_id, net_test in director.getNetTests().items(): - print "* %s (%s/%s)" % (net_test['name'], - net_test['category'], - net_test['id']) - print " %s" % net_test['description'] + net_tests = [net_test for net_test in director.getNetTests().items()] + print "" + print "Installed nettests" + print "==================" + for net_test_id, net_test in net_tests: + optList = [] + for name, details in net_test['arguments'].items(): + optList.append({'long': name, 'doc': details['description']}) + + desc = ('\n' + + net_test['name'] + + '\n' + + '-'*len(net_test['name']) + + '\n' + + '\n'.join(textwrap.wrap(net_test['description'], 80)) + + '\n\n' + + '$ ooniprobe {}/{}'.format(net_test['category'], + net_test['id']) + + '\n\n' + + ''.join(usage.docMakeChunks(optList)) + ) + print desc sys.exit(0) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index c3b179a..172b9cd 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -33,7 +33,6 @@ privacy: includeip: false includeasn: true includecountry: true - includecity: false includepcap: true reports: pcap: null diff --git a/ooni/utils/log.py b/ooni/utils/log.py index d3036f0..f36c0ff 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -52,8 +52,6 @@ class OONILogger(object): def start(self, logfile=None, application_name="ooniprobe"): from ooni.settings import config - daily_logfile = None - if not logfile: logfile = os.path.expanduser(config.basic.logfile)

1 0

[ooni-probe/master] Add --collector option to oonideckgen
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 88cfa90a7b76f2c0ff6cc3253459fce4d0790cd2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Apr 26 16:33:43 2016 +0200 Add --collector option to oonideckgen * Remove unused variable in ooniprobe * Add description to requirements.txt * Remove printing of probe IP --- Vagrantfile | 3 ++- ooni/deckgen/cli.py | 49 +++++++++++++++++++++++++++---------------------- ooni/geoip.py | 5 ++--- ooni/oonicli.py | 1 - requirements.txt | 4 ++++ 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 3eeb54c..59b2751 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -65,7 +65,9 @@ Vagrant.configure("2") do |config| backend.vm.synced_folder "../ooni-backend", "/data/ooni-backend" backend.vm.provision :shell, :inline => $setup_oonibackend end + end + if File.directory?("../ooni-backend") config.vm.provision "shell", inline: <<-EOF echo "To run oonibackend:" echo "1. vagrant ssh backend" @@ -73,7 +75,6 @@ Vagrant.configure("2") do |config| echo "3. cd /data/ooni-backend" echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" EOF - end config.vm.provision "shell", inline: <<-EOF diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 664842e..c2ebfd8 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -22,15 +22,13 @@ class Options(usage.Options): """ % sys.argv[0] optParameters = [ - ["country-code", "c", - None, + ["country-code", "c", None, "Specify the two letter country code for which we should " - "generate the deck." - ], - ["output", "o", - None, - "Specify the directory where to write output." - ] + "generate the deck."], + ["collector", None, None, "Specify a custom collector to use when " + "submitting reports"], + ["output", "o", None, + "Specify the directory where to write output."] ] def opt_version(self): @@ -41,33 +39,40 @@ class Options(usage.Options): class Deck(object): _base_entry = { "options": { + "test_file": None, + "subargs": [], + "annotations": None, + "collector": None, - "help": 0, - "logfile": None, - "no-default-reporter": 0, - "parallelism": None, - "pcapfile": None, + # XXX setting this is currently not supported + "bouncer": None, + "reportfile": None, - "resume": 0, - "testdeck": None + + "no-collector": 0, + "no-geoip": 0, + "no-yamloo": 0, + "verbose": 0 } } - def __init__(self): - self.deck = [] + def __init__(self, collector=None): + self.deck_entries = [] + self.collector = collector def add_test(self, test_file, subargs=[]): deck_entry = copy.deepcopy(self._base_entry) + deck_entry['options']['collector'] = self.collector deck_entry['options']['test_file'] = test_file deck_entry['options']['subargs'] = subargs - self.deck.append(deck_entry) + self.deck_entries.append(deck_entry) def pprint(self): - print yaml.safe_dump(self.deck) + print yaml.safe_dump(self.deck_entries) def write_to_file(self, filename): with open(filename, "w+") as f: - f.write(yaml.safe_dump(self.deck)) + f.write(yaml.safe_dump(self.deck_entries)) def generate_deck(options): @@ -89,11 +94,11 @@ def generate_deck(options): options['output'] ) - deck = Deck() + deck = Deck(collector=options['collector']) deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') - if url_list_country is not None: + if url_list_country is not None: deck.add_test('blocking/http_requests', ['-f', url_list_country]) deck.add_test('blocking/http_requests', ['-f', url_list_global]) diff --git a/ooni/geoip.py b/ooni/geoip.py index 417c7d5..dcc799e 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -6,7 +6,6 @@ import random from hashlib import sha256 from twisted.web import client, http_headers -from ooni.utils.net import hasRawSocketPermission client._HTTP11ClientFactory.noisy = False @@ -183,7 +182,7 @@ class ProbeIP(object): else: try: yield self.askTor() - log.msg("Found your IP via Tor %s" % self.address) + log.msg("Found your IP via Tor") self.resolveGeodata() defer.returnValue(self.address) except errors.TorStateNotFound: @@ -193,7 +192,7 @@ class ProbeIP(object): try: yield self.askGeoIPService() - log.msg("Found your IP via a GeoIP service: %s" % self.address) + log.msg("Found your IP via a GeoIP service") self.resolveGeodata() defer.returnValue(self.address) except Exception: diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 3be64a9..4da81bc 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -59,7 +59,6 @@ class Options(usage.Options): tracer = None def __init__(self): - self['test'] = None usage.Options.__init__(self) def getUsage(self, width=None): diff --git a/requirements.txt b/requirements.txt index fc6f629..229504b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ +# These are dependencies of ooniprobe depedencies that require specific +# versions. pyasn1>=0.1.8 + +# These are direct dependencies of ooniprobe. PyYAML>=3.10 Twisted>=13.2.0 ipaddr>=2.1.10

1 0

[ooni-probe/master] Run ooniresources without any command line options in the setup.py
by art＠torproject.org 29 Apr '16

29 Apr '16

commit a4244fcb6eeb402a606ab99fec6ac1a15e0d8960 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Apr 23 20:44:37 2016 +0200 Run ooniresources without any command line options in the setup.py * Print some progress when running oonireport * Use print function instead of old print --- ooni/report/tool.py | 23 ++++++++++++++--------- setup.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ooni/report/tool.py b/ooni/report/tool.py index 5946c42..5c7bcb2 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -1,4 +1,6 @@ +from __future__ import print_function import yaml +import sys from twisted.internet import defer @@ -49,9 +51,12 @@ def upload(report_file, collector=None, bouncer=None): report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id yield oonib_report_log.created(report_file, collector, report_id) + log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) - log.msg("Closing report.") + sys.stdout.write('.') + sys.stdout.flush() + log.msg("Closing report") yield oonib_reporter.finish() yield oonib_report_log.closed(report_file) @@ -68,24 +73,24 @@ def upload_all(collector=None, bouncer=None): def print_report(report_file, value): - print "* %s" % report_file - print " %s" % value['created_at'] + print("* %s" % report_file) + print(" %s" % value['created_at']) def status(): oonib_report_log = OONIBReportLog() - print "Reports to be uploaded" - print "----------------------" + print("Reports to be uploaded") + print("----------------------") for report_file, value in oonib_report_log.reports_to_upload: print_report(report_file, value) - print "Reports in progress" - print "-------------------" + print("Reports in progress") + print("-------------------") for report_file, value in oonib_report_log.reports_in_progress: print_report(report_file, value) - print "Incomplete reports" - print "------------------" + print("Incomplete reports") + print("------------------") for report_file, value in oonib_report_log.reports_incomplete: print_report(report_file, value) diff --git a/setup.py b/setup.py index 388de5f..1fd9a7b 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,7 @@ class install(_st_install): def ooniresources(self): ooniresources = find_executable("ooniresources") from subprocess import Popen - process = Popen([ooniresources, '--update-inputs', '--update-geoip'], + process = Popen([ooniresources], stdout=sys.stdout.fileno(), stderr=sys.stderr.fileno()) process.wait()

1 0