September 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge pull request #331 from anadahz/setup_dep.fix
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 8a9fcd57d4126a9ef485d282085c81580d235c5c Merge: 49849e5 d1436ab Author: aagbsn <aagbsn(a)extc.org> Date: Tue Sep 2 17:04:35 2014 +0000 Merge pull request #331 from anadahz/setup_dep.fix Add Debian jessie support in setup script setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] Add Debian jessie support
by art＠torproject.org 02 Sep '14

02 Sep '14

commit d1436ab890ea647e898a2459d2871fe1cece1728 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Jun 24 03:59:35 2014 +0300 Add Debian jessie support --- setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-dependencies.sh b/setup-dependencies.sh index 632d347..bf7eb44 100755 --- a/setup-dependencies.sh +++ b/setup-dependencies.sh @@ -867,7 +867,7 @@ install_virtualenv_securely() { } case $DISTRO_VERSION in - lucid|natty|wheezy|squeeze|precise|n/a|raring|saucy|trusty) + lucid|natty|jessie|wheezy|squeeze|precise|n/a|raring|saucy|trusty) # Create the build directories DO "mkdir -p ${BUILD_DIR}" "0"

1 0

[ooni-probe/master] Read the config file before running cli.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 491a78e35af0cffd799319f42ac2051b7673fd02 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 19:08:14 2014 +0200 Read the config file before running cli. --- ooni/resources/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index fdcbe07..3ee7945 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -2,6 +2,8 @@ import sys from twisted.python import usage +from ooni.settings import config + from ooni.resources import __version__ from ooni.resources import update @@ -21,6 +23,7 @@ class Options(usage.Options): def run(): options = Options() + config.read_config_file() try: options.parseOptions() except usage.UsageError as error_message:

1 0

[ooni-probe/master] Add oonideckgen and ooniresources to setup.py
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 2397c14fe4c0a8862976bf359b340c573af52780 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 18:32:44 2014 +0200 Add oonideckgen and ooniresources to setup.py --- bin/oonireport | 3 ++- ooni/settings.py | 1 + ooni/utils/__init__.py | 14 ++++++++++++++ setup.py | 5 ++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/bin/oonireport b/bin/oonireport index 4e6d48a..3a06b3b 100755 --- a/bin/oonireport +++ b/bin/oonireport @@ -7,6 +7,7 @@ sys.path.insert(0, os.path.abspath(os.getcwd())) from twisted.internet import defer, reactor +from ooni.utils import log from ooni.report import cli def failed(failure): @@ -20,7 +21,7 @@ def done(result): def start(): d = defer.maybeDeferred(cli.run) d.addCallback(done) - d.addErrback(done) + d.addErrback(failed) reactor.callWhenRunning(start) reactor.run() diff --git a/ooni/settings.py b/ooni/settings.py index a2a56b8..130ce2e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -54,6 +54,7 @@ class OConfig(object): self.decks_directory = os.path.join(self.ooni_home, 'decks') self.reports_directory = os.path.join(self.ooni_home, 'reports') self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml') + self.resources_directory = os.path.join(self.data_directory, "resources") if self.global_options.get('configfile'): config_file = self.global_options['configfile'] diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index ec1b1ce..5050e14 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -3,6 +3,8 @@ import random import glob import os +from zipfile import ZipFile + from ooni import otime from ooni import errors @@ -142,3 +144,15 @@ def sanitize_options(options): option = os.path.basename(option) sanitized_options.append(option) return sanitized_options + +def unzip(filename, dst): + + assert filename.endswith('.zip') + dst_path = os.path.join( + dst, + os.path.basename(filename).replace(".zip", "") + ) + with open(filename) as zfp: + zip_file = ZipFile(zfp) + zip_file.extractall(dst_path) + return dst_path diff --git a/setup.py b/setup.py index b19cb47..006c289 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ for root, dirs, file_names in os.walk('data/'): packages=[ 'ooni', 'ooni.api', + 'ooni.deckgen', 'ooni.kit', 'ooni.nettests', 'ooni.nettests.manipulation', @@ -81,6 +82,7 @@ packages=[ 'ooni.nettests.blocking', 'ooni.nettests.third_party', 'ooni.report', + 'ooni.resources', 'ooni.templates', 'ooni.tests', 'ooni.utils' @@ -113,7 +115,8 @@ setup( package_dir={'ooni': 'ooni'}, data_files=data_files, packages=packages, - scripts=["bin/ooniprobe", "bin/oonireport"], + scripts=["bin/oonideckgen", "bin/ooniprobe", + "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links, install_requires=install_requires, classifiers=(

1 0

[ooni-probe/master] Make the update-inputs command an option
by art＠torproject.org 02 Sep '14

02 Sep '14

commit ceef52c348bb3f4477c3d031d024c89eb7379039 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 19:01:31 2014 +0200 Make the update-inputs command an option --- ooni/resources/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index 4cc4c6d..fdcbe07 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -9,6 +9,9 @@ from ooni.resources import update class Options(usage.Options): synopsis = """%s""" + optFlags = [ + ["update-inputs", None, "Update the resources needed for inputs"] + ] optParameters = [] def opt_version(self): @@ -25,4 +28,9 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1) - return update.download_inputs() + if options['update-inputs']: + return update.download_inputs() + + print "%s: no command specified" % sys.argv[0] + print "%s: Try --help for usage details." % (sys.argv[0]) + sys.exit(1)

1 0

[ooni-probe/master] Add a tool for generating ooniprobe decks.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit b8c27c2c46aa3f8cd5b1462ba6d07cc3fa653d41 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 18:30:45 2014 +0200 Add a tool for generating ooniprobe decks. * Also adds tool for downloading ooniprobe resources. --- bin/oonideckgen | 27 ++++ bin/ooniresources | 27 ++++ ooni/deckgen/__init__.py | 1 + ooni/deckgen/cli.py | 151 ++++++++++++++++++++++ ooni/deckgen/processors/citizenlab_test_lists.py | 53 ++++++++ ooni/deckgen/processors/namebench_dns_servers.py | 53 ++++++++ ooni/resources/__init__.py | 22 ++++ ooni/resources/cli.py | 28 ++++ ooni/resources/update.py | 47 +++++++ 9 files changed, 409 insertions(+) diff --git a/bin/oonideckgen b/bin/oonideckgen new file mode 100755 index 0000000..c990160 --- /dev/null +++ b/bin/oonideckgen @@ -0,0 +1,27 @@ +#!/usr/bin/env python +import os +import sys + +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + +from twisted.internet import defer, reactor + +from ooni.utils import log +from ooni.deckgen import cli + +def failed(failure): + log.err("Failed to run oonideckgen") + log.exception(failure) + reactor.stop() + +def done(result): + reactor.stop() + +def start(): + d = defer.maybeDeferred(cli.run) + d.addCallback(done) + d.addErrback(failed) + +reactor.callWhenRunning(start) +reactor.run() diff --git a/bin/ooniresources b/bin/ooniresources new file mode 100755 index 0000000..e0123ac --- /dev/null +++ b/bin/ooniresources @@ -0,0 +1,27 @@ +#!/usr/bin/env python +import os +import sys + +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + +from twisted.internet import defer, reactor + +from ooni.utils import log +from ooni.resources import cli + +def failed(failure): + log.err("Failed to run ooniresources") + log.exception(failure) + reactor.stop() + +def done(result): + reactor.stop() + +def start(): + d = defer.maybeDeferred(cli.run) + d.addCallback(done) + d.addErrback(done) + +reactor.callWhenRunning(start) +reactor.run() diff --git a/ooni/deckgen/__init__.py b/ooni/deckgen/__init__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/ooni/deckgen/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py new file mode 100644 index 0000000..406417d --- /dev/null +++ b/ooni/deckgen/cli.py @@ -0,0 +1,151 @@ +import os +import sys +import copy +import errno + +import yaml + +from twisted.python import usage + +from . import __version__ +from ooni.resources import inputs +from ooni.settings import config + + +class Options(usage.Options): + synopsis = """%s [options] + """ + + optParameters = [ + ["country-code", "c", + None, + "Specify the two letter country code for which we should " + "generate the deck." + ], + ["output", "o", + None, + "Specify the directory where to write output." + ] + ] + + def opt_version(self): + print("oonideckgen version: %s" % __version__) + sys.exit(0) + + def postOptions(self): + if not self['output'] or not self['country-code']: + raise usage.UsageError( + "Both --output and --country-code are required" + ) + if len(self['country-code']) != 2: + raise usage.UsageError("--country-code must be 2 characters") + if not os.path.isdir(self['output']): + raise usage.UsageError("%s is not a directory" % self['output']) + + self['country-code'] = self['country-code'].lower() + + output_dir = os.path.abspath(self['output']) + output_dir = os.path.join(output_dir, + "deck-%s" % self['country-code']) + self['output'] = output_dir + + +class Deck(): + _base_entry = { + "options": { + "collector": None, + "help": 0, + "logfile": None, + "no-default-reporter": 0, + "parallelism": None, + "pcapfile": None, + "reportfile": None, + "resume": 0, + "testdeck": None + } + } + + def __init__(self): + self.deck = [] + + def add_test(self, test_file, subargs=[]): + deck_entry = copy.deepcopy(self._base_entry) + deck_entry['options']['test_file'] = test_file + deck_entry['options']['subargs'] = subargs + self.deck.append(deck_entry) + + def pprint(self): + print yaml.safe_dump(self.deck) + + def write_to_file(self, filename): + with open(filename, "w+") as f: + f.write(yaml.safe_dump(self.deck)) + + +def usage(): + print "%s <two letter country code> <output dir>" % sys.argv[0] + + +def run(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as error_message: + print "%s: %s" % (sys.argv[0], error_message) + print "%s: Try --help for usage details." % (sys.argv[0]) + sys.exit(1) + + config.read_config_file() + + try: + os.makedirs(options['output']) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + dns_servers_processor = inputs['namebench-dns-servers.csv']['processor'] + url_lists_processor = inputs['citizenlab-test-lists.zip']['processor'] + + try: + url_list_country = url_lists_processor.generate_country_input( + options['country-code'], + options['output'] + ) + + except Exception: + print "Could not generate country specific url list" + print "We will just use the global one." + url_list_country = None + + url_list_global = url_lists_processor.generate_global_input( + options['output'] + ) + dns_servers = dns_servers_processor.generate_country_input( + options['country-code'], + options['output'] + ) + + deck = Deck() + # deck.add_test('manipulation/http_host', ['-f', 'somefile.txt']) + deck.add_test('blocking/http_requests', ['-f', url_list_global]) + deck.add_test('blocking/dns_consistency', + ['-f', url_list_global, '-T', dns_servers]) + + if url_list_country is not None: + deck.add_test('blocking/dns_consistency', + ['-f', url_list_country, '-T', dns_servers]) + deck.add_test('blocking/http_requests', ['-f', url_list_country]) + + deck.add_test('manipulation/http_invalid_request_line') + deck.add_test('manipulation/http_header_field_manipulation') + # deck.add_test('manipulation/traceroute') + deck.pprint() + deck_filename = os.path.join(options['output'], + "%s-%s-user.deck" % (__version__, + options['country-code'])) + deck.write_to_file(deck_filename) + print "Deck written to %s" % deck_filename + print "Run ooniprobe like so:" + print "ooniprobe -i %s" % deck_filename + +if __name__ == "__main__": + run() diff --git a/ooni/deckgen/processors/__init__.py b/ooni/deckgen/processors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py new file mode 100644 index 0000000..645d714 --- /dev/null +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -0,0 +1,53 @@ +import os +import csv +from ooni.settings import config + +def generate_country_input(country_code, dst): + """ + Write to dst/citizenlab-urls-{country_code}.txt + the list for the given country code. + + Returns: + + the path to the generated input + """ + + country_code = country_code.lower() + filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code) + fw = open(filename, "w+") + + input_list = os.path.join(config.resources_directory, + "citizenlab-test-lists", + "test-lists-master", + "csv", country_code + ".csv") + + if not os.path.exists(input_list): + raise Exception("Could not find list for country %s" % country_code) + + with open(input_list) as f: + csvreader = csv.reader(f) + csvreader.next() + for row in csvreader: + fw.write("%s\n" % row[0]) + + fw.close() + return filename + + +def generate_global_input(dst): + + filename = os.path.join(dst, "citizenlab-urls-global.txt") + fw = open(filename, "w+") + + input_list = os.path.join(config.resources_directory, + "citizenlab-test-lists", + "test-lists-master", + "csv", "global.csv") + with open(input_list) as f: + csvreader = csv.reader(f) + csvreader.next() + for row in csvreader: + fw.write("%s\n" % row[0]) + + fw.close() + return filename diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py new file mode 100644 index 0000000..1079335 --- /dev/null +++ b/ooni/deckgen/processors/namebench_dns_servers.py @@ -0,0 +1,53 @@ +import os +import csv +import GeoIP + +from ooni.settings import config + + +class GeoIPDB(object): + _borg = {} + country = None + asn = None + + def __init__(self): + self.__dict__ = self._borg + if not self.country: + try: + print config.advanced.geoip_data_dir + country_file = os.path.join(config.advanced.geoip_data_dir, + 'GeoIP.dat') + self.country = GeoIP.open(country_file, + GeoIP.GEOIP_STANDARD) + except: + raise Exception("Edit the geoip_data_dir line in your config" + " file to point to your geoip files") + + +def generate_country_input(country_code, dst): + + csv_file = os.path.join(config.resources_directory, + "namebench-dns-servers.csv") + filename = os.path.join(dst, "dns-server-%s.txt" % country_code) + fw = open(filename, "w") + geoip_db = GeoIPDB() + reader = csv.reader(open(csv_file)) + for row in reader: + if row[2] == 'X-Internal-IP': + continue + elif row[2] == 'X-Unroutable': + continue + elif row[2] == 'X-Link_local': + continue + ipaddr = row[0] + cc = geoip_db.country.country_code_by_addr(ipaddr) + if not cc: + continue + if cc.lower() == country_code.lower(): + fw.write(ipaddr + "\n") + fw.close() + return filename + + +def generate_global_input(dst): + pass diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py new file mode 100644 index 0000000..0afc0af --- /dev/null +++ b/ooni/resources/__init__.py @@ -0,0 +1,22 @@ +from ooni.settings import config +from ooni.utils import unzip + +from ooni.deckgen.processors import citizenlab_test_lists +from ooni.deckgen.processors import namebench_dns_servers + +__version__ = "0.0.1" + +inputs = { + "namebench-dns-servers.csv": { + "url": "https://namebench.googlecode.com/svn/trunk/config/servers.csv", + "action": None, + "action_args": [], + "processor": namebench_dns_servers, + }, + "citizenlab-test-lists.zip": { + "url": "https://github.com/citizenlab/test-lists/archive/master.zip", + "action": unzip, + "action_args": [config.resources_directory], + "processor": citizenlab_test_lists + } +} diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py new file mode 100644 index 0000000..4cc4c6d --- /dev/null +++ b/ooni/resources/cli.py @@ -0,0 +1,28 @@ +import sys + +from twisted.python import usage + +from ooni.resources import __version__ +from ooni.resources import update + + +class Options(usage.Options): + synopsis = """%s""" + + optParameters = [] + + def opt_version(self): + print("ooniresources version: %s" % __version__) + sys.exit(0) + + +def run(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as error_message: + print "%s: %s" % (sys.argv[0], error_message) + print "%s: Try --help for usage details." % (sys.argv[0]) + sys.exit(1) + + return update.download_inputs() diff --git a/ooni/resources/update.py b/ooni/resources/update.py new file mode 100644 index 0000000..28a6ec7 --- /dev/null +++ b/ooni/resources/update.py @@ -0,0 +1,47 @@ +import os + +from twisted.internet import reactor, defer, protocol +from twisted.web.client import RedirectAgent, Agent + +from ooni.settings import config +from ooni.resources import inputs + +agent = RedirectAgent(Agent(reactor)) + + +class SaveToFile(protocol.Protocol): + def __init__(self, finished, filesize, filename): + self.finished = finished + self.remaining = filesize + self.outfile = open(filename, 'wb') + + def dataReceived(self, bytes): + if self.remaining: + display = bytes[:self.remaining] + self.outfile.write(display) + self.remaining -= len(display) + else: + self.outfile.close() + + def connectionLost(self, reason): + self.outfile.close() + self.finished.callback(None) + + +(a)defer.inlineCallbacks +def download_inputs(): + for filename, resource in inputs.items(): + print "Downloading %s" % filename + + filename = os.path.join(config.resources_directory, filename) + + response = yield agent.request("GET", resource['url']) + finished = defer.Deferred() + response.deliverBody(SaveToFile(finished, response.length, filename)) + yield finished + + if resource['action'] is not None: + yield defer.maybeDeferred(resource['action'], + filename, + *resource['action_args']) + print "%s written." % filename

1 0

[ooni-probe/master] Add support for downloading GeoIP files via the ooniresource tool.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit a61e3361a0bd3d931ab539301963940154b4277a Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 21:43:46 2014 +0200 Add support for downloading GeoIP files via the ooniresource tool. * Add better exception handling to ooniresource. * Add gunzip to utils. --- ooni/resources/__init__.py | 21 ++++++++++++++++++++- ooni/resources/cli.py | 34 ++++++++++++++++++++++++++-------- ooni/resources/update.py | 15 +++++++++++---- ooni/utils/__init__.py | 13 +++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py index 0afc0af..e60784f 100644 --- a/ooni/resources/__init__.py +++ b/ooni/resources/__init__.py @@ -1,9 +1,11 @@ from ooni.settings import config -from ooni.utils import unzip +from ooni.utils import unzip, gunzip from ooni.deckgen.processors import citizenlab_test_lists from ooni.deckgen.processors import namebench_dns_servers +config.read_config_file() + __version__ = "0.0.1" inputs = { @@ -20,3 +22,20 @@ inputs = { "processor": citizenlab_test_lists } } + +geoip = { + "GeoIPASNum.dat.gz": { + "url": "http://www.maxmind.com/download/" + "geoip/database/asnum/GeoIPASNum.dat.gz", + "action": gunzip, + "action_args": [config.advanced.geoip_data_dir], + "processor": None + }, + "GeoIP.dat.gz": { + "url": "http://geolite.maxmind.com/" + "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", + "action": gunzip, + "action_args": [config.advanced.geoip_data_dir], + "processor": None + } +} diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index 3ee7945..a140c8d 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -1,7 +1,9 @@ import sys +from twisted.internet import defer from twisted.python import usage +from ooni.utils import log from ooni.settings import config from ooni.resources import __version__ @@ -9,10 +11,11 @@ from ooni.resources import update class Options(usage.Options): - synopsis = """%s""" + synopsis = """%s""" % sys.argv[0] optFlags = [ - ["update-inputs", None, "Update the resources needed for inputs"] + ["update-inputs", None, "Update the resources needed for inputs."], + ["update-geoip", None, "Update the geoip related resources."] ] optParameters = [] @@ -21,9 +24,10 @@ class Options(usage.Options): sys.exit(0) +(a)defer.inlineCallbacks def run(): - options = Options() config.read_config_file() + options = Options() try: options.parseOptions() except usage.UsageError as error_message: @@ -31,9 +35,23 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1) - if options['update-inputs']: - return update.download_inputs() + if not any(options.values()): + print("%s: no command specified" % sys.argv[0]) + print options + sys.exit(1) - print "%s: no command specified" % sys.argv[0] - print "%s: Try --help for usage details." % (sys.argv[0]) - sys.exit(1) + if options['update-inputs']: + print "Downloading inputs" + try: + yield update.download_inputs() + except Exception as exc: + log.err("failed to download geoip files") + log.exception(exc) + + if options['update-geoip']: + print "Downloading geoip files" + try: + yield update.download_geoip() + except Exception as exc: + log.err("failed to download geoip files") + log.exception(exc) diff --git a/ooni/resources/update.py b/ooni/resources/update.py index 28a6ec7..4cb0f49 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -4,7 +4,7 @@ from twisted.internet import reactor, defer, protocol from twisted.web.client import RedirectAgent, Agent from ooni.settings import config -from ooni.resources import inputs +from ooni.resources import inputs, geoip agent = RedirectAgent(Agent(reactor)) @@ -29,12 +29,11 @@ class SaveToFile(protocol.Protocol): @defer.inlineCallbacks -def download_inputs(): - for filename, resource in inputs.items(): +def download_resource(resources): + for filename, resource in resources.items(): print "Downloading %s" % filename filename = os.path.join(config.resources_directory, filename) - response = yield agent.request("GET", resource['url']) finished = defer.Deferred() response.deliverBody(SaveToFile(finished, response.length, filename)) @@ -45,3 +44,11 @@ def download_inputs(): filename, *resource['action_args']) print "%s written." % filename + + +def download_inputs(): + return download_resource(inputs) + + +def download_geoip(): + return download_resource(geoip) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5050e14..30a93d5 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,8 +1,10 @@ +import shutil import string import random import glob import os +import gzip from zipfile import ZipFile from ooni import otime @@ -156,3 +158,14 @@ def unzip(filename, dst): zip_file = ZipFile(zfp) zip_file.extractall(dst_path) return dst_path + +def gunzip(filename, dst): + assert filename.endswith(".gz") + dst_path = os.path.join( + dst, + os.path.basename(filename).replace(".gz", "") + ) + with open(dst_path, "w+") as fw: + gzip_file = gzip.open(filename) + shutil.copyfileobj(gzip_file, fw) + gzip_file.close()

1 0

[ooni-probe/master] Add notice about using ooniresources to download geoip files.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit ed7d57c4d71809eb3f27f9ae0fc6542700d3e565 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 21:55:02 2014 +0200 Add notice about using ooniresources to download geoip files. --- ooni/geoip.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 14dde4b..86383d0 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -34,29 +34,30 @@ def IPToLocation(ipaddr): location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} + def error(): + log.err("Could not find GeoIP data file in %s." + "Try running ooniresources --update-geoip or" + " edit your ooniprobe.conf" % config.advanced.geoip_data_dir) + try: country_dat = GeoIP(country_file) location['countrycode'] = country_dat.country_code_by_addr(ipaddr) if not location['countrycode']: location['countrycode'] = 'ZZ' except IOError: - log.err("Could not find GeoIP data file. Go into %s " - "and make sure GeoIP.dat is present or change the location " - "in the config file" % config.advanced.geoip_data_dir) + error() + try: city_dat = GeoIP(city_file) location['city'] = city_dat.record_by_addr(ipaddr)['city'] except: - log.err("Could not find the city your IP is from. " - "Download the GeoLiteCity.dat file into the geoip_data_dir" - " or install geoip-database-contrib.") + error() + try: asn_dat = GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] except: - log.err("Could not find the ASN for your IP. " - "Download the GeoIPASNum.dat file into the geoip_data_dir" - " or install geoip-database-contrib.") + error() return location

1 0

[ooni-probe/master] Improvements to the installation and usage guide
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 8a2e53b5b3387b3cfd5ca98de009f6a83c62342b Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 12 22:20:23 2014 +0200 Improvements to the installation and usage guide * Add section describing how to generate decks * Add usage information for ooniresources * Add a guide on getting up and running with ooniprobe in 5 minutes * Move the development environment setup guide down --- README.rst | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 12d3fff..d85e352 100644 --- a/README.rst +++ b/README.rst @@ -44,6 +44,42 @@ Futhermore, ooniprobe takes no precautions to protect the install target machine from forensics analysis. If the fact that you have installed or used ooni probe is a liability for you, please be aware of this risk. +OONI in 5 minutes +================= + +On debian testing or unstable:: + + sudo apt-get install ooniprobe + +If you are running debian stable you can get it from backports via:: + + sudo sh -c 'echo "deb http://http.debian.net/debian wheezy-backports main" >> /etc/apt/sources.list' + sudo apt-get update && sudo apt-get install ooniprobe + +On unix systems:: + + sudo pip install ooniprobe + +The run:: + + YOUR_COUNTRY_CODE="it" + PATH_TO_MY_DECK=`pwd` + mkdir my_decks + sudo ooniresource --update-inputs --update-geoip + oonideckgen --country-code $YOUR_COUNTRY_CODE -o my_decks/ + ooniprobe -i my_decks/deck-$YOUR_COUNTRY_CODE/0.0.1-$YOUR_COUNTRY_CODE-user.deck + +The output from oonideckgen should give you the full path to the test deck. + +If you would like to contribute measurements to OONI daily you can also add +this to your crontab:: + + @daily ooni /usr/bin/ooniprobe -i REPLACE_WITH_PATH_TO_THE_DECK + +This command should automatically do it if you did not quit the shell:: + + (crontab -l 2>/dev/null; echo "@daily ooni /usr/bin/ooniprobe -i $PATH_TO_MY_DECK/my_decks/$YOUR_COUNTRY_CODE/0.0.1-$YOUR_COUNTRY_CODE-user.deck") | crontab - + Installation ============ @@ -80,15 +116,6 @@ without installing it is supported with the following commands:: ./setup-dependencies.sh python setup.py install -Setting up development environment ----------------------------------- - -On debian based systems this can be done with:: - - sudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper - mkvirtualenv ooniprobe - python setup.py install - pip install -r requirements-dev.txt Other platforms (with Vagrant) ------------------------------ @@ -143,6 +170,43 @@ found inside of ``~/.ooni/ooniprobe.conf``. By default ooniprobe will not include personal identifying information in the test result, nor create a pcap file. This behavior can be personalized. + +Updating resources +------------------ + +To generate decks you will have to update the input resources of ooniprobe. + +This can be done with:: + + ooniresource --update-inputs + +If you get a permission error, you may have to run the command as root or +change the ooniprobe data directory inside of `ooniprobe.conf`. + +On some platforms, for example debian contrib, you will not get all the geoip +related files needed. In that case it is possible to manually download them +with ``ooniresources``:: + + ooniresource --update-geoip + +Generating decks +---------------- + +You can generate decks for your country thanks to the oonideckgen command. + +If you wish, for example, to generate a deck to be run in the country of Italy, +you can do so (be sure to have updated the input resources first) by running:: + + oonideck --country-code IT --output ~/ + +You will now have in your home a folder called `deck-it`, containing the ooni +deck (ends with .deck) and the inputs. +Note: that you should not move the `deck-*` directory once it has been +generated as the paths to the inputs referenced by the test in the deck are +absolute. If you want your deck to live in another directory you must +regenerated it. + + Running decks ------------- @@ -244,7 +308,6 @@ If your distributation supports capabilities you can avoid needing to run OONI a setcap cap_net_admin,cap_net_raw+eip /path/to/your/virtualenv's/python - Reporting bugs ============== @@ -280,3 +343,13 @@ that is different from master) with:: git format-patch master --stdout > my_first_ooniprobe.patch + +Setting up development environment +---------------------------------- + +On debian based systems this can be done with:: + + sudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper + mkvirtualenv ooniprobe + python setup.py install + pip install -r requirements-dev.txt

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.5'
by nickm＠torproject.org 02 Sep '14

02 Sep '14

commit d19cbf3ab17a5afcbe0e94a16686e9ff9129dbfc Merge: 962765a dcb4ee5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Sep 2 19:00:00 2014 -0400 Merge remote-tracking branch 'origin/maint-0.2.5' doc/tor.1.txt | 5 +++++ 1 file changed, 5 insertions(+)

1 0