commit 3b889359329f6a90fd6d5538f6bf4e417cdf9b00 Author: Arturo Filastò art@fuffa.org Date: Mon Apr 29 13:17:37 2013 +0200

Improve setup.py, fixup paths for reporting. --- data/ooniprobe.conf.sample | 13 +++++----- decks/before_i_commit.testdeck | 8 +++--- ooni/__init__.py | 2 +- ooni/api/spec.py | 55 +++++++++++++++++++++++++++++++++++++--- ooni/nettest.py | 8 ++++-- ooni/reporter.py | 6 +++++ setup.py | 17 ++++++++++--- 7 files changed, 89 insertions(+), 20 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 5528199..2b79d3e 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -4,7 +4,7 @@

basic: # Where OONIProbe should be writing it's log file - logfile: ooniprobe.log + logfile: /var/log/ooniprobe.log privacy: # Should we include the IP address of the probe in the report? includeip: false @@ -18,14 +18,13 @@ privacy: includepcap: false reports: # This is a packet capture file (.pcap) to load as a test: - pcap: Null + pcap: null + collector: 'httpo://nkvphnp3p6agi5qq.onion' advanced: - # XXX change this to point to the directory where you have stored the GeoIP - # database file. This should be the directory in which OONI is installed - # /path/to/ooni-probe/data/ - geoip_data_dir: /usr/share/GeoIP/ + geoip_data_dir: /usr/share/ooni/ debug: true - tor_binary: /usr/sbin/tor + # enable if auto detection fails + #tor_binary: /usr/sbin/tor # For auto detection interface: auto # Of specify a specific interface diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index d1b4062..1159a0d 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -5,7 +5,7 @@ pcapfile: null reportfile: reports/captive_portal_test.yamloo subargs: [] - test_file: nettests/manipulation/captiveportal.py + test_file: data/nettests/manipulation/captiveportal.py - options: collector: null help: 0 @@ -13,7 +13,7 @@ pcapfile: null reportfile: reports/dns_tamper_test.yamloo subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt] - test_file: nettests/blocking/dnsconsistency.py + test_file: data/nettests/blocking/dnsconsistency.py - options: collector: null help: 0 @@ -21,7 +21,7 @@ pcapfile: null reportfile: reports/http_host.yamloo subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt] - test_file: nettests/manipulation/http_host.py + test_file: data/nettests/manipulation/http_host.py - options: collector: null help: 0 @@ -29,4 +29,4 @@ pcapfile: null reportfile: reports/header_field_manipulation.yamloo subargs: [-b, 'http://93.95.227.200'] - test_file: nettests/manipulation/http_header_field_manipulation.py + test_file: data/nettests/manipulation/http_header_field_manipulation.py diff --git a/ooni/__init__.py b/ooni/__init__.py index 815e16e..1810a7f 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -7,7 +7,7 @@ from . import templates from . import utils

__author__ = "Arturo Filastò" -__version__ = "0.0.11" +__version__ = "0.0.12"

__all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', diff --git a/ooni/api/spec.py b/ooni/api/spec.py index 39df2fd..071beaf 100644 --- a/ooni/api/spec.py +++ b/ooni/api/spec.py @@ -7,7 +7,7 @@ import types from twisted.python import usage from cyclone import web, escape

-from ooni.reporter import YAMLReporter, OONIBReporter +from ooni.reporter import YAMLReporter, OONIBReporter, collector_supported from ooni import errors from ooni.nettest import NetTestLoader, MissingRequiredOption from ooni.settings import config @@ -43,11 +43,21 @@ def list_inputs(): return input_list

class Inputs(ORequestHandler): + """ + This handler is responsible for listing and adding new inputs. + """ def get(self): + """ + Obtain the list of currently installed inputs. Inputs are stored inside + of $OONI_HOME/inputs/. + """ input_list = list_inputs() self.write(input_list)

def post(self): + """ + Add a new input to the currently installed inputs. + """ input_file = self.request.files.get("file")[0] filename = input_file['filename']

@@ -72,6 +82,15 @@ class ListTests(ORequestHandler): self.write(test_list)

def get_net_test_loader(test_options, test_file): + """ + Args: + test_options: (dict) containing as keys the option names. + + test_file: (string) the path to the test_file to be run. + Returns: + an instance of :class:`ooni.nettest.NetTestLoader` with the specified + test_file and the specified options. + """ options = [] for k, v in test_options.items(): options.append('--'+k) @@ -82,10 +101,26 @@ def get_net_test_loader(test_options, test_file): return net_test_loader

def get_reporters(net_test_loader): + """ + Determines which reports are able to run and returns an instance of them. + + We always report to flat file via the :class:`ooni.reporters.YAMLReporter` + and the :class:`ooni.reporters.OONIBReporter`. + + The later will be used only if we determine that Tor is running. + + Returns: + a list of reporter instances + """ test_details = net_test_loader.testDetails + reporters = [] yaml_reporter = YAMLReporter(test_details, config.reports_directory) - #oonib_reporter = OONIBReporter(test_details, collector) - return [yaml_reporter] + reporters.append(yaml_reporter) + + if config.reports.collector and collector_supported(config.reports.collector): + oonib_reporter = OONIBReporter(test_details, collector) + reporters.append(oonib_reporter) + return reporters

class StartTest(ORequestHandler): def post(self, test_name): @@ -114,6 +149,16 @@ class StopTest(ORequestHandler): pass

def get_test_results(test_id): + """ + Returns: + a list of test dicts that correspond to the test results for the given + test_id. + The dict is made like so: + { + 'name': The name of the report, + 'content': The content of the report + } + """ test_results = [] for test_result in os.listdir(config.reports_directory): if test_result.startswith('report-'+test_id): @@ -126,6 +171,10 @@ def get_test_results(test_id):

class TestStatus(ORequestHandler): def get(self, test_id): + """ + Returns the requested test_id details and the stored results for such + test. + """ try: test = copy.deepcopy(oonidApplication.director.netTests[test_id]) test.pop('path') diff --git a/ooni/nettest.py b/ooni/nettest.py index 964bee3..66639f8 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -238,6 +238,10 @@ class NetTestLoader(object):

if not hasattr(usage_options, 'optParameters'): usage_options.optParameters = [] + else: + for parameter in usage_options.optParameters: + if len(parameter) == 5: + parameter.pop()

if klass.inputFile: usage_options.optParameters.append(klass.inputFile) @@ -267,8 +271,8 @@ class NetTestLoader(object): def loadNetTestString(self, net_test_string): """ Load NetTest from a string. - WARNING input to this function *MUST* be sanitized and *NEVER* be - untrusted. + WARNING input to this function *MUST* be sanitized and *NEVER* take + untrusted input. Failure to do so will result in code exec.

net_test_string: diff --git a/ooni/reporter.py b/ooni/reporter.py index 109eccf..1095cb1 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -227,6 +227,12 @@ class YAMLReporter(OReporter): def finish(self): self._stream.close()

+def collector_supported(collector_address): + if collector_address.startswith('httpo') \ + and (not (config.tor_state or config.tor.socks_port)): + return False + return True + class OONIBReporter(OReporter): def __init__(self, test_details, collector_address): self.collectorAddress = collector_address diff --git a/setup.py b/setup.py index 41b9050..454984d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,9 @@ #!/usr/bin/env python #-*- coding: utf-8 -*-

-from setuptools import setup +import os +import sys +from distutils.core import setup

install_requires = [ 'txsocksx>=0.0.2', @@ -16,6 +18,14 @@ dependency_links = [ 'https://github.com/hellais/pypcap/archive/v1.1.1.tar.gz#egg=pypcap-1.1.1' ]

+files = [] +for root, dirs, file_names in os.walk('data/'): + for file_name in file_names: + if not file_name.endswith('.pyc'): + files.append(os.path.join(root, file_name)) + +data_files = [('/usr/share/ooni/', files)] + with open('requirements.txt') as f: for line in f: if line.startswith("#") or line.startswith('http'): @@ -24,12 +34,13 @@ with open('requirements.txt') as f:

setup( name="ooni-probe", - version="0.0.11", + version="0.0.12", author="Arturo Filastò", author_email = "art@torproject.org", url="https://ooni.torproject.org/", package_dir={'ooni': 'ooni'}, - packages=['ooni', 'ooni.templates', 'ooni.utils'], + data_files=data_files, + packages=['ooni', 'ooni.api', 'ooni.templates', 'ooni.tests', 'ooni.utils'], scripts=["bin/ooniprobe"], dependency_links=dependency_links, install_requires=install_requires,