[tor-commits] [ooni-probe/master] Improve setup.py, fixup paths for reporting.
art at torproject.org
art at torproject.org
Wed Jun 19 12:32:46 UTC 2013
commit 3b889359329f6a90fd6d5538f6bf4e417cdf9b00
Author: Arturo Filastò <art at fuffa.org>
Date: Mon Apr 29 13:17:37 2013 +0200
Improve setup.py, fixup paths for reporting.
---
data/ooniprobe.conf.sample | 13 +++++-----
decks/before_i_commit.testdeck | 8 +++---
ooni/__init__.py | 2 +-
ooni/api/spec.py | 55 +++++++++++++++++++++++++++++++++++++---
ooni/nettest.py | 8 ++++--
ooni/reporter.py | 6 +++++
setup.py | 17 ++++++++++---
7 files changed, 89 insertions(+), 20 deletions(-)
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index 5528199..2b79d3e 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -4,7 +4,7 @@
basic:
# Where OONIProbe should be writing it's log file
- logfile: ooniprobe.log
+ logfile: /var/log/ooniprobe.log
privacy:
# Should we include the IP address of the probe in the report?
includeip: false
@@ -18,14 +18,13 @@ privacy:
includepcap: false
reports:
# This is a packet capture file (.pcap) to load as a test:
- pcap: Null
+ pcap: null
+ collector: 'httpo://nkvphnp3p6agi5qq.onion'
advanced:
- # XXX change this to point to the directory where you have stored the GeoIP
- # database file. This should be the directory in which OONI is installed
- # /path/to/ooni-probe/data/
- geoip_data_dir: /usr/share/GeoIP/
+ geoip_data_dir: /usr/share/ooni/
debug: true
- tor_binary: /usr/sbin/tor
+ # enable if auto detection fails
+ #tor_binary: /usr/sbin/tor
# For auto detection
interface: auto
# Of specify a specific interface
diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck
index d1b4062..1159a0d 100644
--- a/decks/before_i_commit.testdeck
+++ b/decks/before_i_commit.testdeck
@@ -5,7 +5,7 @@
pcapfile: null
reportfile: reports/captive_portal_test.yamloo
subargs: []
- test_file: nettests/manipulation/captiveportal.py
+ test_file: data/nettests/manipulation/captiveportal.py
- options:
collector: null
help: 0
@@ -13,7 +13,7 @@
pcapfile: null
reportfile: reports/dns_tamper_test.yamloo
subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt]
- test_file: nettests/blocking/dnsconsistency.py
+ test_file: data/nettests/blocking/dnsconsistency.py
- options:
collector: null
help: 0
@@ -21,7 +21,7 @@
pcapfile: null
reportfile: reports/http_host.yamloo
subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt]
- test_file: nettests/manipulation/http_host.py
+ test_file: data/nettests/manipulation/http_host.py
- options:
collector: null
help: 0
@@ -29,4 +29,4 @@
pcapfile: null
reportfile: reports/header_field_manipulation.yamloo
subargs: [-b, 'http://93.95.227.200']
- test_file: nettests/manipulation/http_header_field_manipulation.py
+ test_file: data/nettests/manipulation/http_header_field_manipulation.py
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 815e16e..1810a7f 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -7,7 +7,7 @@ from . import templates
from . import utils
__author__ = "Arturo Filastò"
-__version__ = "0.0.11"
+__version__ = "0.0.12"
__all__ = ['config', 'inputunit', 'kit',
'lib', 'nettest', 'oonicli', 'reporter',
diff --git a/ooni/api/spec.py b/ooni/api/spec.py
index 39df2fd..071beaf 100644
--- a/ooni/api/spec.py
+++ b/ooni/api/spec.py
@@ -7,7 +7,7 @@ import types
from twisted.python import usage
from cyclone import web, escape
-from ooni.reporter import YAMLReporter, OONIBReporter
+from ooni.reporter import YAMLReporter, OONIBReporter, collector_supported
from ooni import errors
from ooni.nettest import NetTestLoader, MissingRequiredOption
from ooni.settings import config
@@ -43,11 +43,21 @@ def list_inputs():
return input_list
class Inputs(ORequestHandler):
+ """
+ This handler is responsible for listing and adding new inputs.
+ """
def get(self):
+ """
+ Obtain the list of currently installed inputs. Inputs are stored inside
+ of $OONI_HOME/inputs/.
+ """
input_list = list_inputs()
self.write(input_list)
def post(self):
+ """
+ Add a new input to the currently installed inputs.
+ """
input_file = self.request.files.get("file")[0]
filename = input_file['filename']
@@ -72,6 +82,15 @@ class ListTests(ORequestHandler):
self.write(test_list)
def get_net_test_loader(test_options, test_file):
+ """
+ Args:
+ test_options: (dict) containing as keys the option names.
+
+ test_file: (string) the path to the test_file to be run.
+ Returns:
+ an instance of :class:`ooni.nettest.NetTestLoader` with the specified
+ test_file and the specified options.
+ """
options = []
for k, v in test_options.items():
options.append('--'+k)
@@ -82,10 +101,26 @@ def get_net_test_loader(test_options, test_file):
return net_test_loader
def get_reporters(net_test_loader):
+ """
+ Determines which reports are able to run and returns an instance of them.
+
+ We always report to flat file via the :class:`ooni.reporters.YAMLReporter`
+ and the :class:`ooni.reporters.OONIBReporter`.
+
+ The later will be used only if we determine that Tor is running.
+
+ Returns:
+ a list of reporter instances
+ """
test_details = net_test_loader.testDetails
+ reporters = []
yaml_reporter = YAMLReporter(test_details, config.reports_directory)
- #oonib_reporter = OONIBReporter(test_details, collector)
- return [yaml_reporter]
+ reporters.append(yaml_reporter)
+
+ if config.reports.collector and collector_supported(config.reports.collector):
+ oonib_reporter = OONIBReporter(test_details, collector)
+ reporters.append(oonib_reporter)
+ return reporters
class StartTest(ORequestHandler):
def post(self, test_name):
@@ -114,6 +149,16 @@ class StopTest(ORequestHandler):
pass
def get_test_results(test_id):
+ """
+ Returns:
+ a list of test dicts that correspond to the test results for the given
+ test_id.
+ The dict is made like so:
+ {
+ 'name': The name of the report,
+ 'content': The content of the report
+ }
+ """
test_results = []
for test_result in os.listdir(config.reports_directory):
if test_result.startswith('report-'+test_id):
@@ -126,6 +171,10 @@ def get_test_results(test_id):
class TestStatus(ORequestHandler):
def get(self, test_id):
+ """
+ Returns the requested test_id details and the stored results for such
+ test.
+ """
try:
test = copy.deepcopy(oonidApplication.director.netTests[test_id])
test.pop('path')
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 964bee3..66639f8 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -238,6 +238,10 @@ class NetTestLoader(object):
if not hasattr(usage_options, 'optParameters'):
usage_options.optParameters = []
+ else:
+ for parameter in usage_options.optParameters:
+ if len(parameter) == 5:
+ parameter.pop()
if klass.inputFile:
usage_options.optParameters.append(klass.inputFile)
@@ -267,8 +271,8 @@ class NetTestLoader(object):
def loadNetTestString(self, net_test_string):
"""
Load NetTest from a string.
- WARNING input to this function *MUST* be sanitized and *NEVER* be
- untrusted.
+ WARNING input to this function *MUST* be sanitized and *NEVER* take
+ untrusted input.
Failure to do so will result in code exec.
net_test_string:
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 109eccf..1095cb1 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -227,6 +227,12 @@ class YAMLReporter(OReporter):
def finish(self):
self._stream.close()
+def collector_supported(collector_address):
+ if collector_address.startswith('httpo') \
+ and (not (config.tor_state or config.tor.socks_port)):
+ return False
+ return True
+
class OONIBReporter(OReporter):
def __init__(self, test_details, collector_address):
self.collectorAddress = collector_address
diff --git a/setup.py b/setup.py
index 41b9050..454984d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,9 @@
#!/usr/bin/env python
#-*- coding: utf-8 -*-
-from setuptools import setup
+import os
+import sys
+from distutils.core import setup
install_requires = [
'txsocksx>=0.0.2',
@@ -16,6 +18,14 @@ dependency_links = [
'https://github.com/hellais/pypcap/archive/v1.1.1.tar.gz#egg=pypcap-1.1.1'
]
+files = []
+for root, dirs, file_names in os.walk('data/'):
+ for file_name in file_names:
+ if not file_name.endswith('.pyc'):
+ files.append(os.path.join(root, file_name))
+
+data_files = [('/usr/share/ooni/', files)]
+
with open('requirements.txt') as f:
for line in f:
if line.startswith("#") or line.startswith('http'):
@@ -24,12 +34,13 @@ with open('requirements.txt') as f:
setup(
name="ooni-probe",
- version="0.0.11",
+ version="0.0.12",
author="Arturo Filastò",
author_email = "art at torproject.org",
url="https://ooni.torproject.org/",
package_dir={'ooni': 'ooni'},
- packages=['ooni', 'ooni.templates', 'ooni.utils'],
+ data_files=data_files,
+ packages=['ooni', 'ooni.api', 'ooni.templates', 'ooni.tests', 'ooni.utils'],
scripts=["bin/ooniprobe"],
dependency_links=dependency_links,
install_requires=install_requires,
More information about the tor-commits
mailing list