commit 0b9e1629186a081dac4a72046caa8f1244c20154 Author: Arturo Filastò arturo@filasto.net Date: Tue Nov 6 16:34:34 2012 +0100

Add script that verifys proper functionality of basic tests * As suggested by Jake * There is a subset of inputs that will run with the tests to avoid having to wait loads of time for the test running. * This should not require more than 30 seconds on a decent network --- before_i_commit.sh | 27 +++++++++ ooni/oonicli.py | 7 +-- ooni/utils/log.py | 14 +++-- oonib/oonibackend.py | 12 +++- oonib/report/api.py | 63 ++++------------------ oonib/report/db/models.py | 83 ----------------------------- test_inputs/README | 48 +++++++++++++++++ test_inputs/dns_tamper_file.txt | 3 + test_inputs/dns_tamper_test_resolvers.txt | 2 + test_inputs/http_host_file.txt | 2 + test_inputs/keyword_filtering_file.txt | 2 + test_inputs/url_lists_file.txt | 2 + 12 files changed, 118 insertions(+), 147 deletions(-)

diff --git a/before_i_commit.sh b/before_i_commit.sh new file mode 100755 index 0000000..2816ea5 --- /dev/null +++ b/before_i_commit.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# This script should be run before you commit to verify that the basic tests +# are working as they should +# Once you have run it you can inspect the log file via +# +# $ less before_i_commit.log +# To clean up everything that is left by the running of this tool, do as +# following: +# +# rm *.yamloo; rm before_i_commit.log +# + +find . -type f -name "*.py[co]" -delete + +./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + +./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt + +./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py + +./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt + +./bin/ooniprobe -l before_i_commit.log -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt + +./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index aa3cdc0..1988652 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -46,7 +46,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):

optParameters = [ ["reportfile", "o", None, "report file name"], - ["logfile", "l", "test.log", "log file name"], + ["logfile", "l", None, "log file name"], ['temp-directory', None, '_ooni_temp', 'Path to use as working directory for tests.'] ] @@ -87,9 +87,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):

def run(): - log.start() - log.debug("Started logging") - if len(sys.argv) == 1: sys.argv.append("--help") config = Options() @@ -101,6 +98,8 @@ def run(): if config['debug-stacktraces']: defer.setDebugging(True)

+ log.start(config['logfile']) + classes = runner.findTestClassesFromConfig(config) casesList, options = runner.loadTestsAndOptions(classes, config)

diff --git a/ooni/utils/log.py b/ooni/utils/log.py index e93295a..771bad8 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -13,18 +13,22 @@ from twisted.python.logfile import DailyLogFile from ooni.utils import otime from ooni import oconfig

-# XXX make this a config option -log_file = oconfig.basic.logfile +def start(logfile=None): + daily_logfile = None + + if not logfile: + logfile = oconfig.basic.logfile + + log_folder = os.path.dirname(logfile) + log_filename = os.path.basename(logfile)

-def start(log_file=log_file): - log_folder = os.path.join('/', *log_file.split('/')[:-1]) - log_filename = log_file.split('/')[-1] daily_logfile = DailyLogFile(log_filename, log_folder)

txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(), otime.utcPrettyDateNow())) logging.basicConfig() python_logging = txlog.PythonLoggingObserver() + if oconfig.advanced.debug: python_logging.logger.setLevel(logging.DEBUG) else: diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 29f3f58..e2e800f 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -9,20 +9,26 @@ import json import random import string

-from twisted.application import internet, service from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic +from twisted.application import internet, service from twisted.web import resource, server, static from twisted.web.microdom import escape +from twisted.protocols import basic from twisted.names import dns

+from ooni.utils import log + from oonib.report.api import reportingBackend -from oonib.lib import config from oonib.lib.ssl import SSLContext +from oonib.lib import config + from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer from oonib.testhelpers.dns import ProxyDNSServer from oonib.testhelpers.daphn3 import Daphn3Server

+ +log.start('/tmp/oonib.log') + # This tells twisted to set the server.version = config.main.server_version

diff --git a/oonib/report/api.py b/oonib/report/api.py index ad0f62c..ea6085d 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -8,46 +8,15 @@ This is the async pcap reporting system. It requires the client to have created """ import random import string +import json + from twisted.internet import reactor, defer + from cyclone import web

from oonib.report.db import models backend_version = '0.0.1'

-def generateReportID(): - size = 100 - report_id = ''.join(random.choice(string.ascii_letters) for x in range(size)) - return report_id - -@defer.inlineCallbacks -def newReport(software_name, software_version, test_name, test_version, - progress, content): - - report_id = generateReportID() - - new_report = models.Report() - - new_report.report_id = unicode(report_id) - - new_report.software_name = unicode(software_name) - new_report.software_version = unicode(software_version) - new_report.test_name = unicode(test_name) - new_report.test_version = unicode(test_version) - new_report.progress = unicode(progress) - new_report.content = unicode(content) - - print "Software Name: %s" % software_name - print "Software Version: %s" % software_version - print "Test Name: %s" % test_name - print "Test Version: %s" % test_version - print "Progress: %s" % progress - print "Content: %s" % content - - yield new_report.save() - - defer.returnValue({'backend_version': backend_version, 'report_id': - report_id}) - def updateReport(report_id, content): print "Report ID: %s" % report_id print "Content: %s" % content @@ -61,7 +30,7 @@ class NewReportHandler(web.RequestHandler):

@web.asynchronous @defer.inlineCallbacks - def get(self): + def post(self): """ Creates a new report with the input

@@ -84,27 +53,18 @@ class NewReportHandler(web.RequestHandler): {'backend_version': 'XXX', 'report_id': 'XXX'}

""" - # This is the list of supported arguments - arguments = ['software_name', 'software_version', - 'test_name','test_version', - 'progress', 'content'] - report = {} - for arg in arguments: - if len(self.get_arguments(arg)) == 0: - raise web.HTTPError(400, "%s not specified as argument of POST" - % arg) - report[arg] = self.get_argument(arg) + parsed_request = json.loads(self.request.body)

- try: - test_helper = self.get_argument('test_helper') + # XXX here we should validate and sanitize the request

+ report_data = parsed_request

- except web.HTTPError: - pass + new_report = models.Report()

- new_report = yield newReport(**report) + print "Got %s as request" % parsed_request + result = yield new_report.new(report_data)

- self.write(new_report) + self.write(result) self.finish()

def put(self): @@ -128,4 +88,3 @@ spec = [(r"/report/new", NewReportHandler), (r"/report/pcap", PCAPReportHandler)]

reportingBackend = web.Application(spec) - diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py deleted file mode 100644 index 21e60eb..0000000 --- a/oonib/report/db/models.py +++ /dev/null @@ -1,83 +0,0 @@ -__all__ = ['Report', 'TestHelperTMP'] -from storm.twisted.transact import transact -from storm.locals import * - -from oonib.report.db import getStore, transactor - -class OModel(object): - - transactor = transactor - - @transact - def create(query): - store = Store(database) - store.execute(query) - store.commit() - - @transact - def save(self): - store = getStore() - store.add(self) - store.commit() - -class Report(OModel): - """ - This represents an OONI Report as stored in the database. - - report_id: this is generated by the backend and is used by the client to - reference a previous report and append to it. It should be - treated as a shared secret between the probe and backend. - - software_name: this indicates the name of the software performing the test - (this will default to ooniprobe) - - software_version: this is the version number of the software running the - test. - - test_name: the name of the test on which the report is being created. - - test_version: indicates the version of the test - - progress: what is the current progress of the report. This allows clients - to report event partial reports up to a certain percentage of - progress. Once the report is complete progress will be 100. - - content: what is the content of the report. If the current progress is less - than 100 we should append to the YAML data structure that is - currently stored in such field. - """ - __storm_table__ = 'reports' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ - "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ - "progress VARCHAR, content VARCHAR)" - - - id = Int(primary=True) - - report_id = Unicode() - - software_name = Unicode() - software_version = Unicode() - test_name = Unicode() - test_version = Unicode() - progress = Unicode() - - content = Unicode() - -class TestHelperTMP(OModel): - __storm_table__ = 'testhelpertmp' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ - " client_ip VARCHAR, creation_time VARCHAR)" - - id = Int(primary=True) - - report_id = Unicode() - - test_helper = Unicode() - client_ip = Unicode() - - creation_time = Date() diff --git a/test_inputs/README b/test_inputs/README new file mode 100644 index 0000000..1eff781 --- /dev/null +++ b/test_inputs/README @@ -0,0 +1,48 @@ +In here you will find some very simple input lists that are useful for testing +the correct functionality of the various OONIProbe tests. + +# DNS Tamper + +./bin/ooniprobe -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt + +less dns_tamper_test.yamloo + +# Captive Portal + +./bin/ooniprobe -o captive_portal_test.yamloo nettests/core/captiveportal.py + +less captive_portal_test.yamloo + +# HTTP Host + +./bin/ooniprobe -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt + +less http_host.yamloo + +# Keyword filtering + +./bin/ooniprobe -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt + +less keyword_filtering.yamloo + +# URL List + +./bin/ooniprobe -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + +less url_lists.yamloo + +# Squid transparent proxy + +./bin/ooniprobe -o squid.yamloo nettests/core/squid.py + +less squid.yamloo + +# HTTP Requests + +Not Implemented + +# Traceroute + +Not Implemented + + diff --git a/test_inputs/dns_tamper_file.txt b/test_inputs/dns_tamper_file.txt new file mode 100644 index 0000000..25f365c --- /dev/null +++ b/test_inputs/dns_tamper_file.txt @@ -0,0 +1,3 @@ +torproject.org +google.com +measurementlab.net diff --git a/test_inputs/dns_tamper_test_resolvers.txt b/test_inputs/dns_tamper_test_resolvers.txt new file mode 100644 index 0000000..14c77e0 --- /dev/null +++ b/test_inputs/dns_tamper_test_resolvers.txt @@ -0,0 +1,2 @@ +8.8.8.8 +8.8.4.4 diff --git a/test_inputs/http_host_file.txt b/test_inputs/http_host_file.txt new file mode 100644 index 0000000..12afb18 --- /dev/null +++ b/test_inputs/http_host_file.txt @@ -0,0 +1,2 @@ +torproject.org +ooni.nu diff --git a/test_inputs/keyword_filtering_file.txt b/test_inputs/keyword_filtering_file.txt new file mode 100644 index 0000000..4583bae --- /dev/null +++ b/test_inputs/keyword_filtering_file.txt @@ -0,0 +1,2 @@ +antani +sblinda diff --git a/test_inputs/url_lists_file.txt b/test_inputs/url_lists_file.txt new file mode 100644 index 0000000..16a4f58 --- /dev/null +++ b/test_inputs/url_lists_file.txt @@ -0,0 +1,2 @@ +http://ooni.nu/test +http://torproject.org/