[tor-commits] [ooni-probe/master] Add script that verifys proper functionality of basic tests
art at torproject.org
art at torproject.org
Tue Nov 6 15:35:36 UTC 2012
commit 0b9e1629186a081dac4a72046caa8f1244c20154
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue Nov 6 16:34:34 2012 +0100
Add script that verifys proper functionality of basic tests
* As suggested by Jake
* There is a subset of inputs that will run with the tests to avoid having to
wait loads of time for the test running.
* This should not require more than 30 seconds on a decent network
---
before_i_commit.sh | 27 +++++++++
ooni/oonicli.py | 7 +--
ooni/utils/log.py | 14 +++--
oonib/oonibackend.py | 12 +++-
oonib/report/api.py | 63 ++++------------------
oonib/report/db/models.py | 83 -----------------------------
test_inputs/README | 48 +++++++++++++++++
test_inputs/dns_tamper_file.txt | 3 +
test_inputs/dns_tamper_test_resolvers.txt | 2 +
test_inputs/http_host_file.txt | 2 +
test_inputs/keyword_filtering_file.txt | 2 +
test_inputs/url_lists_file.txt | 2 +
12 files changed, 118 insertions(+), 147 deletions(-)
diff --git a/before_i_commit.sh b/before_i_commit.sh
new file mode 100755
index 0000000..2816ea5
--- /dev/null
+++ b/before_i_commit.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+# This script should be run before you commit to verify that the basic tests
+# are working as they should
+# Once you have run it you can inspect the log file via
+#
+# $ less before_i_commit.log
+# To clean up everything that is left by the running of this tool, do as
+# following:
+#
+# rm *.yamloo; rm before_i_commit.log
+#
+
+find . -type f -name "*.py[co]" -delete
+
+./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py
+
+./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index aa3cdc0..1988652 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -46,7 +46,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
optParameters = [
["reportfile", "o", None, "report file name"],
- ["logfile", "l", "test.log", "log file name"],
+ ["logfile", "l", None, "log file name"],
['temp-directory', None, '_ooni_temp',
'Path to use as working directory for tests.']
]
@@ -87,9 +87,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
def run():
- log.start()
- log.debug("Started logging")
-
if len(sys.argv) == 1:
sys.argv.append("--help")
config = Options()
@@ -101,6 +98,8 @@ def run():
if config['debug-stacktraces']:
defer.setDebugging(True)
+ log.start(config['logfile'])
+
classes = runner.findTestClassesFromConfig(config)
casesList, options = runner.loadTestsAndOptions(classes, config)
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index e93295a..771bad8 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -13,18 +13,22 @@ from twisted.python.logfile import DailyLogFile
from ooni.utils import otime
from ooni import oconfig
-# XXX make this a config option
-log_file = oconfig.basic.logfile
+def start(logfile=None):
+ daily_logfile = None
+
+ if not logfile:
+ logfile = oconfig.basic.logfile
+
+ log_folder = os.path.dirname(logfile)
+ log_filename = os.path.basename(logfile)
-def start(log_file=log_file):
- log_folder = os.path.join('/', *log_file.split('/')[:-1])
- log_filename = log_file.split('/')[-1]
daily_logfile = DailyLogFile(log_filename, log_folder)
txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(),
otime.utcPrettyDateNow()))
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver()
+
if oconfig.advanced.debug:
python_logging.logger.setLevel(logging.DEBUG)
else:
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 29f3f58..e2e800f 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -9,20 +9,26 @@ import json
import random
import string
-from twisted.application import internet, service
from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
+from twisted.application import internet, service
from twisted.web import resource, server, static
from twisted.web.microdom import escape
+from twisted.protocols import basic
from twisted.names import dns
+from ooni.utils import log
+
from oonib.report.api import reportingBackend
-from oonib.lib import config
from oonib.lib.ssl import SSLContext
+from oonib.lib import config
+
from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
from oonib.testhelpers.dns import ProxyDNSServer
from oonib.testhelpers.daphn3 import Daphn3Server
+
+log.start('/tmp/oonib.log')
+
# This tells twisted to set the
server.version = config.main.server_version
diff --git a/oonib/report/api.py b/oonib/report/api.py
index ad0f62c..ea6085d 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -8,46 +8,15 @@ This is the async pcap reporting system. It requires the client to have created
"""
import random
import string
+import json
+
from twisted.internet import reactor, defer
+
from cyclone import web
from oonib.report.db import models
backend_version = '0.0.1'
-def generateReportID():
- size = 100
- report_id = ''.join(random.choice(string.ascii_letters) for x in range(size))
- return report_id
-
- at defer.inlineCallbacks
-def newReport(software_name, software_version, test_name, test_version,
- progress, content):
-
- report_id = generateReportID()
-
- new_report = models.Report()
-
- new_report.report_id = unicode(report_id)
-
- new_report.software_name = unicode(software_name)
- new_report.software_version = unicode(software_version)
- new_report.test_name = unicode(test_name)
- new_report.test_version = unicode(test_version)
- new_report.progress = unicode(progress)
- new_report.content = unicode(content)
-
- print "Software Name: %s" % software_name
- print "Software Version: %s" % software_version
- print "Test Name: %s" % test_name
- print "Test Version: %s" % test_version
- print "Progress: %s" % progress
- print "Content: %s" % content
-
- yield new_report.save()
-
- defer.returnValue({'backend_version': backend_version, 'report_id':
- report_id})
-
def updateReport(report_id, content):
print "Report ID: %s" % report_id
print "Content: %s" % content
@@ -61,7 +30,7 @@ class NewReportHandler(web.RequestHandler):
@web.asynchronous
@defer.inlineCallbacks
- def get(self):
+ def post(self):
"""
Creates a new report with the input
@@ -84,27 +53,18 @@ class NewReportHandler(web.RequestHandler):
{'backend_version': 'XXX', 'report_id': 'XXX'}
"""
- # This is the list of supported arguments
- arguments = ['software_name', 'software_version',
- 'test_name','test_version',
- 'progress', 'content']
- report = {}
- for arg in arguments:
- if len(self.get_arguments(arg)) == 0:
- raise web.HTTPError(400, "%s not specified as argument of POST"
- % arg)
- report[arg] = self.get_argument(arg)
+ parsed_request = json.loads(self.request.body)
- try:
- test_helper = self.get_argument('test_helper')
+ # XXX here we should validate and sanitize the request
+ report_data = parsed_request
- except web.HTTPError:
- pass
+ new_report = models.Report()
- new_report = yield newReport(**report)
+ print "Got %s as request" % parsed_request
+ result = yield new_report.new(report_data)
- self.write(new_report)
+ self.write(result)
self.finish()
def put(self):
@@ -128,4 +88,3 @@ spec = [(r"/report/new", NewReportHandler),
(r"/report/pcap", PCAPReportHandler)]
reportingBackend = web.Application(spec)
-
diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py
deleted file mode 100644
index 21e60eb..0000000
--- a/oonib/report/db/models.py
+++ /dev/null
@@ -1,83 +0,0 @@
-__all__ = ['Report', 'TestHelperTMP']
-from storm.twisted.transact import transact
-from storm.locals import *
-
-from oonib.report.db import getStore, transactor
-
-class OModel(object):
-
- transactor = transactor
-
- @transact
- def create(query):
- store = Store(database)
- store.execute(query)
- store.commit()
-
- @transact
- def save(self):
- store = getStore()
- store.add(self)
- store.commit()
-
-class Report(OModel):
- """
- This represents an OONI Report as stored in the database.
-
- report_id: this is generated by the backend and is used by the client to
- reference a previous report and append to it. It should be
- treated as a shared secret between the probe and backend.
-
- software_name: this indicates the name of the software performing the test
- (this will default to ooniprobe)
-
- software_version: this is the version number of the software running the
- test.
-
- test_name: the name of the test on which the report is being created.
-
- test_version: indicates the version of the test
-
- progress: what is the current progress of the report. This allows clients
- to report event partial reports up to a certain percentage of
- progress. Once the report is complete progress will be 100.
-
- content: what is the content of the report. If the current progress is less
- than 100 we should append to the YAML data structure that is
- currently stored in such field.
- """
- __storm_table__ = 'reports'
-
- createQuery = "CREATE TABLE " + __storm_table__ +\
- "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
- "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
- "progress VARCHAR, content VARCHAR)"
-
-
- id = Int(primary=True)
-
- report_id = Unicode()
-
- software_name = Unicode()
- software_version = Unicode()
- test_name = Unicode()
- test_version = Unicode()
- progress = Unicode()
-
- content = Unicode()
-
-class TestHelperTMP(OModel):
- __storm_table__ = 'testhelpertmp'
-
- createQuery = "CREATE TABLE " + __storm_table__ +\
- "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
- " client_ip VARCHAR, creation_time VARCHAR)"
-
- id = Int(primary=True)
-
- report_id = Unicode()
-
- test_helper = Unicode()
- client_ip = Unicode()
-
- creation_time = Date()
diff --git a/test_inputs/README b/test_inputs/README
new file mode 100644
index 0000000..1eff781
--- /dev/null
+++ b/test_inputs/README
@@ -0,0 +1,48 @@
+In here you will find some very simple input lists that are useful for testing
+the correct functionality of the various OONIProbe tests.
+
+# DNS Tamper
+
+./bin/ooniprobe -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt
+
+less dns_tamper_test.yamloo
+
+# Captive Portal
+
+./bin/ooniprobe -o captive_portal_test.yamloo nettests/core/captiveportal.py
+
+less captive_portal_test.yamloo
+
+# HTTP Host
+
+./bin/ooniprobe -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt
+
+less http_host.yamloo
+
+# Keyword filtering
+
+./bin/ooniprobe -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
+
+less keyword_filtering.yamloo
+
+# URL List
+
+./bin/ooniprobe -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+less url_lists.yamloo
+
+# Squid transparent proxy
+
+./bin/ooniprobe -o squid.yamloo nettests/core/squid.py
+
+less squid.yamloo
+
+# HTTP Requests
+
+Not Implemented
+
+# Traceroute
+
+Not Implemented
+
+
diff --git a/test_inputs/dns_tamper_file.txt b/test_inputs/dns_tamper_file.txt
new file mode 100644
index 0000000..25f365c
--- /dev/null
+++ b/test_inputs/dns_tamper_file.txt
@@ -0,0 +1,3 @@
+torproject.org
+google.com
+measurementlab.net
diff --git a/test_inputs/dns_tamper_test_resolvers.txt b/test_inputs/dns_tamper_test_resolvers.txt
new file mode 100644
index 0000000..14c77e0
--- /dev/null
+++ b/test_inputs/dns_tamper_test_resolvers.txt
@@ -0,0 +1,2 @@
+8.8.8.8
+8.8.4.4
diff --git a/test_inputs/http_host_file.txt b/test_inputs/http_host_file.txt
new file mode 100644
index 0000000..12afb18
--- /dev/null
+++ b/test_inputs/http_host_file.txt
@@ -0,0 +1,2 @@
+torproject.org
+ooni.nu
diff --git a/test_inputs/keyword_filtering_file.txt b/test_inputs/keyword_filtering_file.txt
new file mode 100644
index 0000000..4583bae
--- /dev/null
+++ b/test_inputs/keyword_filtering_file.txt
@@ -0,0 +1,2 @@
+antani
+sblinda
diff --git a/test_inputs/url_lists_file.txt b/test_inputs/url_lists_file.txt
new file mode 100644
index 0000000..16a4f58
--- /dev/null
+++ b/test_inputs/url_lists_file.txt
@@ -0,0 +1,2 @@
+http://ooni.nu/test
+http://torproject.org/
More information about the tor-commits
mailing list