[tor-commits] [ooni-probe/master] Merge branch 'master' into packetcapture
art at torproject.org
art at torproject.org
Fri Nov 9 12:22:50 UTC 2012
commit 0bbb2d5314d4470c4cfdf9fc5a398a9991b3b8ce
Merge: 9e632c5 c44c528
Author: Arturo Filastò <art at fuffa.org>
Date: Fri Nov 9 10:36:52 2012 +0100
Merge branch 'master' into packetcapture
* master:
Finish refactor of the core of OONI. * Everything is passing * XXX there is a bug in blocking tests not making the reactor stop
Completely refactor the logic for running tests * We no longer rely on calling trial * The code is now *much* more clean and readable * The purpose of this is to make room for the threadpool to capture packets
Port china trigger to new API
Rename keyword filtering test to HTTP keyword filtering
Refactor install documentation.
Convert header of ooniprobe script to a non docstring
Conflicts:
ooni/config.py
ooni/runner.py
ooniprobe.conf
before_i_commit.sh | 4 +-
bin/ooniprobe | 19 ++--
docs/source/install.rst | 23 +---
nettests/core/chinatrigger.py | 107 ++++++++++++++++
nettests/core/http_keyword_filtering.py | 42 +++++++
nettests/core/keyword_filtering.py | 39 ------
nettests/core/url_list.py | 3 +-
nettests/simpletest.py | 2 -
ooni/config.py | 5 +-
ooni/inputunit.py | 16 ---
ooni/nettest.py | 81 +++----------
ooni/oonicli.py | 24 ++--
ooni/reporter.py | 182 +++++----------------------
ooni/runner.py | 206 ++++++++++++++++---------------
ooni/templates/scapyt.py | 12 +-
ooni/utils/net.py | 20 ++-
ooniprobe.conf | 3 +
to-be-ported/old-api/chinatrigger.py | 140 ---------------------
18 files changed, 355 insertions(+), 573 deletions(-)
diff --cc ooni/config.py
index faf830f,d3907af..de5f45b
--- a/ooni/config.py
+++ b/ooni/config.py
@@@ -41,6 -41,3 +41,7 @@@ advanced = Storage(
for k, v in configuration['advanced'].items():
advanced[k] = v
+threadpool = ThreadPool(0, advanced.threadpool_size)
+threadpool.start()
- sniffer_d = None
++# This is used to keep track of the state of the sniffer
++sniffer_running = None
diff --cc ooni/oonicli.py
index ae78583,be29b4c..2a6f0cd
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@@ -23,11 -23,9 +23,11 @@@ from twisted.application import ap
from twisted.python import usage, failure
from twisted.python.util import spewer
- from ooni import nettest, runner, reporter
+ from ooni import nettest, runner, reporter, config
from ooni.inputunit import InputUnitFactory
- from ooni.reporter import ReporterFactory
- from ooni.nettest import InputTestSuite
++
++from ooni.utils import net
from ooni.utils import log
@@@ -94,11 -88,11 +90,11 @@@ def run()
if cmd_line_options['debug-stacktraces']:
defer.setDebugging(True)
+ log.start(cmd_line_options['logfile'])
classes = runner.findTestClassesFromConfig(cmd_line_options)
- casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options)
+ test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options)
-
++ d =
+ runner.runTestCases(test_cases, options, cmd_line_options)
+ reactor.run()
- for idx, cases in enumerate(casesList):
- orunner = runner.ORunner(cases, options[idx], cmd_line_options)
- log.start(cmd_line_options['logfile'])
- orunner.run()
diff --cc ooni/reporter.py
index c9654e8,08a72b2..cdbf355
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@@ -30,8 -30,8 +30,6 @@@ except
Packet = object
packet = FooClass
--pyunit = __import__('unittest')
--
class OSafeRepresenter(SafeRepresenter):
"""
This is a custom YAML representer that allows us to represent reports
@@@ -223,53 -203,13 +201,13 @@@ class OReporter(YamlReporter)
'test_name': options['name'],
'test_version': options['version'],
}
-
self.writeReportEntry(test_details)
- def create(self):
- r = OONIReporter(self._stream, self.tbformat, self.realtime,
- self._publisher)
- self._reporters.append(OONIReporter)
- return r
-
-
- class OONIReporter(OReporter):
- """
- This is a special reporter that has knowledge about the fact that there can
- exist more test runs of the same kind per run.
- These multiple test runs are kept track of through idx.
-
- An instance of such reporter should be created per InputUnit. Every input
- unit will invoke size_of_input_unit * test_cases times startTest().
- """
- def __init__(self, stream=sys.stdout, tbformat='default', realtime=False,
- publisher=None):
- super(OONIReporter, self).__init__(stream=stream,
- tbformat=tbformat, realtime=realtime, publisher=publisher)
-
- self._tests = {}
-
- def getTestIndex(self, test):
- try:
- idx = test._idx
- except:
- idx = 0
- return idx
-
-
- def startTest(self, test):
- super(OONIReporter, self).startTest(test)
-
- idx = self.getTestIndex(test)
- if not self._startTime:
- self._startTime = self._getTime()
-
- log.debug("startTest on %s" % idx)
- test.report = {}
-
- self._tests[idx] = {}
- self._tests[idx]['test_started'] = self._getTime()
+ def testDone(self, test):
+ test_report = dict(test.report)
-
+
+ # XXX the scapy test has an example of how
+ # to do this properly.
if isinstance(test.input, packet.Packet):
test_input = repr(test.input)
else:
diff --cc ooni/runner.py
index a6e0908,c67f2a7..083de35
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@@ -10,20 -10,20 +10,21 @@@
# :version: 0.1.0-pre-alpha
import os
+ import sys
+ import time
import inspect
+ import traceback
from twisted.python import reflect, usage
-
- from twisted.trial.runner import isTestCase
+ from twisted.internet import defer
from twisted.trial.runner import filenameToModule
+from twisted.internet import reactor, threads
from ooni.inputunit import InputUnitFactory
- from ooni.nettest import InputTestSuite
+ from ooni.nettest import NetTestCase
- from ooni.reporter import ReporterFactory
- from ooni.utils import log, date, net
- from ooni import config
+ from ooni import reporter
+ from ooni.utils import log, date
def processTest(obj, cmd_line_options):
"""
@@@ -64,13 -63,14 +64,14 @@@
if obj.optFlags:
log.debug("Got optFlags")
optFlags = obj.optFlags
-
options = Options()
-
+
- options.parseOptions(cmd_line_options['subArgs'])
- obj.localOptions = options
+ if options:
+ options.parseOptions(cmd_line_options['subArgs'])
+ obj.localOptions = options
- if input_file:
+ if input_file and options:
+ log.debug("Got input file")
obj.inputFile = options[input_file[0]]
try:
diff --cc ooni/utils/net.py
index 3bebcdb,46c29e5..a5a512d
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@@ -2,21 -2,19 +2,27 @@@
#
# net.py
# --------
--# OONI utilities for network infrastructure and hardware.
--#
--# :authors: Isis Lovecruft, Arturo Filasto
--# :version: 0.0.1-pre-alpha
--# :license: (c) 2012 Isis Lovecruft, Arturo Filasto
--# see attached LICENCE file
++# OONI utilities for networking related operations
+
++from scapy.all import utils
++from twisted.internet import defer
++from ooni.utils import log
++from ooni.config import threadpool
def getClientAddress():
address = {'asn': 'REPLACE_ME',
'ip': 'REPLACE_ME'}
return address
++def writePacketToPcap(pkt):
++ from scapy.all import utils
++ log.debug("Writing to pcap file %s" % pkt)
++ utils.wrpcap('/tmp/foo.pcap', pkt)
++
+def capturePackets():
+ from scapy.all import sniff
- sniff()
++ return defer.deferToThread(sniff, writePacketToPcap,
++ lfilter=writePacketToPcap)
class PermissionsError(SystemExit):
def __init__(self, *args, **kwargs):
diff --cc ooniprobe.conf
index 058394e,2039951..b7ea1f3
--- a/ooniprobe.conf
+++ b/ooniprobe.conf
@@@ -22,5 -22,4 +22,8 @@@ advanced
# /path/to/ooni-probe/data/
geoip_data_dir: /home/x/code/networking/ooni-probe/data/
debug: true
++<<<<<<< HEAD
+ threadpool_size: 10
++=======
++>>>>>>> master
More information about the tor-commits
mailing list