commit 46133a4a1cdc03e741f00c57f2d53fe20095827f Author: Arturo Filastò arturo@filasto.net Date: Tue Nov 6 15:53:45 2012 +0100

Make important refactoring to various parts. * Fix bug in captive portal test --- nettests/core/captiveportal.py | 5 +- nettests/echo.py | 4 +- ooni/__init__.py | 4 +- ooni/config.py | 7 --- ooni/lib/txscapy.py | 50 ++++++++++--------- ooni/oconfig.py | 1 + ooni/runner.py | 2 +- ooni/templates/scapyt.py | 108 +++++++++++++++++++++++++++++++++++---- 8 files changed, 132 insertions(+), 49 deletions(-)

diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py index fdc37a0..6641985 100644 --- a/nettests/core/captiveportal.py +++ b/nettests/core/captiveportal.py @@ -65,7 +65,8 @@ class CaptivePortal(nettest.NetTestCase):

name = "captivep" description = "Captive Portal Test" - requirements = None + version = '0.1' + author = "Isis LoveCruft isis@torproject.org"

def http_fetch(self, url, headers={}): """ @@ -636,5 +637,3 @@ class CaptivePortal(nettest.NetTestCase): log.msg("") log.msg("Captive portal test finished!")

- self.control(self.report) - diff --git a/nettests/echo.py b/nettests/echo.py index bc47519..611970e 100644 --- a/nettests/echo.py +++ b/nettests/echo.py @@ -70,9 +70,9 @@ class EchoTest(ScapyTest): form (*ifa_name, AF_FAMILY, *ifa_addr) '''

- if self.local_options: + if self.localOptions: log.debug("%s: local_options found" % self.name) - for key, value in self.local_options: + for key, value in self.localOptions.items(): log.debug("%s: setting self.%s = %s" % (key, value)) setattr(self, key, value)

diff --git a/ooni/__init__.py b/ooni/__init__.py index f025d30..659d4af 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,4 +1,4 @@ -from . import config +from . import oconfig from . import inputunit from . import kit from . import lib @@ -14,7 +14,7 @@ from . import utils #from . import plugoo #from . import plugins

-__all__ = ['config', 'inputunit', 'kit', +__all__ = ['oconfig', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] # XXX below are legacy related modules diff --git a/ooni/config.py b/ooni/config.py deleted file mode 100644 index 96a1ee0..0000000 --- a/ooni/config.py +++ /dev/null @@ -1,7 +0,0 @@ -from ooni.utils import Storage - -# XXX move this to an actual configuration file -basic = Storage() -advanced = Storage() - -advanced.debug = True diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index 4446be0..dffda36 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -18,7 +18,6 @@ import time

from twisted.internet import protocol, base, fdesc, error, defer from twisted.internet import reactor, threads -from twisted.python import log from zope.interface import implements

from scapy.all import Gen @@ -77,7 +76,7 @@ class ScapySocket(object): else: return self.ssocket.recv(self.MTU)

-class Scapy(object): +class TXScapy(object): """ A twisted based wrapper for scapy send and receive functionality.

@@ -90,12 +89,12 @@ class Scapy(object): write_only_answers = False pcapwriter = None recv = False + timeout_call = None + answers = [] + questions = []

def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, iface=None, nofilter=None, pcapfile=None, timeout=None, *arg, **kw): - if self.debug: - log.startLogging(sys.stdout) - self.maxPacketSize = maxPacketSize if not reactor: from twisted.internet import reactor @@ -173,13 +172,15 @@ class Scapy(object): @param question: the sent packet that matches that response.

""" - if self.pcapwriter and self.write_only_answers: self.pcapwriter.write(question) self.pcapwriter.write(answer) + self.answers.append(answers) + self.questions.append(question) self.answer_count += 1 - if self.answer_count >= self.total_count: - print "Got all the answers I need" + if self.answer_count >= self.total_count and self.running: + log.debug("Got all the answers I need") + self.finalClose() self.deferred.callback(None)

def processAnswer(self, pkt, hlst): @@ -211,7 +212,6 @@ class Scapy(object): """ pkt.show()

- def doRead(self): """ There is something to be read on the wire. Do all the processing on the @@ -262,10 +262,13 @@ class Scapy(object): self.socket.send(pkt)

def timeout(self, *arg, **kw): - log.debug("Caught a timeout with %s %s" % (arg, kw)) if not self.done: - self._reactor.callLater(self.timeoutSeconds, self.timeout, None) - else: + log.debug("I have not finished. Setting to call in %s" % + self.timeoutSeconds) + self.timeout_call = self._reactor.callLater(self.timeoutSeconds, self.timeout, None) + elif self.running: + log.debug("Cancelling timeout call") + self.finalClose() self.deferred.callback(None)

def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): @@ -287,11 +290,11 @@ class Scapy(object): @param filter: provide a BPF filter @param iface: listen answers only on the given interface """ - log.debug("Calling with %s" % pkts) + log.debug("TXScapy sending and receiving packets") self.recv = True if timeout: self.timeoutSeconds = timeout - self._reactor.callLater(timeout, self.timeout, None) + self.timeout_call = self._reactor.callLater(timeout, self.timeout, None) self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)

def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): @@ -327,7 +330,8 @@ class Scapy(object): pkt = self.outqueue.pop() except: self.done = True - if not self.recv: + if not self.recv and self.running: + log.debug("I am not in receiving state running callback") self.deferred.callback(None) return d = threads.deferToThreadPool(reactor, self.threadpool, @@ -353,25 +357,25 @@ class Scapy(object):

def finalClose(self): """ - Clean all the thread related stuff up. + Clean all the shutdown related functions. """ self.shutdownID = None self.threadpool.stop() + if self.timeout_call: + self.timeout_call.cancel() + self.timeout_call = None self.running = False

@defer.inlineCallbacks -def txsr(*arg, **kw): - tr = Scapy(*arg, **kw) - log.debug("Calling sr with %s, %s" % (arg, kw)) - tr.sr(*arg, **kw) +def txsr(*args, **kw): + tr = TXScapy(*args, **kw) + tr.sr(*args, **kw) yield tr.deferred tr.finalClose()

@defer.inlineCallbacks def txsend(*arg, **kw): - tr = Scapy(**kw) - log.debug("Calling send with %s, %s" % (arg, kw)) + tr = TXScapy(*arg, **kw) tr.send(*arg, **kw) yield tr.deferred tr.finalClose() - diff --git a/ooni/oconfig.py b/ooni/oconfig.py index 96a1ee0..202b569 100644 --- a/ooni/oconfig.py +++ b/ooni/oconfig.py @@ -2,6 +2,7 @@ from ooni.utils import Storage

# XXX move this to an actual configuration file basic = Storage() +basic.logfile = '/tmp/ooniprobe.log' advanced = Storage()

advanced.debug = True diff --git a/ooni/runner.py b/ooni/runner.py index e1ecaac..97b4003 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -205,7 +205,7 @@ class ORunner(object):

try: reportFile = open(config['reportfile'], 'a+') - except: + except TypeError: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') self.reporterFactory = ReporterFactory(reportFile, diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 8751587..9e18fbd 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -7,14 +7,14 @@ import random from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from twisted.internet import protocol, defer +from twisted.internet import protocol, defer, threads

-from scapy.all import IP, TCP +from scapy.all import IP, TCP, send, sr

from ooni.nettest import NetTestCase from ooni.utils import log

-from ooni.lib.txscapy import txsr, txsend +from ooni.lib.txscapy import TXScapy

class ScapyTest(NetTestCase): """ @@ -31,30 +31,116 @@ class ScapyTest(NetTestCase):

receive = True timeout = 1 - pcapfile = None + pcapfile = 'packet_capture.pcap' packet = IP()/TCP() reactor = None + + answered = None + unanswered = None + + def setUp(self): if not self.reactor: from twisted.internet import reactor self.reactor = reactor - self.request = {} - self.response = {} + self.questions = [] + self.answers = [] + self.processInputs() + + def processInputs(self): + """ + Place here the logic for validating and processing of inputs and + command line arguments. + """ + pass

def tearDown(self): - self.reactor.stop() + log.debug("Tearing down reactor") + + def finished(self, *arg): + log.debug("Calling final close") + + self.questions = self.txscapy.questions + self.answers = self.txscapy.answers + + log.debug("These are the questions: %s" % self.questions) + log.debug("These are the answers: %s" % self.answers) + + self.txscapy.finalClose() + + def sendReceivePackets(self): + packets = self.buildPackets()

- def sendReceivePackets(self, *kw): - d = txsr(self.request, pcapfile=self.pcapfile, + log.debug("Sending and receiving %s" % packets) + + self.txscapy = TXScapy(packets, pcapfile=self.pcapfile, + timeout=self.timeout, reactor=self.reactor) + + self.txscapy.sr(packets, pcapfile=self.pcapfile, timeout=self.timeout, reactor=self.reactor) + + d = self.txscapy.deferred + d.addCallback(self.finished) + + return d + + def sendPackets(self): + log.debug("Sending and receiving of packets %s" % packets) + + packets = self.buildPackets() + + self.txscapy = TXScapy(packets, pcapfile=self.pcapfile, + timeout=self.timeout, reactor=self.reactor) + + self.txscapy.send(packets, reactor=self.reactor).deferred + + d = self.txscapy.deferred + d.addCallback(self.finished) + return d

+ def buildPackets(self): + """ + Override this method to build scapy packets. + """ + pass + +class BlockingScapyTest(ScapyTest): + """ + This is a very basic Scapy Test template that does not do all the + multithreading kung-fu of txscapy, but maintains the same API. + + This will allow tests implemented using the BlockingScapyTest API to easily + migrate to the new API. + """ + name = "Blocking Scapy Test" + version = 0.1 + + timeout = None + + answered = None + unanswered = None + + def sendReceivePackets(self): + packets = self.buildPackets() + + log.debug("Sending and receiving %s" % packets) + + self.answered, self.unanswered = sr(packets, timeout=self.timeout) + + log.debug("%s %s" % (ans, unans)) + def sendPackets(self): - return txsend(self.buildPackets(), reactor=self.reactor) + packets = self.buildPackets() + + log.debug("Sending packets %s" % packets) + + send(packets) +

def buildPackets(self): """ Override this method to build scapy packets. """ - return self.packet + pass