November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Add txscapy to the commit
by art＠torproject.org 09 Nov '12

09 Nov '12

commit b97a99c9f9acba460d098b1c6241f625882d553b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Nov 9 13:26:56 2012 +0100 Add txscapy to the commit This should have been in the previous commit and I should not have tagged the release I must run git clean -r;./before_i_commit.sh; before pushing to master. Disregard the v0.0.2-alpha tag --- ooni/utils/txscapy.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 40 insertions(+), 0 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py new file mode 100644 index 0000000..a3a5610 --- /dev/null +++ b/ooni/utils/txscapy.py @@ -0,0 +1,40 @@ +# -*- coding:utf8 -*- +# txscapy +# ******* +# Here shall go functions related to using scapy with twisted. +# +# This software has been written to be part of OONI, the Open Observatory of +# Network Interference. More information on that here: http://ooni.nu/ + +import struct +import socket +import os +import sys +import time + +from twisted.internet import protocol, base, fdesc, error, defer +from twisted.internet import reactor, threads +from zope.interface import implements + +from scapy.all import Gen +from scapy.all import SetGen + +from ooni.utils import log + +from scapy.all import PcapWriter, MTU +from scapy.all import BasePacketList, conf, PcapReader + +class TXPcapWriter(PcapWriter): + def __init__(self, *arg, **kw): + PcapWriter.__init__(self, *arg, **kw) + fdesc.setNonBlocking(self.f) + +def txSniff(count=0, store=1, offline=None, + prn = None, lfilter=None, + L2socket=None, timeout=None, + opened_socket=None, stop_filter=None, + *arg, **karg): + """ + XXX we probably want to rewrite the scapy sniff function to better suite our needs. + """ +

1 0

[ooni-probe/master] Further progress on packet capture implementation
by art＠torproject.org 09 Nov '12

09 Nov '12

commit 9e632c5d7c8d7f73add3aa38900898a86f6ca9fc Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 8 21:46:28 2012 +0100 Further progress on packet capture implementation --- ooni/config.py | 3 ++- ooni/runner.py | 8 +++++--- ooni/utils/net.py | 4 +++- ooniprobe.conf | 2 -- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 64bc973..faf830f 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -41,5 +41,6 @@ advanced = Storage() for k, v in configuration['advanced'].items(): advanced[k] = v -threadpool = ThreadPool(0, config.advanced.threadpool_size) +threadpool = ThreadPool(0, advanced.threadpool_size) threadpool.start() +sniffer_d = None diff --git a/ooni/runner.py b/ooni/runner.py index 7e9a5d6..a6e0908 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -16,12 +16,13 @@ from twisted.python import reflect, usage from twisted.trial.runner import isTestCase from twisted.trial.runner import filenameToModule +from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory from ooni.nettest import InputTestSuite from ooni.reporter import ReporterFactory -from ooni.utils import log, date +from ooni.utils import log, date, net from ooni import config def processTest(obj, cmd_line_options): @@ -210,12 +211,13 @@ class ORunner(object): idx += (suite._idx - idx) log.debug("I am now at the index %s" % idx) - log.debug("Finished") + log.debug("Finished running of all tests") result.done() - config.threadpool.stop() def run(self): + if config.privacy.includepcap: + config.sniffer_d = threads.deferToThreadPool(reactor, config.threadpool, net.capturePackets) self.reporterFactory.options = self.options for input_unit in InputUnitFactory(self.inputs): self.runWithInputUnit(input_unit) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 46c29e5..3bebcdb 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -9,12 +9,14 @@ # :license: (c) 2012 Isis Lovecruft, Arturo Filasto # see attached LICENCE file - def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address +def capturePackets(): + from scapy.all import sniff + sniff() class PermissionsError(SystemExit): def __init__(self, *args, **kwargs): diff --git a/ooniprobe.conf b/ooniprobe.conf index 3250c4d..058394e 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -5,7 +5,6 @@ basic: # Where OONIProbe should be writing it's log file logfile: /tmp/ooniprobe.log - privacy: # Should we include the IP address of the probe in the report? includeip: false @@ -17,7 +16,6 @@ privacy: includecity: false # Should we collect a full packet capture on the client? includepcap: true - advanced: # XXX change this to point to the directory where you have stored the GeoIP # database file. This should be the directory in which OONI is installed

1 0

[ooni-probe/master] Merge branch 'master' into packetcapture
by art＠torproject.org 09 Nov '12

09 Nov '12

commit 0bbb2d5314d4470c4cfdf9fc5a398a9991b3b8ce Merge: 9e632c5 c44c528 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Nov 9 10:36:52 2012 +0100 Merge branch 'master' into packetcapture * master: Finish refactor of the core of OONI. * Everything is passing * XXX there is a bug in blocking tests not making the reactor stop Completely refactor the logic for running tests * We no longer rely on calling trial * The code is now *much* more clean and readable * The purpose of this is to make room for the threadpool to capture packets Port china trigger to new API Rename keyword filtering test to HTTP keyword filtering Refactor install documentation. Convert header of ooniprobe script to a non docstring Conflicts: ooni/config.py ooni/runner.py ooniprobe.conf before_i_commit.sh | 4 +- bin/ooniprobe | 19 ++-- docs/source/install.rst | 23 +--- nettests/core/chinatrigger.py | 107 ++++++++++++++++ nettests/core/http_keyword_filtering.py | 42 +++++++ nettests/core/keyword_filtering.py | 39 ------ nettests/core/url_list.py | 3 +- nettests/simpletest.py | 2 - ooni/config.py | 5 +- ooni/inputunit.py | 16 --- ooni/nettest.py | 81 +++---------- ooni/oonicli.py | 24 ++-- ooni/reporter.py | 182 +++++---------------------- ooni/runner.py | 206 ++++++++++++++++--------------- ooni/templates/scapyt.py | 12 +- ooni/utils/net.py | 20 ++- ooniprobe.conf | 3 + to-be-ported/old-api/chinatrigger.py | 140 --------------------- 18 files changed, 355 insertions(+), 573 deletions(-) diff --cc ooni/config.py index faf830f,d3907af..de5f45b --- a/ooni/config.py +++ b/ooni/config.py @@@ -41,6 -41,3 +41,7 @@@ advanced = Storage( for k, v in configuration['advanced'].items(): advanced[k] = v +threadpool = ThreadPool(0, advanced.threadpool_size) +threadpool.start() - sniffer_d = None ++# This is used to keep track of the state of the sniffer ++sniffer_running = None diff --cc ooni/oonicli.py index ae78583,be29b4c..2a6f0cd --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -23,11 -23,9 +23,11 @@@ from twisted.application import ap from twisted.python import usage, failure from twisted.python.util import spewer - from ooni import nettest, runner, reporter + from ooni import nettest, runner, reporter, config from ooni.inputunit import InputUnitFactory - from ooni.reporter import ReporterFactory - from ooni.nettest import InputTestSuite ++ ++from ooni.utils import net from ooni.utils import log @@@ -94,11 -88,11 +90,11 @@@ def run() if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) + log.start(cmd_line_options['logfile']) classes = runner.findTestClassesFromConfig(cmd_line_options) - casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options) + test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) - ++ d = + runner.runTestCases(test_cases, options, cmd_line_options) + reactor.run() - for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx], cmd_line_options) - log.start(cmd_line_options['logfile']) - orunner.run() diff --cc ooni/reporter.py index c9654e8,08a72b2..cdbf355 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -30,8 -30,8 +30,6 @@@ except Packet = object packet = FooClass --pyunit = __import__('unittest') -- class OSafeRepresenter(SafeRepresenter): """ This is a custom YAML representer that allows us to represent reports @@@ -223,53 -203,13 +201,13 @@@ class OReporter(YamlReporter) 'test_name': options['name'], 'test_version': options['version'], } - self.writeReportEntry(test_details) - def create(self): - r = OONIReporter(self._stream, self.tbformat, self.realtime, - self._publisher) - self._reporters.append(OONIReporter) - return r - - - class OONIReporter(OReporter): - """ - This is a special reporter that has knowledge about the fact that there can - exist more test runs of the same kind per run. - These multiple test runs are kept track of through idx. - - An instance of such reporter should be created per InputUnit. Every input - unit will invoke size_of_input_unit * test_cases times startTest(). - """ - def __init__(self, stream=sys.stdout, tbformat='default', realtime=False, - publisher=None): - super(OONIReporter, self).__init__(stream=stream, - tbformat=tbformat, realtime=realtime, publisher=publisher) - - self._tests = {} - - def getTestIndex(self, test): - try: - idx = test._idx - except: - idx = 0 - return idx - - - def startTest(self, test): - super(OONIReporter, self).startTest(test) - - idx = self.getTestIndex(test) - if not self._startTime: - self._startTime = self._getTime() - - log.debug("startTest on %s" % idx) - test.report = {} - - self._tests[idx] = {} - self._tests[idx]['test_started'] = self._getTime() + def testDone(self, test): + test_report = dict(test.report) - + + # XXX the scapy test has an example of how + # to do this properly. if isinstance(test.input, packet.Packet): test_input = repr(test.input) else: diff --cc ooni/runner.py index a6e0908,c67f2a7..083de35 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -10,20 -10,20 +10,21 @@@ # :version: 0.1.0-pre-alpha import os + import sys + import time import inspect + import traceback from twisted.python import reflect, usage - - from twisted.trial.runner import isTestCase + from twisted.internet import defer from twisted.trial.runner import filenameToModule +from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory - from ooni.nettest import InputTestSuite + from ooni.nettest import NetTestCase - from ooni.reporter import ReporterFactory - from ooni.utils import log, date, net - from ooni import config + from ooni import reporter + from ooni.utils import log, date def processTest(obj, cmd_line_options): """ @@@ -64,13 -63,14 +64,14 @@@ if obj.optFlags: log.debug("Got optFlags") optFlags = obj.optFlags - options = Options() - + - options.parseOptions(cmd_line_options['subArgs']) - obj.localOptions = options + if options: + options.parseOptions(cmd_line_options['subArgs']) + obj.localOptions = options - if input_file: + if input_file and options: + log.debug("Got input file") obj.inputFile = options[input_file[0]] try: diff --cc ooni/utils/net.py index 3bebcdb,46c29e5..a5a512d --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@@ -2,21 -2,19 +2,27 @@@ # # net.py # -------- --# OONI utilities for network infrastructure and hardware. --# --# :authors: Isis Lovecruft, Arturo Filasto --# :version: 0.0.1-pre-alpha --# :license: (c) 2012 Isis Lovecruft, Arturo Filasto --# see attached LICENCE file ++# OONI utilities for networking related operations + ++from scapy.all import utils ++from twisted.internet import defer ++from ooni.utils import log ++from ooni.config import threadpool def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address ++def writePacketToPcap(pkt): ++ from scapy.all import utils ++ log.debug("Writing to pcap file %s" % pkt) ++ utils.wrpcap('/tmp/foo.pcap', pkt) ++ +def capturePackets(): + from scapy.all import sniff - sniff() ++ return defer.deferToThread(sniff, writePacketToPcap, ++ lfilter=writePacketToPcap) class PermissionsError(SystemExit): def __init__(self, *args, **kwargs): diff --cc ooniprobe.conf index 058394e,2039951..b7ea1f3 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@@ -22,5 -22,4 +22,8 @@@ advanced # /path/to/ooni-probe/data/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true ++<<<<<<< HEAD + threadpool_size: 10 ++======= ++>>>>>>> master

1 0

[ooni-probe/master] Jot down some quick notes
by art＠torproject.org 09 Nov '12

09 Nov '12

commit df54cb8f831b1165450cf0a09c2b4d4ff3a79f9d Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 7 21:19:27 2012 +0100 Jot down some quick notes --- ooni/config.py | 4 ++++ ooni/runner.py | 5 ++++- ooniprobe.conf | 3 +++ 3 files changed, 11 insertions(+), 1 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 5880d69..64bc973 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -6,6 +6,8 @@ import os import yaml +from twisted.python.threadpool import ThreadPool + from ooni.utils import Storage def get_root_path(): @@ -39,3 +41,5 @@ advanced = Storage() for k, v in configuration['advanced'].items(): advanced[k] = v +threadpool = ThreadPool(0, config.advanced.threadpool_size) +threadpool.start() diff --git a/ooni/runner.py b/ooni/runner.py index b4d9560..ead81ea 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -8,7 +8,7 @@ # :license: see included LICENSE file # :copyright: (c) 2012 Isis Lovecruft, Arturo Filasto, The Tor Project, Inc. # :version: 0.1.0-pre-alpha -# + import os import inspect @@ -22,6 +22,7 @@ from ooni.nettest import InputTestSuite from ooni.reporter import ReporterFactory from ooni.utils import log, date +from ooni import config def processTest(obj, cmd_line_options): """ @@ -203,6 +204,8 @@ class ORunner(object): log.debug("Finished") result.done() + config.threadpool.stop() + def run(self): self.reporterFactory.options = self.options for input_unit in InputUnitFactory(self.inputs): diff --git a/ooniprobe.conf b/ooniprobe.conf index 2039951..3250c4d 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -15,6 +15,8 @@ privacy: includecountry: false # Should we include the ASN of the probe in the report? includecity: false + # Should we collect a full packet capture on the client? + includepcap: true advanced: # XXX change this to point to the directory where you have stored the GeoIP @@ -22,4 +24,5 @@ advanced: # /path/to/ooni-probe/data/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true + threadpool_size: 10

1 0

[ooni-probe/master] Implement logging to PCAP file support
by art＠torproject.org 09 Nov '12

09 Nov '12

commit 6a3ee55b574adaa8740ccafe2e4a01719dc0e86e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Nov 9 13:20:38 2012 +0100 Implement logging to PCAP file support --- docs/source/index.rst | 2 +- ooni/config.py | 17 ++- ooni/lib/txscapy.py | 381 ------------------------------------------------ ooni/nettest.py | 2 +- ooni/oonicli.py | 43 +++++- ooni/runner.py | 24 +-- ooni/utils/__init__.py | 4 + ooni/utils/net.py | 38 ++++-- ooniprobe.conf | 5 +- 9 files changed, 94 insertions(+), 422 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 2497a09..132381f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -28,7 +28,7 @@ On debian based systems these can be installed with: The python dependencies required for running ooniprobe are: * Twisted - * Scapy + * Scapy >= 2.2.0 * txtorcon They can be installed from the requirements.txt with: diff --git a/ooni/config.py b/ooni/config.py index de5f45b..f3d1a80 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -6,8 +6,9 @@ import os import yaml -from twisted.internet import reactor +from twisted.internet import reactor, threads +from ooni.utils import date from ooni.utils import Storage def get_root_path(): @@ -16,6 +17,18 @@ def get_root_path(): root = os.path.abspath(root) return root +def oreport_filenames(): + """ + returns the filenames for the pcap file and the yamloo report + + returns + yamloo_filename, pcap_filename + """ + base_filename = "%s_"+date.timestamp()+".%s" + yamloo_filename = base_filename % ("report", "yamloo") + pcap_filename = base_filename % ("packets", "pcap") + return yamloo_filename, pcap_filename + config_file = os.path.join(get_root_path(), 'ooniprobe.conf') try: f = open(config_file) @@ -41,7 +54,5 @@ advanced = Storage() for k, v in configuration['advanced'].items(): advanced[k] = v -threadpool = ThreadPool(0, advanced.threadpool_size) -threadpool.start() # This is used to keep track of the state of the sniffer sniffer_running = None diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py deleted file mode 100644 index 00224d6..0000000 --- a/ooni/lib/txscapy.py +++ /dev/null @@ -1,381 +0,0 @@ -# -*- coding:utf8 -*- -""" -txscapy -******* -(c) 2012 Arturo Filastò -a twisted wrapper for scapys send and receive functions. - -This software has been written to be part of OONI, the Open Observatory of -Network Interference. More information on that here: http://ooni.nu/ - -""" - -import struct -import socket -import os -import sys -import time - -from twisted.internet import protocol, base, fdesc, error, defer -from twisted.internet import reactor, threads -from zope.interface import implements - -from scapy.all import Gen -from scapy.all import SetGen - -from ooni.utils import log - -LINUX=sys.platform.startswith("linux") -OPENBSD=sys.platform.startswith("openbsd") -FREEBSD=sys.platform.startswith("freebsd") -NETBSD=sys.platform.startswith("netbsd") -DARWIN=sys.platform.startswith("darwin") -SOLARIS=sys.platform.startswith("sunos") -WINDOWS=sys.platform.startswith("win32") - -from scapy.all import RawPcapWriter, MTU, BasePacketList, conf -class PcapWriter(RawPcapWriter): - def __init__(self, filename, linktype=None, gz=False, endianness="", - append=False, sync=False): - RawPcapWriter.__init__(self, filename, linktype=linktype, gz=gz, - endianness=endianness, append=append, sync=sync) - fdesc.setNonBlocking(self.f) - - def _write_header(self, pkt): - if self.linktype == None: - if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList): - pkt = pkt[0] - try: - self.linktype = conf.l2types[pkt.__class__] - except KeyError: - self.linktype = 1 - RawPcapWriter._write_header(self, pkt) - - def _write_packet(self, packet): - sec = int(packet.time) - usec = int(round((packet.time-sec)*1000000)) - s = str(packet) - caplen = len(s) - RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen) - -class ScapySocket(object): - MTU = 1500 - def __init__(self, filter=None, iface=None, nofilter=None): - from scapy.all import conf - self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - - def fileno(self): - return self.ssocket.ins.fileno() - - def send(self, data): - return self.ssocket.send(data) - - def recv(self): - if FREEBSD or DARWIN: - return self.ssocket.nonblock_recv() - else: - return self.ssocket.recv(self.MTU) - -class TXScapy(object): - """ - A twisted based wrapper for scapy send and receive functionality. - - It sends packets inside of a threadpool and receives packets using the - libdnet receive non blocking file descriptor. - """ - min = 2 - max = 6 - debug = False - write_only_answers = False - pcapwriter = None - recv = False - timeout_call = None - answers = [] - questions = [] - - def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, - iface=None, nofilter=None, pcapfile=None, timeout=None, *arg, **kw): - self.maxPacketSize = maxPacketSize - if not reactor: - from twisted.internet import reactor - - self._reactor = reactor - - if pkts: - self._buildPacketQueues(pkts) - try: - self._buildSocket() - except Exception, e: - log.err("Unable to build socket. Are you root?") - sys.exit() - - self.cthreads = 0 - self.mthreads = 80 - - self.running = False - self.done = False - self.finished = False - - import thread - from twisted.python import threadpool - self.threadID = thread.get_ident - self.threadpool = threadpool.ThreadPool(self.min, self.max) - self.startID = self._reactor.callWhenRunning(self._start) - - self.deferred = defer.Deferred() - - if pcapfile: - self.pcapwriter = PcapWriter(pcapfile) - - if timeout and self.recv: - pass - - def _buildSocket(self, filter=None, iface=None, nofilter=None): - self.socket = ScapySocket(filter, iface, nofilter) - if self.recv: - self._reactor.addReader(self) - - def _buildPacketQueues(self, pkts): - """ - Converts the list of packets to a Scapy generator and sets up all the - necessary attributes for understanding if all the needed responses have - been received. - """ - if not isinstance(pkts, Gen): - self.pkts = SetGen(pkts) - - self.outqueue = [p for p in pkts] - - self.total_count = len(self.outqueue) - self.answer_count = 0 - self.out_count = 0 - - self.hsent = {} - for p in self.outqueue: - h = p.hashret() - if h in self.hsent: - self.hsent[h].append(p) - else: - self.hsent[h] = [p] - - - def gotAnswer(self, answer, question): - """ - Got a packet that has been identified as an answer to one of the sent - out packets. - - If the answer count matches the sent count the finish callback is - fired. - - @param answer: the packet received on the wire. - - @param question: the sent packet that matches that response. - - """ - if self.pcapwriter and self.write_only_answers: - self.pcapwriter.write(question) - self.pcapwriter.write(answer) - self.answers.append(answers) - self.questions.append(question) - self.answer_count += 1 - if self.answer_count >= self.total_count and self.running: - log.debug("Got all the answers I need") - self.finalClose() - self.deferred.callback(None) - - def processAnswer(self, pkt, hlst): - """ - Checks if the potential answer is in fact an answer to one of the - matched sent packets. Uses the scapy .answers() function to verify - this. - - @param pkt: The packet to be tested if is the answer to a sent packet. - - @param hlst: a list of packets that match the hash for an answer to - pkt. - """ - for i in range(len(hlst)): - if pkt.answers(hlst[i]): - self.gotAnswer(pkt, hlst[i]) - - def fileno(self): - """ - Returns a fileno for use by twisteds Reader. - """ - return self.socket.fileno() - - def processPacket(self, pkt): - """ - Override this method to process your packets. - - @param pkt: the packet that has been received. - """ - pkt.show() - - def doRead(self): - """ - There is something to be read on the wire. Do all the processing on the - received packet. - """ - pkt = self.socket.recv() - if not pkt: - return - if self.pcapwriter and not self.write_only_answers: - self.pcapwriter.write(pkt) - - self.processPacket(pkt) - - h = pkt.hashret() - if h in self.hsent: - hlst = self.hsent[h] - self.processAnswer(pkt, hlst) - - def logPrefix(self): - """ - The prefix to be prepended in logging. - """ - return "txScapy" - - def _start(self): - """ - Start the twisted thread pool. - """ - self.startID = None - return self.start() - - def start(self): - """ - Actually start the thread pool. - """ - if not self.running: - self.threadpool.start() - self.shutdownID = self._reactor.addSystemEventTrigger( - 'during', 'shutdown', self.finalClose) - self.running = True - - def sendPkt(self, pkt): - """ - Send a packet to the wire. - - @param pkt: The packet to be sent. - """ - self.socket.send(pkt) - - def timeout(self, *arg, **kw): - if not self.done: - log.debug("I have not finished. Setting to call in %s" % - self.timeoutSeconds) - self.timeout_call = self._reactor.callLater(self.timeoutSeconds, self.timeout, None) - elif self.running: - log.debug("Cancelling timeout call") - self.finalClose() - self.deferred.callback(None) - - def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): - """ - Wraps the scapy sr function. - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - log.debug("TXScapy sending and receiving packets") - self.recv = True - if timeout: - self.timeoutSeconds = timeout - self.timeout_call = self._reactor.callLater(timeout, self.timeout, None) - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): - """ - Wraps the scapy send function. Its the same as send and receive, except - it does not receive. Who would have ever guessed? ;) - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - self.recv = False - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0): - self._buildSocket(filter, iface, nofilter) - self._buildPacketQueues(pkts) - def sent(cb): - if self.cthreads < self.mthreads and not self.done: - pkt = None - try: - pkt = self.outqueue.pop() - except: - self.done = True - if not self.recv and self.running: - log.debug("I am not in receiving state running callback") - self.deferred.callback(None) - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - for x in range(self.mthreads): - try: - pkt = self.outqueue.pop() - except: - self.done = True - return - if self.cthreads >= self.mthreads and self.done: - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - def connectionLost(self, why): - pass - - def finalClose(self): - """ - Clean all the shutdown related functions. - """ - self.shutdownID = None - self.threadpool.stop() - if self.timeout_call: - self.timeout_call.cancel() - self.timeout_call = None - self.running = False - -(a)defer.inlineCallbacks -def txsr(*args, **kw): - tr = TXScapy(*args, **kw) - tr.sr(*args, **kw) - yield tr.deferred - tr.finalClose() - -(a)defer.inlineCallbacks -def txsend(*arg, **kw): - tr = TXScapy(*arg, **kw) - tr.send(*arg, **kw) - yield tr.deferred - tr.finalClose() diff --git a/ooni/nettest.py b/ooni/nettest.py index 289cd23..6221a3f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -136,7 +136,7 @@ class NetTestCase(object): for x in fp.xreadlines(): yield x.strip() fp.close() - + def _checkRequiredOptions(self): for required_option in self.requiredOptions: log.debug("Checking if %s is present" % required_option) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 2a6f0cd..4b63a61 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -27,7 +27,7 @@ from ooni import nettest, runner, reporter, config from ooni.inputunit import InputUnitFactory -from ooni.utils import net +from ooni.utils import net, checkForRoot from ooni.utils import log @@ -44,7 +44,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): 'Report deferred creation and callback stack traces'],] optParameters = [["reportfile", "o", None, "report file name"], - ["logfile", "l", None, "log file name"],] + ["logfile", "l", None, "log file name"], + ["pcapfile", "p", None, "pcap file name"]] compData = usage.Completions( extraActions=[usage.CompleteFiles( @@ -75,6 +76,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): except: raise usage.UsageError("No test filename specified!") +def testsEnded(*arg, **kw): + """ + You can place here all the post shutdown tasks. + """ + log.debug("Finished running all tests") + def run(): """ Call me to begin testing from a file. @@ -90,11 +97,37 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) + yamloo_filename, pcap_filename = config.oreport_filenames() + + if cmd_line_options['reportfile']: + yamloo_filename = cmd_line_options['reportfile'] + pcap_filename = yamloo_filename+".pcap" + + if os.path.exists(yamloo_filename): + log.msg("Report already exists with filename %s" % yamloo_filename) + log.msg("Renaming it to %s" % yamloo_filename+'.old') + os.rename(yamloo_filename, yamloo_filename+'.old') + if os.path.exists(pcap_filename): + log.msg("Report already exists with filename %s" % pcap_filename) + log.msg("Renaming it to %s" % pcap_filename+'.old') + os.rename(pcap_filename, pcap_filename+'.old') + log.start(cmd_line_options['logfile']) classes = runner.findTestClassesFromConfig(cmd_line_options) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) - d = - runner.runTestCases(test_cases, options, cmd_line_options) - reactor.run() + if config.privacy.includepcap: + try: + checkForRoot() + except: + log.err("includepcap options requires root priviledges to run") + log.err("disable it in your ooniprobe.conf file") + sys.exit(1) + log.debug("Starting sniffer") + sniffer_d = net.capturePackets(pcap_filename) + tests_d = runner.runTestCases(test_cases, options, + cmd_line_options, yamloo_filename) + tests_d.addBoth(testsEnded) + + reactor.run() diff --git a/ooni/runner.py b/ooni/runner.py index 083de35..d8b7df8 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -24,7 +24,8 @@ from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase from ooni import reporter -from ooni.utils import log, date + +from ooni.utils import log, date, checkForRoot def processTest(obj, cmd_line_options): """ @@ -41,8 +42,7 @@ def processTest(obj, cmd_line_options): input_file = obj.inputFile if obj.requiresRoot: - if os.getuid() != 0: - raise Exception("This test requires root to run") + checkForRoot("test") if obj.optParameters or input_file \ or obj.usageOptions or obj.optFlags: @@ -184,7 +184,8 @@ def runTestWithInputUnit(test_class, return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, cmd_line_options): +def runTestCases(test_cases, options, + cmd_line_options, yamloo_filename): try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -203,17 +204,7 @@ def runTestCases(test_cases, options, cmd_line_options): log.msg("options[0] = %s" % first) test_inputs = [None] - if cmd_line_options['reportfile']: - report_filename = cmd_line_options['reportfile'] - else: - report_filename = 'report_'+date.timestamp()+'.yamloo' - - if os.path.exists(report_filename): - print "Report already exists with filename %s" % report_filename - print "Renaming it to %s" % report_filename+'.old' - os.rename(report_filename, report_filename+'.old') - - reportFile = open(report_filename, 'w+') + reportFile = open(yamloo_filename, 'w+') oreporter = reporter.OReporter(reportFile) input_unit_factory = InputUnitFactory(test_inputs) @@ -226,6 +217,7 @@ def runTestCases(test_cases, options, cmd_line_options): test_class = test_case[0] test_method = test_case[1] yield runTestWithInputUnit(test_class, - test_method, input_unit, oreporter) + test_method, input_unit, + oreporter) oreporter.allDone() diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index cd82ab4..9961e03 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -3,6 +3,7 @@ """ import imp +import os import logging import string import random @@ -55,6 +56,9 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v +def checkForRoot(what): + if os.getuid() != 0: + raise Exception("This %s requires root to run" % what) def get_logger(config): loglevel = getattr(logging, config.loglevel.upper()) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index a5a512d..3fd4b41 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -4,25 +4,41 @@ # -------- # OONI utilities for networking related operations +import sys +from twisted.internet import threads, reactor + from scapy.all import utils -from twisted.internet import defer -from ooni.utils import log -from ooni.config import threadpool + +from ooni.utils import log, txscapy def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address -def writePacketToPcap(pkt): - from scapy.all import utils - log.debug("Writing to pcap file %s" % pkt) - utils.wrpcap('/tmp/foo.pcap', pkt) - -def capturePackets(): +def capturePackets(pcap_filename): from scapy.all import sniff - return defer.deferToThread(sniff, writePacketToPcap, - lfilter=writePacketToPcap) + global stop_packet_capture + stop_packet_capture = False + + def stopCapture(): + # XXX this is a bit of a hack to stop capturing packets when we close + # the reactor. Ideally we would want to be able to do this + # programmatically, but this requires some work on implementing + # properly the sniff function with deferreds. + global stop_packet_capture + stop_packet_capture = True + + def writePacketToPcap(pkt): + from scapy.all import utils + pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True) + pcapwriter.write(pkt) + if stop_packet_capture: + sys.exit(1) + + d = threads.deferToThread(sniff, lfilter=writePacketToPcap) + reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) + return d class PermissionsError(SystemExit): def __init__(self, *args, **kwargs): diff --git a/ooniprobe.conf b/ooniprobe.conf index b7ea1f3..1e76ad7 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -15,15 +15,12 @@ privacy: # Should we include the ASN of the probe in the report? includecity: false # Should we collect a full packet capture on the client? - includepcap: true + includepcap: false advanced: # XXX change this to point to the directory where you have stored the GeoIP # database file. This should be the directory in which OONI is installed # /path/to/ooni-probe/data/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true -<<<<<<< HEAD threadpool_size: 10 -======= ->>>>>>> master

1 0

[ooni-probe/master] Merge branch 'master' into packetcapture
by art＠torproject.org 09 Nov '12

09 Nov '12

commit c594b96685bad43354ba4de3ee018947d0a39cbf Merge: df54cb8 92cf32d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 8 17:10:16 2012 +0100 Merge branch 'master' into packetcapture * master: Move test inputs back to where they were. Disabled debug log in default config file * Make runner rename file to .old if it already exists Refactor reporting system * Fix import error in http host test * Moved metaclass code to separate branch 'meta'. * Removed unused code from assertions.py. * Deleted config.py and process.py. * Removed bug as a result of merging which reintroduced the bug for log messages being printed twice. * Made the submit-patch script create the /patches directory if it doesn't already exist. * Moved /test_inputs/* to /inputs. * Adding new Makefile in top dir * Adding saved copy of *.secdev.org SSL cert so we can correctly check for cert warning. * Updated installation documentation. Move peculiar code to wtf directory. * Comment on tor related code and propose that it gets removed. Move peculiar unused file to wtf directory. * Added brief doctrings to get pylint to shutup about it. * Changed the nettest inputParser to no longer be wrapped by @staticmethod, because that might be confusing to test writers. * Changed runner.getTestOptions() to return the localoptions, in addition to setting them as an attribute. * There are still bugs in the options parser, and inputs is a listiterater, which also needs to be fixed. * Fixed oonicli.py trying to catch the wrong exception. * Added a docstring in oonicli.py. * Fixed oonicli.py not printing the help/info message when called without arguments. Conflicts: ooniprobe.conf Makefile | 198 ++++++++++++++++++++++++++++ bin/ooniprobe | 23 ++-- docs/source/install.rst | 12 ++- inputs/README | 46 +++++++ nettests/bridge_reachability/echo.py | 8 +- nettests/core/http_host.py | 1 + nettests/simpletest.py | 23 +++- ooni/lib/secdev.org.pem | 20 +++ ooni/nettest.py | 17 ++- ooni/oonicli.py | 22 ++-- ooni/reporter.py | 31 ++--- ooni/runner.py | 19 ++- ooni/utils/assertions.py | 180 ------------------------- ooni/utils/config.py | 121 ----------------- ooni/utils/meta.py | 241 ---------------------------------- ooni/utils/onion.py | 17 +++- ooni/utils/process.py | 73 ---------- ooni/utils/timer.py | 107 --------------- submit-patch | 7 +- test_inputs/README | 48 +------- 20 files changed, 380 insertions(+), 834 deletions(-)

1 0

[translation/vidalia_alpha_completed] Update translations for vidalia_alpha_completed
by translation＠torproject.org 09 Nov '12

09 Nov '12

commit 9bc0dc55a09fc17acbf983131b784ce990033501 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Nov 9 05:45:24 2012 +0000 Update translations for vidalia_alpha_completed --- es/vidalia_es.po | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 973c0b6..4d166fe 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-08 23:48+0000\n" +"PO-Revision-Date: 2012-11-09 05:33+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -344,7 +344,7 @@ msgstr "Conectando a %1:%2... " msgctxt "BridgeDownloader" msgid "Sending an HTTPS request for bridges..." -msgstr "Enviando una solicitud HTTPS para bridges (puentes a Tor no propagados)... " +msgstr "Enviando una solicitud HTTPS de bridges (puentes a Tor no propagados)... " msgctxt "BridgeDownloader" msgid "Downloading a list of bridges..." @@ -356,7 +356,7 @@ msgstr "Descargando bridges (puentes a Tor no propagados)" msgctxt "BridgeDownloaderProgressDialog" msgid "Unable to download bridges: %1" -msgstr "No pudo descargar bridges (puentes a Tor no propagados): %1" +msgstr "No se pudo descargar bridges (puentes a Tor no propagados): %1" msgctxt "BridgeDownloaderProgressDialog" msgid "Retrying bridge request..." @@ -372,11 +372,11 @@ msgstr "Nº. de Clientes" msgctxt "BridgeUsageDialog" msgid "Clients from the following countries have used your relay since %1" -msgstr "Clientes de los siguientes países han usado su retransmisor desde %1" +msgstr "Clientes de los siguientes países han usado su repetidor (relay) desde %1" msgctxt "BridgeUsageDialog" msgid "Bridge Usage Summary" -msgstr "Resumen de utilización del puente" +msgstr "Resumen de utilización del bridge (puente a Tor no propagado)" msgctxt "BridgeUsageDialog" msgid "Client Summary"

1 0

[translation/vidalia_alpha] Update translations for vidalia_alpha
by translation＠torproject.org 09 Nov '12

09 Nov '12

commit 07d11b32cb5d085afb8057b89c9f5306d7d1688e Author: Translation commit bot <translation(a)torproject.org> Date: Fri Nov 9 05:45:21 2012 +0000 Update translations for vidalia_alpha --- es/vidalia_es.po | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 973c0b6..4d166fe 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-08 23:48+0000\n" +"PO-Revision-Date: 2012-11-09 05:33+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -344,7 +344,7 @@ msgstr "Conectando a %1:%2... " msgctxt "BridgeDownloader" msgid "Sending an HTTPS request for bridges..." -msgstr "Enviando una solicitud HTTPS para bridges (puentes a Tor no propagados)... " +msgstr "Enviando una solicitud HTTPS de bridges (puentes a Tor no propagados)... " msgctxt "BridgeDownloader" msgid "Downloading a list of bridges..." @@ -356,7 +356,7 @@ msgstr "Descargando bridges (puentes a Tor no propagados)" msgctxt "BridgeDownloaderProgressDialog" msgid "Unable to download bridges: %1" -msgstr "No pudo descargar bridges (puentes a Tor no propagados): %1" +msgstr "No se pudo descargar bridges (puentes a Tor no propagados): %1" msgctxt "BridgeDownloaderProgressDialog" msgid "Retrying bridge request..." @@ -372,11 +372,11 @@ msgstr "Nº. de Clientes" msgctxt "BridgeUsageDialog" msgid "Clients from the following countries have used your relay since %1" -msgstr "Clientes de los siguientes países han usado su retransmisor desde %1" +msgstr "Clientes de los siguientes países han usado su repetidor (relay) desde %1" msgctxt "BridgeUsageDialog" msgid "Bridge Usage Summary" -msgstr "Resumen de utilización del puente" +msgstr "Resumen de utilización del bridge (puente a Tor no propagado)" msgctxt "BridgeUsageDialog" msgid "Client Summary"

1 0

[tor/master] Fix test.c compilation on mingw32.
by nickm＠torproject.org 09 Nov '12

09 Nov '12

commit 8b5d95db0df18bca6e608e95248340cf7f67b673 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Nov 8 20:09:06 2012 -0500 Fix test.c compilation on mingw32. Looks like windows doesn't have an s6_addr32 in its in6_addr. Bug not in any released version of Tor; bugfix on abb886014e1ee. --- src/test/test.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/src/test/test.c b/src/test/test.c index 9848770..1eaa65c 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1547,7 +1547,10 @@ test_geoip(void) memset(&in6, 0, sizeof(in6)); /* Make sure that country ID actually works. */ -#define SET_TEST_IPV6(i) in6.s6_addr32[3] = htonl((uint32_t) i) +#define SET_TEST_IPV6(i) \ + do { \ + set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ + } while (0) #define CHECK_COUNTRY(country, val) do { \ /* test ipv4 country lookup */ \ test_streq(country, \

1 0

[ooni-probe/master] Convert header of ooniprobe script to a non docstring
by art＠torproject.org 09 Nov '12

09 Nov '12

commit 2332356219f9b301db31c55393a0f4aab3d73d50 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 8 17:41:27 2012 +0100 Convert header of ooniprobe script to a non docstring --- bin/ooniprobe | 18 ++++++++---------- 1 files changed, 8 insertions(+), 10 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index 0742da9..18dab7d 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,16 +1,14 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -''' - oonicli - ----------- +# oonicli +# ----------- +# +# Run the Twisted Trial (unittest module) based oonicli. Startup script based +# on twisted trial, see http://twistedmatrix.com/ . - Run the Twisted Trial (unittest module) based oonicli. Startup script based - on twisted trial, see http://twistedmatrix.com/ . - - :authors: Arturo Filastò, Isis Lovecruft - :licence: see included LICENSE file - :version: 0.0.1-pre-alpha -''' +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see included LICENSE file +# :version: 0.0.1-pre-alpha import os, sys import copy_reg

1 0