November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix bug that lead processPacket being called even if no packet has been received
by art＠torproject.org 25 Nov '12

25 Nov '12

commit fc24e4f29d25a2cef91525e4d47c39da3c6b4d8d Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:49:06 2012 +0100 Fix bug that lead processPacket being called even if no packet has been received --- ooni/utils/txscapy.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index fbe9c38..7902133 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -113,8 +113,9 @@ class ScapyFactory(abstract.FileDescriptor): def doRead(self): packet = self.super_socket.recv(MTU) - for protocol in self.protocols: - protocol.packetReceived(packet) + if packet: + for protocol in self.protocols: + protocol.packetReceived(packet) def registerProtocol(self, protocol): if not self.connected:

1 0

[ooni-probe/master] Handle the case in which we do not get a DNS lookup response and include it in the report
by art＠torproject.org 25 Nov '12

25 Nov '12

commit b2b2b54306f503335cc52edc82426baea45da242 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:30:17 2012 +0100 Handle the case in which we do not get a DNS lookup response and include it in the report --- nettests/core/dnsspoof.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nettests/core/dnsspoof.py b/nettests/core/dnsspoof.py index 48991ea..5c50c2f 100644 --- a/nettests/core/dnsspoof.py +++ b/nettests/core/dnsspoof.py @@ -38,8 +38,13 @@ class DNSSpoof(scapyt.ScapyTest): This is not tested, but the concept is that if the two responses match up then spoofing is occuring. """ - test_answer = report['test_a_lookup']['answered_packets'][0][1] - control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + try: + test_answer = report['test_a_lookup']['answered_packets'][0][1] + control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + except IndexError: + self.report['spoofing'] = 'no_answer' + return + if test_answer[UDP] == control_answer[UDP]: self.report['spoofing'] = True else:

1 0

[ooni-probe/master] The dicts and lists used for keeping track of answers should be instance attributes not class attributes
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 15c501d57c1876a21ee60b53f0f38ee845ebe782 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:28:16 2012 +0100 The dicts and lists used for keeping track of answers should be instance attributes not class attributes --- ooni/utils/txscapy.py | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 8e479f8..fbe9c38 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -148,16 +148,7 @@ class ScapyProtocol(object): class ScapySender(ScapyProtocol): timeout = 5 - # This dict is used to store the unique hashes that allow scapy to - # match up request with answer - hr_sent_packets = {} - - # These are the packets we have received as answer to the ones we sent - answered_packets = [] - - # These are the packets we send - sent_packets = [] - + # This deferred will fire when we have finished sending a receiving packets. # Should we look for multiple answers for the same sent packet? multi = False @@ -222,6 +213,16 @@ class ScapySender(ScapyProtocol): self.factory.send(packet) def startSending(self, packets): + # This dict is used to store the unique hashes that allow scapy to + # match up request with answer + self.hr_sent_packets = {} + + # These are the packets we have received as answer to the ones we sent + self.answered_packets = [] + + # These are the packets we send + self.sent_packets = [] + self._start_time = time.time() self.d = defer.Deferred() self.sendPackets(packets)

1 0

[ooni-probe/master] Register scapyFactor if it is not registered
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 29505b25df7b133d634c9c0b5b5f33be12be7adf Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:31:47 2012 +0100 Register scapyFactor if it is not registered * Better debugging messages --- ooni/config.py | 1 + ooni/templates/scapyt.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 5168c34..b017e87 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -14,6 +14,7 @@ from ooni.utils import Storage reports = Storage() basic = None cmd_line_options = None +scapyFactory = None def get_root_path(): this_directory = os.path.dirname(__file__) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index cb02300..a1eade4 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -16,7 +16,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log from ooni import config -from ooni.utils.txscapy import ScapySender, getDefaultIface +from ooni.utils.txscapy import ScapySender, getDefaultIface, ScapyFactory class BaseScapyTest(NetTestCase): """ @@ -42,6 +42,10 @@ class BaseScapyTest(NetTestCase): ] def _setUp(self): + if not config.scapyFactory: + log.debug("Scapy factoring not set, registering it.") + config.scapyFactory = ScapyFactory(config.advanced.interface) + self.report['answer_flags'] = [] if self.localOptions['ipsrc']: config.checkIPsrc = 0 @@ -98,6 +102,7 @@ class BaseScapyTest(NetTestCase): scapySender = ScapySender() config.scapyFactory.registerProtocol(scapySender) + log.debug("Using sending with hash %s" % scapySender.__hash__) d = scapySender.startSending(packets) d.addCallback(self.finishedSendReceive)

1 0

[ooni-probe/master] Randomize source port by default in traceroute test and include source port in report
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 2e06b7c0a6c779f174d047d6a22dac36fe6d4922 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:31:04 2012 +0100 Randomize source port by default in traceroute test and include source port in report --- nettests/core/traceroute.py | 19 +++++++------------ 1 files changed, 7 insertions(+), 12 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index 45bbb67..f8311fd 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -20,8 +20,6 @@ class UsageOptions(usage.Options): ['srcport', 'p', None, 'Set the source port to a specific value (only applies to TCP and UDP)'] ] - optFlags = [['randomize','r', 'Randomize the source port']] - class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" author = "Arturo Filastò" @@ -34,12 +32,8 @@ class TracerouteTest(scapyt.BaseScapyTest): def get_sport(protocol): if self.localOptions['srcport']: return int(self.localOptions['srcport']) - elif self.localOptions['randomize']: + else: return random.randint(1024, 65535) - elif protocol == 'tcp': - return 80 - elif protocol == 'udp': - return 53 self.get_sport = get_sport @@ -65,11 +59,12 @@ class TracerouteTest(scapyt.BaseScapyTest): def finished(packets, port): log.debug("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets - self.report['hops_'+str(port)] = [] + self.report['hops_'+str(port)] = [] for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time, + 'sport': snd[UDP].sport } log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) @@ -98,7 +93,8 @@ class TracerouteTest(scapyt.BaseScapyTest): for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time, + 'sport': snd[UDP].sport } log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) @@ -126,7 +122,7 @@ class TracerouteTest(scapyt.BaseScapyTest): for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time } log.debug("%s" % (report)) self.report['hops'].append(report) @@ -139,4 +135,3 @@ class TracerouteTest(scapyt.BaseScapyTest): d.addCallback(finished) return d -

1 0

[ooni-probe/master] Add proper documentation to HTTP Test template
by art＠torproject.org 25 Nov '12

25 Nov '12

commit e496f0d1b3d395fec039574fd212675885b9e017 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 08:20:57 2012 +0100 Add proper documentation to HTTP Test template * Do some minor refactoring --- ooni/templates/httpt.py | 177 +++++++++++++++++++++++++++++------------------ 1 files changed, 110 insertions(+), 67 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e166263..e36c049 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -24,6 +24,9 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders +class InvalidSocksProxyOption(Exception): + pass + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -51,11 +54,10 @@ class HTTPTest(NetTestCase): response = {} def _setUp(self): - log.debug("Setting up HTTPTest") try: import OpenSSL except: - log.err("Warning! pyOpenSSL is not installed. https websites will" + log.err("Warning! pyOpenSSL is not installed. https websites will " "not work") self.control_agent = Agent(reactor, sockshost="127.0.0.1", @@ -63,27 +65,35 @@ class HTTPTest(NetTestCase): sockshost, socksport = (None, None) if self.localOptions['socksproxy']: - self.report['socksproxy'] = self.localOptions['socksproxy'] - sockshost, socksport = self.localOptions['socksproxy'].split(':') + try: + sockshost, socksport = self.localOptions['socksproxy'].split(':') + self.report['socksproxy'] = self.localOptions['socksproxy'] + except ValueError: + raise InvalidSocksProxyOption socksport = int(socksport) - self.agent = Agent(reactor, sockshost=sockshost, - socksport=socksport) + self.agent = Agent(reactor, sockshost=sockshost, socksport=socksport) + + self.report['agent'] = 'agent' if self.followRedirects: try: from twisted.web.client import RedirectAgent self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) + self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted"\ "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") - self.report['errors'].append("Could not import RedirectAgent") self.processInputs() log.debug("Finished test setup") + def randomize_useragent(self, request): + user_agent = random.choice(userAgents) + request['headers']['User-Agent'] = [user_agent] + def processInputs(self): pass @@ -107,12 +117,14 @@ class HTTPTest(NetTestCase): else: self.processResponseBody(response_body) - def processResponseBody(self, data): + def processResponseBody(self, body): """ - This should handle all the response body smushing for getting it ready - to be passed onto the control. + Overwrite this method if you wish to interact with the response body of + every request that is made. + + Args: - @param data: The content of the body returned. + body (str): The body of the HTTP response """ pass @@ -120,7 +132,9 @@ class HTTPTest(NetTestCase): """ This should take care of dealing with the returned HTTP headers. - @param headers: The content of the returned headers. + Args: + + headers (dict): The returned header fields. """ pass @@ -128,32 +142,96 @@ class HTTPTest(NetTestCase): """ Handle a redirection via a 3XX HTTP status code. - @param location: the url that is being redirected to. + Here you may place logic that evaluates the destination that you are + being redirected to. Matches against known censor redirects, etc. + + Note: if self.followRedirects is set to True, then this method will + never be called. + XXX perhaps we may want to hook _handleResponse in RedirectAgent to + call processRedirect every time we get redirected. + + Args: + + location (str): the url that we are being redirected to. """ pass + def _cbResponse(self, response, request, + headers_processor, body_processor): + """ + This callback is fired once we have gotten a response for our request. + If we are using a RedirectAgent then this will fire once we have + reached the end of the redirect chain. + + Args: + + response (:twisted.web.iweb.IResponse:): a provider for getting our response + + request (dict): the dict containing our response (XXX this should be dropped) + + header_processor (func): a function to be called with argument a + dict containing the response headers. This will lead + self.headerProcessor to not be called. + + + body_processor (func): a function to be called with as argument the + body of the response. This will lead self.bodyProcessor to not + be called. + + """ + if not response: + log.err("Got no response for request %s" % request) + return + else: + log.debug("Got response %s" % response) + + if str(response.code).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + # [!] We are passing to the headers_processor the headers dict and + # not the Headers() object + response_headers_dict = list(response.headers.getAllRawHeaders()) + if headers_processor: + headers_processor(response_headers_dict) + else: + self.processResponseHeaders(response_headers_dict) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody, request, + response, body_processor) + + return finished + def doRequest(self, url, method="GET", headers={}, body=None, headers_processor=None, body_processor=None, use_tor=False): """ - Perform an HTTP request with the specified method. + Perform an HTTP request with the specified method and headers. + + Args: - url: the full url path of the request + url (str): the full URL of the request. The scheme may be either + http, https, or httpo for http over Tor Hidden Service. - method: the HTTP Method to be used + Kwargs: - headers: the request headers to be sent as a dict + method (str): the HTTP method name to use for the request - body: the request body + headers (dict): the request headers to send - headers_processor: a function to be used for processing the HTTP header - responses (defaults to self.processResponseHeaders). - This function takes as argument the HTTP headers as a - dict. + body (str): the request body - body_processory: a function to be used for processing the HTTP response - body (defaults to self.processResponseBody). - This function takes the response body as an argument. + headers_processor : a function to be used for processing the HTTP + header responses (defaults to self.processResponseHeaders). + This function takes as argument the HTTP headers as a dict. + + body_processory: a function to be used for processing the HTTP + response body (defaults to self.processResponseBody). This + function takes the response body as an argument. + + use_tor (bool): specify if the HTTP request should be done over Tor + or not. """ @@ -170,7 +248,7 @@ class HTTPTest(NetTestCase): log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) url = 's'+url - log.debug("Performing request %s %s %s" % (url, method, headers)) + log.msg("Performing request %s %s %s" % (url, method, headers)) request = {} request['method'] = method @@ -182,8 +260,6 @@ class HTTPTest(NetTestCase): log.debug("Randomizing user agent") self.randomize_useragent(request) - log.debug("Writing to report the request") - if 'requests' not in self.report: self.report['requests'] = [] @@ -198,51 +274,18 @@ class HTTPTest(NetTestCase): def errback(failure): failure.trap(ConnectionRefusedError, SOCKSError) - if type(failure.value) is ConnectionRefusedError: + if isinstance(failure.value, ConnectionRefusedError): log.err("Connection refused. The backend may be down") - else: - log.err("Sock error. The SOCK proxy may be down") - self.report["failure"] = str(failure.value) + self.report['failure'] = 'connection_refused_error' - def finished(data): - return + elif isinstance(failure.value, SOCKSError): + log.err("Sock error. The SOCK proxy may be down") + self.report['failure'] = 'sockserror' d = agent.request(request['method'], request['url'], headers, body_producer) - d.addErrback(errback) d.addCallback(self._cbResponse, request, headers_processor, body_processor) - d.addCallback(finished) + d.addErrback(errback) return d - def _cbResponse(self, response, request, headers_processor, - body_processor): - - if not response: - log.err("Got no response") - return - else: - log.debug("Got response %s" % response) - - if str(response.code).startswith('3'): - self.processRedirect(response.headers.getRawHeaders('Location')[0]) - - # [!] We are passing to the headers_processor the headers dict and - # not the Headers() object - response_headers_dict = list(response.headers.getAllRawHeaders()) - if headers_processor: - headers_processor(response_headers_dict) - else: - self.processResponseHeaders(response_headers_dict) - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody, request, - response, body_processor) - - return finished - - def randomize_useragent(self, request): - user_agent = random.choice(userAgents) - request['headers']['User-Agent'] = [user_agent] -

1 0

[ooni-probe/master] Make the sniffer not run in a separate thread, but use a non blocking fdesc
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 615ce75c47aec249b6b5a4c0b58fdf7a93f09582 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 10:08:47 2012 +0100 Make the sniffer not run in a separate thread, but use a non blocking fdesc * Do some refactoring of scapy testing, following Factory creational pattern and a pub-sub pattern for the readers and writers (inspired by muxTCP). * Other misc refactoring --- ooni/oonicli.py | 6 ++- ooni/templates/scapyt.py | 50 ++++++++--------- ooni/utils/net.py | 23 -------- ooni/utils/txscapy.py | 138 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 126 insertions(+), 91 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5c582b2..1a316b3 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -100,6 +100,7 @@ def runTest(cmd_line_options): classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: + from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() except NotRootError: @@ -108,7 +109,10 @@ def runTest(cmd_line_options): sys.exit(1) print "Starting sniffer" - net.capturePackets(config.reports.pcap) + config.scapyFactory = ScapyFactory(config.advanced.interface) + + sniffer = ScapySniffer(config.reports.pcap) + config.scapyFactory.registerProtocol(sniffer) return runner.runTestCases(test_cases, options, cmd_line_options) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 11b4381..cb02300 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -16,7 +16,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log from ooni import config -from ooni.utils.txscapy import ScapyProtocol, getDefaultIface +from ooni.utils.txscapy import ScapySender, getDefaultIface class BaseScapyTest(NetTestCase): """ @@ -66,20 +66,8 @@ class BaseScapyTest(NetTestCase): else: config.check_TCPerror_seqack = 0 - if config.advanced.interface == 'auto': - self.interface = getDefaultIface() - else: - self.interface = config.advanced.interface - - def reportSentPacket(self, packet): - if 'sent_packets' not in self.report: - self.report['sent_packets'] = [] - self.report['sent_packets'].append(packet) - - def reportReceivedPacket(self, packet): - if 'answered_packets' not in self.report: - self.report['answered_packets'] = [] - self.report['answered_packets'].append(packet) + self.report['sent_packets'] = [] + self.report['answered_packets'] = [] def finishedSendReceive(self, packets): """ @@ -98,8 +86,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - self.reportSentPacket(sent_packet) - self.reportReceivedPacket(received_packet) + self.report['sent_packets'].append(sent_packet) + self.report['answered_packets'].append(received_packet) return packets def sr(self, packets, *arg, **kw): @@ -107,8 +95,11 @@ class BaseScapyTest(NetTestCase): Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - d = scapyProtocol.startSending(packets) + scapySender = ScapySender() + + config.scapyFactory.registerProtocol(scapySender) + + d = scapySender.startSending(packets) d.addCallback(self.finishedSendReceive) return d @@ -123,12 +114,15 @@ class BaseScapyTest(NetTestCase): return packets[0][0][1] except IndexError: log.err("Got no response...") - return None + return packets + + scapySender = ScapySender() + scapySender.expected_answers = 1 + + config.scapyFactory.registerProtocol(scapySender) - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - scapyProtocol.expected_answers = 1 log.debug("Running sr1") - d = scapyProtocol.startSending(packets) + d = scapySender.startSending(packets) log.debug("Started to send") d.addCallback(self.finishedSendReceive) d.addCallback(done) @@ -138,9 +132,13 @@ class BaseScapyTest(NetTestCase): """ Wrapper around scapy.sendrecv.send for sending of packets at layer 3 """ - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - scapyProtocol.sendPackets(packets) - scapyProtocol.stopSending() + scapySender = ScapySender() + + config.scapyFactory.registerProtocol(scapySender) + + scapySender.sendPackets(packets) + + scapySender.stopSending() for packet in packets: self.reportSentPacket(packet) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 649dc64..df98412 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -82,29 +82,6 @@ class BodyReceiver(protocol.Protocol): def connectionLost(self, reason): self.finished.callback(self.data) -def capturePackets(pcap_filename): - from scapy.all import sniff - global stop_packet_capture - stop_packet_capture = False - - def stopCapture(): - # XXX this is a bit of a hack to stop capturing packets when we close - # the reactor. Ideally we would want to be able to do this - # programmatically, but this requires some work on implementing - # properly the sniff function with deferreds. - global stop_packet_capture - stop_packet_capture = True - - def writePacketToPcap(pkt): - from scapy.all import utils - pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True) - pcapwriter.write(pkt) - if stop_packet_capture: - sys.exit(1) - d = threads.deferToThread(sniff, lfilter=writePacketToPcap) - reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) - return d - def getSystemResolver(): """ XXX implement a function that returns the resolver that is currently diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 679eb42..8e479f8 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -32,7 +32,9 @@ try: config.pcap_dnet = True except ImportError, e: - log.err("pypcap or dnet not installed. Certain tests may not work.") + log.err("pypcap or dnet not installed. " + "Certain tests may not work.") + config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False @@ -59,6 +61,9 @@ def getNetworksFromRoutes(): return networks +class IfaceError(Exception): + pass + def getDefaultIface(): networks = getNetworksFromRoutes() for net in networks: @@ -66,45 +71,100 @@ def getDefaultIface(): return net.iface raise IfaceError -class TXPcapWriter(PcapWriter): - def __init__(self, *arg, **kw): - PcapWriter.__init__(self, *arg, **kw) - fdesc.setNonBlocking(self.f) +class ProtocolNotRegistered(Exception): + pass -class ScapyProtocol(abstract.FileDescriptor): +class ProtocolAlreadyRegistered(Exception): + pass + +class ScapyFactory(abstract.FileDescriptor): + """ + Inspired by muxTCP scapyLink: + https://github.com/enki/muXTCP/blob/master/scapyLink.py + """ def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) + if interface == 'auto': + interface = getDefaultIface() if not super_socket: - super_socket = conf.L3socket(iface=interface, promisc=True, filter='') + super_socket = conf.L3socket(iface=interface, + promisc=True, filter='') #super_socket = conf.L2socket(iface=interface) + self.protocols = [] fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket - self.interface = interface - self.timeout = timeout + def writeSomeData(self, data): + """ + XXX we actually want to use this, but this requires overriding doWrite + or writeSequence. + """ + pass - # This dict is used to store the unique hashes that allow scapy to - # match up request with answer - self.hr_sent_packets = {} + def send(self, packet): + """ + Write a scapy packet to the wire. + """ + return self.super_socket.send(packet) - # These are the packets we have received as answer to the ones we sent - self.answered_packets = [] + def fileno(self): + return self.super_socket.ins.fileno() - # These are the packets we send - self.sent_packets = [] + def doRead(self): + packet = self.super_socket.recv(MTU) + for protocol in self.protocols: + protocol.packetReceived(packet) + + def registerProtocol(self, protocol): + if not self.connected: + self.startReading() + + if protocol not in self.protocols: + protocol.factory = self + self.protocols.append(protocol) + else: + raise ProtocolAlreadyRegistered + + def unRegisterProtocol(self, protocol): + if protocol in self.protocols: + self.protocols.remove(protocol) + if len(self.protocols) == 0: + self.loseConnection() + else: + raise ProtocolNotRegistered + +class ScapyProtocol(object): + factory = None + + def packetReceived(self, packet): + """ + When you register a protocol, this method will be called with argument + the packet it received. - # This deferred will fire when we have finished sending a receiving packets. - self.d = defer.Deferred() - # Should we look for multiple answers for the same sent packet? - self.multi = False + Every protocol that is registered will have this method called. + """ + raise NotImplementedError - # When 0 we stop when all the packets we have sent have received an - # answer - self.expected_answers = 0 +class ScapySender(ScapyProtocol): + timeout = 5 + # This dict is used to store the unique hashes that allow scapy to + # match up request with answer + hr_sent_packets = {} - def fileno(self): - return self.super_socket.ins.fileno() + # These are the packets we have received as answer to the ones we sent + answered_packets = [] + + # These are the packets we send + sent_packets = [] + + # This deferred will fire when we have finished sending a receiving packets. + # Should we look for multiple answers for the same sent packet? + multi = False + + # When 0 we stop when all the packets we have sent have received an + # answer + expected_answers = 0 def processPacket(self, packet): """ @@ -131,11 +191,10 @@ class ScapyProtocol(abstract.FileDescriptor): log.debug("Got the number of expected answers") self.stopSending() - def doRead(self): + def packetReceived(self, packet): timeout = time.time() - self._start_time if self.timeout and time.time() - self._start_time > self.timeout: self.stopSending() - packet = self.super_socket.recv(MTU) if packet: self.processPacket(packet) # A string that has the same value for the request than for the @@ -146,18 +205,9 @@ class ScapyProtocol(abstract.FileDescriptor): self.processAnswer(packet, answer_hr) def stopSending(self): - self.stopReading() - self.super_socket.close() - if hasattr(self, "d"): - result = (self.answered_packets, self.sent_packets) - self.d.callback(result) - del self.d - - def write(self, packet): - """ - Write a scapy packet to the wire. - """ - return self.super_socket.send(packet) + result = (self.answered_packets, self.sent_packets) + self.d.callback(result) + self.factory.unRegisterProtocol(self) def sendPackets(self, packets): if not isinstance(packets, Gen): @@ -169,12 +219,18 @@ class ScapyProtocol(abstract.FileDescriptor): else: self.hr_sent_packets[hashret] = [packet] self.sent_packets.append(packet) - self.write(packet) + self.factory.send(packet) def startSending(self, packets): self._start_time = time.time() - self.startReading() + self.d = defer.Deferred() self.sendPackets(packets) return self.d +class ScapySniffer(ScapyProtocol): + def __init__(self, pcap_filename, *arg, **kw): + self.pcapwriter = PcapWriter(pcap_filename, *arg, **kw) + + def packetReceived(self, packet): + self.pcapwriter.write(packet)

1 0

[ooni-probe/master] Test and Implement HTTP Header Field Manipulation Test
by art＠torproject.org 25 Nov '12

25 Nov '12

commit fb5ab197c4efabd005286c32335010f24a102267 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 06:50:37 2012 +0100 Test and Implement HTTP Header Field Manipulation Test (rename it to what we had originally called it since it made most sense) * Extend TrueHeaders to support calculation of difference between two HTTP headers respectful of capitalization * Write unittests for TrueHeaders functions with decent code path coverage * Add commented out testdeck for running HTTP Header Field manipulation test * Fix bug in calculation of runtime of test --- before_i_commit.testdeck | 9 + nettests/core/http_header_field_manipulation.py | 181 ++++++++++++++++++++++ nettests/core/http_requests.py | 187 ----------------------- ooni/reporter.py | 2 +- ooni/utils/txagentwithsocks.py | 36 +++++ oonib/testhelpers/http_helpers.py | 5 +- tests/test_trueheaders.py | 41 +++++ 7 files changed, 272 insertions(+), 189 deletions(-) diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck index e0b30ea..7fb33f9 100644 --- a/before_i_commit.testdeck +++ b/before_i_commit.testdeck @@ -38,3 +38,12 @@ reportfile: http_url_lists.yamloo subargs: [-f, test_inputs/url_lists_file.txt] test: nettests/core/http_url_list.py +# XXX this is disabled because it requires oonib to be running +#- options: +# collector: null +# help: 0 +# logfile: null +# pcapfile: null +# reportfile: null +# subargs: [-h, test_inputs/test_header_field_manipulation.txt] +# test: nettests/core/http_header_field_manipulation.py diff --git a/nettests/core/http_header_field_manipulation.py b/nettests/core/http_header_field_manipulation.py new file mode 100644 index 0000000..08ee8c7 --- /dev/null +++ b/nettests/core/http_header_field_manipulation.py @@ -0,0 +1,181 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random +import json +import yaml + +from twisted.python import usage + +from ooni.utils import log, net, randomStr +from ooni.templates import httpt +from ooni.utils.txagentwithsocks import TrueHeaders + +def random_capitalization(string): + output = "" + original_string = string + string = string.swapcase() + for i in range(len(string)): + if random.randint(0, 1): + output += string[i].swapcase() + else: + output += string[i] + if original_string == output: + return random_capitalization(output) + else: + return output + +class UsageOptions(usage.Options): + optParameters = [ + ['backend', 'b', 'http://127.0.0.1:57001', + 'URL of the backend to use for sending the requests'], + ['headers', 'h', None, + 'Specify a yaml formatted file from which to read the request headers to send'] + ] + +class HTTPHeaderFieldManipulation(httpt.HTTPTest): + """ + It performes HTTP requests with request headers that vary capitalization + towards a backend. If we detect that the headers the backend received + matches the ones we have sent then we have detected tampering. + """ + name = "HTTP Header Field Manipulation" + author = "Arturo Filastò" + version = "0.1.3" + + randomizeUA = False + usageOptions = UsageOptions + + requiredOptions = ['backend'] + + def get_headers(self): + headers = {} + if self.localOptions['headers']: + try: + f = open(self.localOptions['headers']) + except IOError: + raise Exception("Specified input file does not exist") + content = ''.join(f.readlines()) + f.close() + headers = yaml.safe_load(content) + return headers + else: + # XXX generate these from a random choice taken from whatheaders.com + # http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip + headers = {"User-Agent": [random.choice(net.userAgents)[0]], + "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + "Accept-Encoding": ["gzip,deflate,sdch"], + "Accept-Language": ["en-US,en;q=0.8"], + "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], + "Host": [randomStr(15)+'.com'] + } + return headers + + def get_random_caps_headers(self): + headers = {} + normal_headers = self.get_headers() + for k, v in normal_headers.items(): + new_key = random_capitalization(k) + headers[new_key] = v + return headers + + def processInputs(self): + if self.localOptions['backend']: + self.url = self.localOptions['backend'] + else: + raise Exception("No backend specified") + + def processResponseBody(self, data): + self.check_for_tampering(data) + + def check_for_tampering(self, data): + """ + Here we do checks to verify if the request we made has been tampered + with. We have 3 categories of tampering: + + * **total** when the response is not a json object and therefore we were not + able to reach the ooniprobe test backend + + * **request_line_capitalization** when the HTTP Request line (e.x. GET / + HTTP/1.1) does not match the capitalization we set. + + * **header_field_number** when the number of headers we sent does not match + with the ones the backend received + + * **header_name_capitalization** when the header field names do not match + those that we sent. + + * **header_field_value** when the header field value does not match with the + one we transmitted. + """ + self.report['tampering'] = { + 'total': False, + 'request_line_capitalization': False, + 'header_name_capitalization': False, + 'header_field_value': False, + 'header_field_number': False + } + try: + response = json.loads(data) + except ValueError: + self.report['tampering']['total'] = True + return + + request_request_line = "%s / HTTP/1.1" % self.request_method + + try: + response_request_line = response['request_line'] + response_headers_dict = response['headers_dict'] + except KeyError: + self.report['tampering']['total'] = True + return + + if request_request_line != response_request_line: + self.report['tampering']['request_line_capitalization'] = True + + request_headers = TrueHeaders(self.request_headers) + diff = request_headers.getDiff(response_headers_dict, ignore=['Connection']) + if diff: + self.report['tampering']['header_field_name'] = True + else: + self.report['tampering']['header_field_name'] = False + self.report['tampering']['header_name_diff'] = list(diff) + + def test_get(self): + self.request_method = "GET" + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_get_random_capitalization(self): + self.request_method = random_capitalization("GET") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_post(self): + self.request_method = "POST" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_post_random_capitalization(self): + self.request_method = random_capitalization("POST") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_put(self): + self.request_method = "PUT" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_put_random_capitalization(self): + self.request_method = random_capitalization("PUT") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py deleted file mode 100644 index 5d67070..0000000 --- a/nettests/core/http_requests.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -import random -import json - -from twisted.python import usage - -from ooni.utils import log, net, randomStr -from ooni.templates import httpt - -def random_capitalization(string): - output = "" - original_string = string - string = string.swapcase() - for i in range(len(string)): - if random.randint(0, 1): - output += string[i].swapcase() - else: - output += string[i] - if original_string == output: - return random_capitalization(output) - else: - return output - -class UsageOptions(usage.Options): - optParameters = [ - ['backend', 'b', 'http://127.0.0.1:57001', - 'URL of the backend to use for sending the requests'], - ['headers', 'h', None, - 'Specify a yaml formatted file from which to read the request headers to send'] - ] - -class HTTPRequests(httpt.HTTPTest): - """ - This test is also known as Header Field manipulation. It performes HTTP - requests with variations in capitalization towards the backend. - """ - name = "HTTP Requests" - author = "Arturo Filastò" - version = "0.1.1" - - randomizeUA = False - usageOptions = UsageOptions - - requiredOptions = ['backend'] - - def processInputs(self): - if self.localOptions['backend']: - self.url = self.localOptions['backend'] - else: - raise Exception("No backend specified") - - def processResponseBody(self, data): - self.check_for_tampering(data) - - def check_for_tampering(self, data): - """ - Here we do checks to verify if the request we made has been tampered - with. We have 3 categories of tampering: - - * **total** when the response is not a json object and therefore we were not - able to reach the ooniprobe test backend - - * **request_line_capitalization** when the HTTP Request line (e.x. GET / - HTTP/1.1) does not match the capitalization we set. - - * **header_field_number** when the number of headers we sent does not match - with the ones the backend received - - * **header_name_capitalization** when the header field names do not match - those that we sent. - - * **header_field_value** when the header field value does not match with the - one we transmitted. - """ - self.report['tampering'] = {'total': False, - 'request_line_capitalization': False, - 'header_name_capitalization': False, - 'header_field_value': False, - 'header_field_number': False - } - - try: - response = json.loads(data) - except ValueError: - self.report['tampering']['total'] = True - return - - requestLine = "%s / HTTP/1.1" % self.request_method - if response['request_line'] != requestLine: - self.report['tampering']['request_line_capitalization'] = True - - # We compare against length -1 because the response will also contain - # the Connection: close header since we do not do persistent - # connections - if len(self.request_headers) != (len(response['headers_dict']) - 1): - self.report['tampering']['header_field_number'] = True - - for header, value in self.request_headers.items(): - # XXX this still needs some work - # in particular if the response headers are of different length or - # some extra headers get added in the response (so the lengths - # match), we will get header_name_capitalization set to true, while - # the actual tampering is the addition of an extraneous header - # field. - if header == "Connection": - # Ignore Connection header - continue - try: - response_value = response['headers_dict'][header] - if response_value != value[0]: - log.msg("Tampering detected because %s != %s" % (response_value, value[0])) - self.report['tampering']['header_field_value'] = True - except KeyError: - log.msg("Tampering detected because %s not in %s" % (header, response['headers_dict'])) - self.report['tampering']['header_name_capitalization'] = True - - def get_headers(self): - headers = {} - if self.localOptions['headers']: - # XXX test this code - try: - f = open(self.localOptions['headers']) - except IOError: - raise Exception("Specified input file does not exist") - content = ''.join(f.readlines()) - f.close() - headers = yaml.load(content) - return headers - else: - headers = {"User-Agent": [random.choice(net.userAgents)[0]], - "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], - "Accept-Encoding": ["gzip,deflate,sdch"], - "Accept-Language": ["en-US,en;q=0.8"], - "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], - "Host": [randomStr(15)+'.com'] - } - return headers - - def get_random_caps_headers(self): - headers = {} - normal_headers = self.get_headers() - for k, v in normal_headers.items(): - new_key = random_capitalization(k) - headers[new_key] = v - return headers - - def test_get(self): - self.request_method = "GET" - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_get_random_capitalization(self): - self.request_method = random_capitalization("GET") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_post(self): - self.request_method = "POST" - self.request_headers = self.get_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_post_random_capitalization(self): - self.request_method = random_capitalization("POST") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_put(self): - self.request_method = "PUT" - self.request_headers = self.get_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_put_random_capitalization(self): - self.request_method = random_capitalization("PUT") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - diff --git a/ooni/reporter.py b/ooni/reporter.py index b79f27f..63f501e 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -184,7 +184,7 @@ class OReporter(object): test_input = test.input test_started = test._start_time - test_runtime = test_started - time.time() + test_runtime = time.time() - test_started report = {'input': test_input, 'test_name': test_name, diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 57c27e4..7b000fc 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -4,6 +4,8 @@ # :licence: see LICENSE import struct +import itertools +from copy import copy from zope.interface import implements from twisted.web import client, _newclient, http_headers @@ -137,6 +139,40 @@ class TrueHeaders(http_headers.Headers): self._rawHeaders[name.lower()]['name'] = name self._rawHeaders[name.lower()]['values'] = values + def getDiff(self, header_dict, ignore=[]): + """ + ignore: specify a list of header fields to ignore + + Returns a set containing the header names that are not present in + header_dict or not present in self. + """ + diff = set() + field_names = [] + + headers_a = copy(self) + headers_b = TrueHeaders(header_dict) + for name in ignore: + try: + del headers_a._rawHeaders[name.lower()] + except KeyError: + pass + try: + del headers_b._rawHeaders[name.lower()] + except KeyError: + pass + + for k, v in itertools.chain(headers_a.getAllRawHeaders(), \ + headers_b.getAllRawHeaders()): + field_names.append(k) + + for name in field_names: + if self.getRawHeaders(name) and \ + name in header_dict: + pass + else: + diff.add(name) + return diff + def getAllRawHeaders(self): for k, v in self._rawHeaders.iteritems(): yield v['name'], v['values'] diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index b384216..1fa0ccb 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -77,7 +77,10 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): def allHeadersReceived(self): headers_dict = {} for k, v in self.headers: - headers_dict[k] = v + if k not in headers_dict: + headers_dict[k] = [] + headers_dict[k].append(v) + response = {'request_headers': self.headers, 'request_line': self.requestLine, 'headers_dict': headers_dict diff --git a/tests/test_trueheaders.py b/tests/test_trueheaders.py new file mode 100644 index 0000000..33521b8 --- /dev/null +++ b/tests/test_trueheaders.py @@ -0,0 +1,41 @@ +from twisted.trial import unittest + +from ooni.utils.txagentwithsocks import TrueHeaders + +dummy_headers_dict = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'] +} + +dummy_headers_dict2 = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header3': ['ValueA', 'ValueB'], +} + +dummy_headers_dict3 = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header4': ['ValueA', 'ValueB'], +} + + +class TestTrueHeaders(unittest.TestCase): + def test_names_match(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict), set()) + + def test_names_not_match(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3'])) + + th = TrueHeaders(dummy_headers_dict3) + self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3', 'Header4'])) + + def test_names_match_expect_ignore(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict2, ignore=['Header3']), set()) + + + +

1 0

[ooni-probe/master] Update unittests for runner
by art＠torproject.org 25 Nov '12

25 Nov '12

commit df21761de0f0f887a2650a4fd97f0e3778bd4e6d Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 02:52:20 2012 +0100 Update unittests for runner * Remove superflous debug messages --- ooni/reporter.py | 8 -------- ooni/runner.py | 1 - tests/test_runner.py | 10 ++++++---- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 26c6e43..b79f27f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -193,14 +193,6 @@ class OReporter(object): 'report': test_report} return self.writeReportEntry(report) - def allDone(self): - log.debug("allDone: Finished running all tests") - try: - log.debug("Stopping the reactor") - except: - log.debug("Unable to stop the reactor") - return None - class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. diff --git a/ooni/runner.py b/ooni/runner.py index bc271a6..d7856e6 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -279,5 +279,4 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") - oreporter.allDone() diff --git a/tests/test_runner.py b/tests/test_runner.py index fac4cd7..9b37d71 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -36,8 +36,6 @@ class TestRunner(unittest.TestCase): self.assertEqual(test_cases[1][1], 'test_a') def test_run_testcase_with_input_unit(self): - oreporter = DummyReporter() - oreporter.dummy_report = [] def done(result): report = oreporter.dummy_report self.assertEqual(len(report), 5*2) @@ -50,6 +48,9 @@ class TestRunner(unittest.TestCase): input_unit = InputUnit([0,1,2,3,4]) cmd_line_options = {'collector': None} + oreporter = DummyReporter(cmd_line_options) + oreporter.dummy_report = [] + test_cases, options = loadTestsAndOptions([DummyTestCase], cmd_line_options) @@ -58,8 +59,6 @@ class TestRunner(unittest.TestCase): return d def test_with_post_processing(self): - oreporter = DummyReporter() - oreporter.dummy_report = [] def done(result): report = oreporter.dummy_report self.assertEqual(len(report), 3) @@ -70,6 +69,9 @@ class TestRunner(unittest.TestCase): input_unit = InputUnit([None]) cmd_line_options = {'collector': None} + oreporter = DummyReporter(cmd_line_options) + oreporter.dummy_report = [] + test_cases, options = loadTestsAndOptions([DummyTestCasePP], cmd_line_options)

1 0

[ooni-probe/master] Refactoring on runner, reporter and oonicli
by art＠torproject.org 25 Nov '12

25 Nov '12

commit eb46b078184500efd2024d403170047e64b272de Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 01:48:51 2012 +0100 Refactoring on runner, reporter and oonicli * Include the command line arguments in the test report --- ooni/config.py | 32 ++++++++++++++++++-------------- ooni/oonicli.py | 35 +++++++++++++++-------------------- ooni/reporter.py | 24 +++++++++++++++++++----- ooni/runner.py | 40 +++++++++++++++++----------------------- ooniprobe.conf | 2 +- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index ef44ee4..5168c34 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -11,6 +11,9 @@ from twisted.internet import reactor, threads from ooni.utils import otime from ooni.utils import Storage +reports = Storage() +basic = None +cmd_line_options = None def get_root_path(): this_directory = os.path.dirname(__file__) @@ -18,19 +21,6 @@ def get_root_path(): root = os.path.abspath(root) return root -def oreport_filenames(file_name): - """ - returns the filenames for the pcap file and the yamloo report - - returns - yamloo_filename, pcap_filename - """ - test_name = '.'.join(file_name.split(".")[:-1]) - base_filename = "%s_%s_"+otime.timestamp()+".%s" - yamloo_filename = base_filename % (test_name, "report", "yamloo") - pcap_filename = base_filename % (test_name, "packets", "pcap") - return yamloo_filename, pcap_filename - def loadConfigFile(): """ This is a helper function that makes sure that the configuration attributes @@ -62,7 +52,21 @@ def loadConfigFile(): advanced[k] = v return basic, privacy, advanced -basic = None +class TestFilenameNotSet(Exception): + pass + +def generateReportFilenames(): + try: + test_file_name = os.path.basename(cmd_line_options['test']) + except IndexError: + raise TestFilenameNotSet + + test_name = '.'.join(test_file_name.split(".")[:-1]) + base_filename = "%s_%s_"+otime.timestamp()+".%s" + print "Setting yamloo to %s" % base_filename + reports.yamloo = base_filename % (test_name, "report", "yamloo") + reports.pcap = base_filename % (test_name, "packets", "pcap") + if not basic: # Here we make sure that we instance the config file attributes only once basic, privacy, advanced = loadConfigFile() diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c8385ed..41a31f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -70,7 +70,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] - self['subArgs'] = args[1:] + self['subargs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -95,24 +95,19 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - test_file_name = os.path.basename(cmd_line_options['test']) - - yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) + config.cmd_line_options = cmd_line_options + config.generateReportFilenames() if cmd_line_options['reportfile']: - yamloo_filename = cmd_line_options['reportfile'] - pcap_filename = yamloo_filename+".pcap" - - if os.path.exists(yamloo_filename): - print "Report already exists with filename %s" % yamloo_filename - print "Renaming it to %s" % yamloo_filename+'.old' - os.rename(yamloo_filename, yamloo_filename+'.old') - if os.path.exists(pcap_filename): - print "Report PCAP already exists with filename %s" % pcap_filename - print "Renaming it to %s" % pcap_filename+'.old' - os.rename(pcap_filename, pcap_filename+'.old') - - classes = runner.findTestClassesFromConfig(cmd_line_options) + config.reports.yamloo = cmd_line_options['reportfile'] + config.reports.pcap = config.reports.yamloo+".pcap" + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming it to %s" % config.reports.pcap+'.old' + os.rename(config.reports.pcap, config.reports.pcap+'.old') + + classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: try: @@ -121,13 +116,13 @@ def run(): print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) + print "Starting sniffer" - net.capturePackets(pcap_filename) + net.capturePackets(config.reports.pcap) log.start(cmd_line_options['logfile']) - tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + tests_d = runner.runTestCases(test_cases, options, cmd_line_options) tests_d.addBoth(testsEnded) reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 1e2924d..99b7f7a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -10,6 +10,7 @@ import itertools import logging import sys +import os import time import yaml import json @@ -155,6 +156,9 @@ def getTestDetails(options): defer.returnValue(test_details) class OReporter(object): + def __init__(self, cmd_line_options): + self.cmd_line_options = dict(cmd_line_options) + def createReport(self, options): """ Override this with your own logic to implement tests. @@ -172,7 +176,6 @@ class OReporter(object): def testDone(self, test, test_name): log.msg("Finished running %s" % test_name) - log.debug("Writing report") test_report = dict(test.report) if isinstance(test.input, Packet): @@ -204,8 +207,15 @@ class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ - def __init__(self, stream): - self._stream = stream + def __init__(self, cmd_line_options): + if os.path.exists(config.reports.yamloo): + log.msg("Report already exists with filename %s" % config.reports.yamloo) + log.msg("Renaming it to %s" % config.reports.yamloo+'.old') + os.rename(config.reports.yamloo, config.reports.yamloo+'.old') + + log.debug("Creating %s" % config.reports.yamloo) + self._stream = open(config.reports.yamloo, 'w+') + OReporter.__init__(self, cmd_line_options) def _writeln(self, line): self._write("%s\n" % line) @@ -233,6 +243,7 @@ class YAMLReporter(OReporter): self._writeln("###########################################") test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options self.writeReportEntry(test_details) @@ -250,7 +261,9 @@ class OONIBTestDetailsLookupFailed(Exception): pass class OONIBReporter(OReporter): - def __init__(self, backend_url): + def __init__(self, cmd_line_options): + self.backend_url = cmd_line_options['collector'] + from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: @@ -259,7 +272,7 @@ class OONIBReporter(OReporter): except Exception, e: log.exception(e) - self.backend_url = backend_url + OReporter.__init__(self, cmd_line_options) @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -306,6 +319,7 @@ class OONIBReporter(OReporter): software_version = '0.0.1' test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options content = '---\n' content += safe_dump(test_details) diff --git a/ooni/runner.py b/ooni/runner.py index 92aa406..5e8536c 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -23,11 +23,11 @@ from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor -from ooni import reporter +from ooni import reporter, config from ooni.utils import log, checkForRoot, NotRootError -def processTest(obj, cmd_line_options): +def processTest(obj): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -57,7 +57,7 @@ def processTest(obj, cmd_line_options): options = obj.usageOptions() - options.parseOptions(cmd_line_options['subArgs']) + options.parseOptions(config.cmd_line_options['subargs']) obj.localOptions = options if obj.inputFile: @@ -73,7 +73,7 @@ def processTest(obj, cmd_line_options): log.err("There was an error in running %s!" % test_name) log.err("%s" % e) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) + raise usage.UsageError("Error in parsing command line args for %s" % test_name) if obj.requiresRoot: try: @@ -90,28 +90,23 @@ def isTestCase(obj): except TypeError: return False -def findTestClassesFromConfig(cmd_line_options): +def findTestClassesFromFile(filename): """ Takes as input the command line config parameters and returns the test case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - :param cmd_line_options: - A configured and instantiated :class:`twisted.python.usage.Options` - class. + :param filename: + the absolute path to the file containing the ooniprobe test classes + :return: A list of class objects found in a file or module given on the commandline. """ - - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) + classes.append(processTest(val)) return classes def makeTestCases(klass, tests, method_prefix): @@ -227,8 +222,10 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): +def runTestCases(test_cases, options, cmd_line_options): + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -247,15 +244,12 @@ def runTestCases(test_cases, options, log.msg("options[0] = %s" % first) test_inputs = [None] - log.debug("Creating %s" % yamloo_filename) - if cmd_line_options['collector']: - log.debug("Using remote collector %s" % cmd_line_options['collector']) - oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + log.debug("Using remote collector") + oreporter = reporter.OONIBReporter(cmd_line_options) else: - reportFile = open(yamloo_filename, 'w+') - log.debug("Reporting to file %s" % reportFile) - oreporter = reporter.YAMLReporter(reportFile) + log.debug("Reporting to file %s" % config.reports.yamloo) + oreporter = reporter.YAMLReporter(cmd_line_options) try: input_unit_factory = InputUnitFactory(test_inputs) diff --git a/ooniprobe.conf b/ooniprobe.conf index ad761e1..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: false + debug: true threadpool_size: 10 tor_socksport: 9050 # For auto detection

1 0