- tor-commits - lists.torproject.org

[ooni-probe/master] Add proper documentation to HTTP Test template
by art＠torproject.org 25 Nov '12

25 Nov '12

commit e496f0d1b3d395fec039574fd212675885b9e017 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 08:20:57 2012 +0100 Add proper documentation to HTTP Test template * Do some minor refactoring --- ooni/templates/httpt.py | 177 +++++++++++++++++++++++++++++------------------ 1 files changed, 110 insertions(+), 67 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e166263..e36c049 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -24,6 +24,9 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders +class InvalidSocksProxyOption(Exception): + pass + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -51,11 +54,10 @@ class HTTPTest(NetTestCase): response = {} def _setUp(self): - log.debug("Setting up HTTPTest") try: import OpenSSL except: - log.err("Warning! pyOpenSSL is not installed. https websites will" + log.err("Warning! pyOpenSSL is not installed. https websites will " "not work") self.control_agent = Agent(reactor, sockshost="127.0.0.1", @@ -63,27 +65,35 @@ class HTTPTest(NetTestCase): sockshost, socksport = (None, None) if self.localOptions['socksproxy']: - self.report['socksproxy'] = self.localOptions['socksproxy'] - sockshost, socksport = self.localOptions['socksproxy'].split(':') + try: + sockshost, socksport = self.localOptions['socksproxy'].split(':') + self.report['socksproxy'] = self.localOptions['socksproxy'] + except ValueError: + raise InvalidSocksProxyOption socksport = int(socksport) - self.agent = Agent(reactor, sockshost=sockshost, - socksport=socksport) + self.agent = Agent(reactor, sockshost=sockshost, socksport=socksport) + + self.report['agent'] = 'agent' if self.followRedirects: try: from twisted.web.client import RedirectAgent self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) + self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted"\ "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") - self.report['errors'].append("Could not import RedirectAgent") self.processInputs() log.debug("Finished test setup") + def randomize_useragent(self, request): + user_agent = random.choice(userAgents) + request['headers']['User-Agent'] = [user_agent] + def processInputs(self): pass @@ -107,12 +117,14 @@ class HTTPTest(NetTestCase): else: self.processResponseBody(response_body) - def processResponseBody(self, data): + def processResponseBody(self, body): """ - This should handle all the response body smushing for getting it ready - to be passed onto the control. + Overwrite this method if you wish to interact with the response body of + every request that is made. + + Args: - @param data: The content of the body returned. + body (str): The body of the HTTP response """ pass @@ -120,7 +132,9 @@ class HTTPTest(NetTestCase): """ This should take care of dealing with the returned HTTP headers. - @param headers: The content of the returned headers. + Args: + + headers (dict): The returned header fields. """ pass @@ -128,32 +142,96 @@ class HTTPTest(NetTestCase): """ Handle a redirection via a 3XX HTTP status code. - @param location: the url that is being redirected to. + Here you may place logic that evaluates the destination that you are + being redirected to. Matches against known censor redirects, etc. + + Note: if self.followRedirects is set to True, then this method will + never be called. + XXX perhaps we may want to hook _handleResponse in RedirectAgent to + call processRedirect every time we get redirected. + + Args: + + location (str): the url that we are being redirected to. """ pass + def _cbResponse(self, response, request, + headers_processor, body_processor): + """ + This callback is fired once we have gotten a response for our request. + If we are using a RedirectAgent then this will fire once we have + reached the end of the redirect chain. + + Args: + + response (:twisted.web.iweb.IResponse:): a provider for getting our response + + request (dict): the dict containing our response (XXX this should be dropped) + + header_processor (func): a function to be called with argument a + dict containing the response headers. This will lead + self.headerProcessor to not be called. + + + body_processor (func): a function to be called with as argument the + body of the response. This will lead self.bodyProcessor to not + be called. + + """ + if not response: + log.err("Got no response for request %s" % request) + return + else: + log.debug("Got response %s" % response) + + if str(response.code).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + # [!] We are passing to the headers_processor the headers dict and + # not the Headers() object + response_headers_dict = list(response.headers.getAllRawHeaders()) + if headers_processor: + headers_processor(response_headers_dict) + else: + self.processResponseHeaders(response_headers_dict) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody, request, + response, body_processor) + + return finished + def doRequest(self, url, method="GET", headers={}, body=None, headers_processor=None, body_processor=None, use_tor=False): """ - Perform an HTTP request with the specified method. + Perform an HTTP request with the specified method and headers. + + Args: - url: the full url path of the request + url (str): the full URL of the request. The scheme may be either + http, https, or httpo for http over Tor Hidden Service. - method: the HTTP Method to be used + Kwargs: - headers: the request headers to be sent as a dict + method (str): the HTTP method name to use for the request - body: the request body + headers (dict): the request headers to send - headers_processor: a function to be used for processing the HTTP header - responses (defaults to self.processResponseHeaders). - This function takes as argument the HTTP headers as a - dict. + body (str): the request body - body_processory: a function to be used for processing the HTTP response - body (defaults to self.processResponseBody). - This function takes the response body as an argument. + headers_processor : a function to be used for processing the HTTP + header responses (defaults to self.processResponseHeaders). + This function takes as argument the HTTP headers as a dict. + + body_processory: a function to be used for processing the HTTP + response body (defaults to self.processResponseBody). This + function takes the response body as an argument. + + use_tor (bool): specify if the HTTP request should be done over Tor + or not. """ @@ -170,7 +248,7 @@ class HTTPTest(NetTestCase): log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) url = 's'+url - log.debug("Performing request %s %s %s" % (url, method, headers)) + log.msg("Performing request %s %s %s" % (url, method, headers)) request = {} request['method'] = method @@ -182,8 +260,6 @@ class HTTPTest(NetTestCase): log.debug("Randomizing user agent") self.randomize_useragent(request) - log.debug("Writing to report the request") - if 'requests' not in self.report: self.report['requests'] = [] @@ -198,51 +274,18 @@ class HTTPTest(NetTestCase): def errback(failure): failure.trap(ConnectionRefusedError, SOCKSError) - if type(failure.value) is ConnectionRefusedError: + if isinstance(failure.value, ConnectionRefusedError): log.err("Connection refused. The backend may be down") - else: - log.err("Sock error. The SOCK proxy may be down") - self.report["failure"] = str(failure.value) + self.report['failure'] = 'connection_refused_error' - def finished(data): - return + elif isinstance(failure.value, SOCKSError): + log.err("Sock error. The SOCK proxy may be down") + self.report['failure'] = 'sockserror' d = agent.request(request['method'], request['url'], headers, body_producer) - d.addErrback(errback) d.addCallback(self._cbResponse, request, headers_processor, body_processor) - d.addCallback(finished) + d.addErrback(errback) return d - def _cbResponse(self, response, request, headers_processor, - body_processor): - - if not response: - log.err("Got no response") - return - else: - log.debug("Got response %s" % response) - - if str(response.code).startswith('3'): - self.processRedirect(response.headers.getRawHeaders('Location')[0]) - - # [!] We are passing to the headers_processor the headers dict and - # not the Headers() object - response_headers_dict = list(response.headers.getAllRawHeaders()) - if headers_processor: - headers_processor(response_headers_dict) - else: - self.processResponseHeaders(response_headers_dict) - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody, request, - response, body_processor) - - return finished - - def randomize_useragent(self, request): - user_agent = random.choice(userAgents) - request['headers']['User-Agent'] = [user_agent] -

1 0

[ooni-probe/master] Make the sniffer not run in a separate thread, but use a non blocking fdesc
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 615ce75c47aec249b6b5a4c0b58fdf7a93f09582 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 10:08:47 2012 +0100 Make the sniffer not run in a separate thread, but use a non blocking fdesc * Do some refactoring of scapy testing, following Factory creational pattern and a pub-sub pattern for the readers and writers (inspired by muxTCP). * Other misc refactoring --- ooni/oonicli.py | 6 ++- ooni/templates/scapyt.py | 50 ++++++++--------- ooni/utils/net.py | 23 -------- ooni/utils/txscapy.py | 138 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 126 insertions(+), 91 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5c582b2..1a316b3 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -100,6 +100,7 @@ def runTest(cmd_line_options): classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: + from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() except NotRootError: @@ -108,7 +109,10 @@ def runTest(cmd_line_options): sys.exit(1) print "Starting sniffer" - net.capturePackets(config.reports.pcap) + config.scapyFactory = ScapyFactory(config.advanced.interface) + + sniffer = ScapySniffer(config.reports.pcap) + config.scapyFactory.registerProtocol(sniffer) return runner.runTestCases(test_cases, options, cmd_line_options) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 11b4381..cb02300 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -16,7 +16,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log from ooni import config -from ooni.utils.txscapy import ScapyProtocol, getDefaultIface +from ooni.utils.txscapy import ScapySender, getDefaultIface class BaseScapyTest(NetTestCase): """ @@ -66,20 +66,8 @@ class BaseScapyTest(NetTestCase): else: config.check_TCPerror_seqack = 0 - if config.advanced.interface == 'auto': - self.interface = getDefaultIface() - else: - self.interface = config.advanced.interface - - def reportSentPacket(self, packet): - if 'sent_packets' not in self.report: - self.report['sent_packets'] = [] - self.report['sent_packets'].append(packet) - - def reportReceivedPacket(self, packet): - if 'answered_packets' not in self.report: - self.report['answered_packets'] = [] - self.report['answered_packets'].append(packet) + self.report['sent_packets'] = [] + self.report['answered_packets'] = [] def finishedSendReceive(self, packets): """ @@ -98,8 +86,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - self.reportSentPacket(sent_packet) - self.reportReceivedPacket(received_packet) + self.report['sent_packets'].append(sent_packet) + self.report['answered_packets'].append(received_packet) return packets def sr(self, packets, *arg, **kw): @@ -107,8 +95,11 @@ class BaseScapyTest(NetTestCase): Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - d = scapyProtocol.startSending(packets) + scapySender = ScapySender() + + config.scapyFactory.registerProtocol(scapySender) + + d = scapySender.startSending(packets) d.addCallback(self.finishedSendReceive) return d @@ -123,12 +114,15 @@ class BaseScapyTest(NetTestCase): return packets[0][0][1] except IndexError: log.err("Got no response...") - return None + return packets + + scapySender = ScapySender() + scapySender.expected_answers = 1 + + config.scapyFactory.registerProtocol(scapySender) - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - scapyProtocol.expected_answers = 1 log.debug("Running sr1") - d = scapyProtocol.startSending(packets) + d = scapySender.startSending(packets) log.debug("Started to send") d.addCallback(self.finishedSendReceive) d.addCallback(done) @@ -138,9 +132,13 @@ class BaseScapyTest(NetTestCase): """ Wrapper around scapy.sendrecv.send for sending of packets at layer 3 """ - scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) - scapyProtocol.sendPackets(packets) - scapyProtocol.stopSending() + scapySender = ScapySender() + + config.scapyFactory.registerProtocol(scapySender) + + scapySender.sendPackets(packets) + + scapySender.stopSending() for packet in packets: self.reportSentPacket(packet) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 649dc64..df98412 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -82,29 +82,6 @@ class BodyReceiver(protocol.Protocol): def connectionLost(self, reason): self.finished.callback(self.data) -def capturePackets(pcap_filename): - from scapy.all import sniff - global stop_packet_capture - stop_packet_capture = False - - def stopCapture(): - # XXX this is a bit of a hack to stop capturing packets when we close - # the reactor. Ideally we would want to be able to do this - # programmatically, but this requires some work on implementing - # properly the sniff function with deferreds. - global stop_packet_capture - stop_packet_capture = True - - def writePacketToPcap(pkt): - from scapy.all import utils - pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True) - pcapwriter.write(pkt) - if stop_packet_capture: - sys.exit(1) - d = threads.deferToThread(sniff, lfilter=writePacketToPcap) - reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) - return d - def getSystemResolver(): """ XXX implement a function that returns the resolver that is currently diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 679eb42..8e479f8 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -32,7 +32,9 @@ try: config.pcap_dnet = True except ImportError, e: - log.err("pypcap or dnet not installed. Certain tests may not work.") + log.err("pypcap or dnet not installed. " + "Certain tests may not work.") + config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False @@ -59,6 +61,9 @@ def getNetworksFromRoutes(): return networks +class IfaceError(Exception): + pass + def getDefaultIface(): networks = getNetworksFromRoutes() for net in networks: @@ -66,45 +71,100 @@ def getDefaultIface(): return net.iface raise IfaceError -class TXPcapWriter(PcapWriter): - def __init__(self, *arg, **kw): - PcapWriter.__init__(self, *arg, **kw) - fdesc.setNonBlocking(self.f) +class ProtocolNotRegistered(Exception): + pass -class ScapyProtocol(abstract.FileDescriptor): +class ProtocolAlreadyRegistered(Exception): + pass + +class ScapyFactory(abstract.FileDescriptor): + """ + Inspired by muxTCP scapyLink: + https://github.com/enki/muXTCP/blob/master/scapyLink.py + """ def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) + if interface == 'auto': + interface = getDefaultIface() if not super_socket: - super_socket = conf.L3socket(iface=interface, promisc=True, filter='') + super_socket = conf.L3socket(iface=interface, + promisc=True, filter='') #super_socket = conf.L2socket(iface=interface) + self.protocols = [] fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket - self.interface = interface - self.timeout = timeout + def writeSomeData(self, data): + """ + XXX we actually want to use this, but this requires overriding doWrite + or writeSequence. + """ + pass - # This dict is used to store the unique hashes that allow scapy to - # match up request with answer - self.hr_sent_packets = {} + def send(self, packet): + """ + Write a scapy packet to the wire. + """ + return self.super_socket.send(packet) - # These are the packets we have received as answer to the ones we sent - self.answered_packets = [] + def fileno(self): + return self.super_socket.ins.fileno() - # These are the packets we send - self.sent_packets = [] + def doRead(self): + packet = self.super_socket.recv(MTU) + for protocol in self.protocols: + protocol.packetReceived(packet) + + def registerProtocol(self, protocol): + if not self.connected: + self.startReading() + + if protocol not in self.protocols: + protocol.factory = self + self.protocols.append(protocol) + else: + raise ProtocolAlreadyRegistered + + def unRegisterProtocol(self, protocol): + if protocol in self.protocols: + self.protocols.remove(protocol) + if len(self.protocols) == 0: + self.loseConnection() + else: + raise ProtocolNotRegistered + +class ScapyProtocol(object): + factory = None + + def packetReceived(self, packet): + """ + When you register a protocol, this method will be called with argument + the packet it received. - # This deferred will fire when we have finished sending a receiving packets. - self.d = defer.Deferred() - # Should we look for multiple answers for the same sent packet? - self.multi = False + Every protocol that is registered will have this method called. + """ + raise NotImplementedError - # When 0 we stop when all the packets we have sent have received an - # answer - self.expected_answers = 0 +class ScapySender(ScapyProtocol): + timeout = 5 + # This dict is used to store the unique hashes that allow scapy to + # match up request with answer + hr_sent_packets = {} - def fileno(self): - return self.super_socket.ins.fileno() + # These are the packets we have received as answer to the ones we sent + answered_packets = [] + + # These are the packets we send + sent_packets = [] + + # This deferred will fire when we have finished sending a receiving packets. + # Should we look for multiple answers for the same sent packet? + multi = False + + # When 0 we stop when all the packets we have sent have received an + # answer + expected_answers = 0 def processPacket(self, packet): """ @@ -131,11 +191,10 @@ class ScapyProtocol(abstract.FileDescriptor): log.debug("Got the number of expected answers") self.stopSending() - def doRead(self): + def packetReceived(self, packet): timeout = time.time() - self._start_time if self.timeout and time.time() - self._start_time > self.timeout: self.stopSending() - packet = self.super_socket.recv(MTU) if packet: self.processPacket(packet) # A string that has the same value for the request than for the @@ -146,18 +205,9 @@ class ScapyProtocol(abstract.FileDescriptor): self.processAnswer(packet, answer_hr) def stopSending(self): - self.stopReading() - self.super_socket.close() - if hasattr(self, "d"): - result = (self.answered_packets, self.sent_packets) - self.d.callback(result) - del self.d - - def write(self, packet): - """ - Write a scapy packet to the wire. - """ - return self.super_socket.send(packet) + result = (self.answered_packets, self.sent_packets) + self.d.callback(result) + self.factory.unRegisterProtocol(self) def sendPackets(self, packets): if not isinstance(packets, Gen): @@ -169,12 +219,18 @@ class ScapyProtocol(abstract.FileDescriptor): else: self.hr_sent_packets[hashret] = [packet] self.sent_packets.append(packet) - self.write(packet) + self.factory.send(packet) def startSending(self, packets): self._start_time = time.time() - self.startReading() + self.d = defer.Deferred() self.sendPackets(packets) return self.d +class ScapySniffer(ScapyProtocol): + def __init__(self, pcap_filename, *arg, **kw): + self.pcapwriter = PcapWriter(pcap_filename, *arg, **kw) + + def packetReceived(self, packet): + self.pcapwriter.write(packet)

1 0

[ooni-probe/master] Test and Implement HTTP Header Field Manipulation Test
by art＠torproject.org 25 Nov '12

25 Nov '12

commit fb5ab197c4efabd005286c32335010f24a102267 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 06:50:37 2012 +0100 Test and Implement HTTP Header Field Manipulation Test (rename it to what we had originally called it since it made most sense) * Extend TrueHeaders to support calculation of difference between two HTTP headers respectful of capitalization * Write unittests for TrueHeaders functions with decent code path coverage * Add commented out testdeck for running HTTP Header Field manipulation test * Fix bug in calculation of runtime of test --- before_i_commit.testdeck | 9 + nettests/core/http_header_field_manipulation.py | 181 ++++++++++++++++++++++ nettests/core/http_requests.py | 187 ----------------------- ooni/reporter.py | 2 +- ooni/utils/txagentwithsocks.py | 36 +++++ oonib/testhelpers/http_helpers.py | 5 +- tests/test_trueheaders.py | 41 +++++ 7 files changed, 272 insertions(+), 189 deletions(-) diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck index e0b30ea..7fb33f9 100644 --- a/before_i_commit.testdeck +++ b/before_i_commit.testdeck @@ -38,3 +38,12 @@ reportfile: http_url_lists.yamloo subargs: [-f, test_inputs/url_lists_file.txt] test: nettests/core/http_url_list.py +# XXX this is disabled because it requires oonib to be running +#- options: +# collector: null +# help: 0 +# logfile: null +# pcapfile: null +# reportfile: null +# subargs: [-h, test_inputs/test_header_field_manipulation.txt] +# test: nettests/core/http_header_field_manipulation.py diff --git a/nettests/core/http_header_field_manipulation.py b/nettests/core/http_header_field_manipulation.py new file mode 100644 index 0000000..08ee8c7 --- /dev/null +++ b/nettests/core/http_header_field_manipulation.py @@ -0,0 +1,181 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random +import json +import yaml + +from twisted.python import usage + +from ooni.utils import log, net, randomStr +from ooni.templates import httpt +from ooni.utils.txagentwithsocks import TrueHeaders + +def random_capitalization(string): + output = "" + original_string = string + string = string.swapcase() + for i in range(len(string)): + if random.randint(0, 1): + output += string[i].swapcase() + else: + output += string[i] + if original_string == output: + return random_capitalization(output) + else: + return output + +class UsageOptions(usage.Options): + optParameters = [ + ['backend', 'b', 'http://127.0.0.1:57001', + 'URL of the backend to use for sending the requests'], + ['headers', 'h', None, + 'Specify a yaml formatted file from which to read the request headers to send'] + ] + +class HTTPHeaderFieldManipulation(httpt.HTTPTest): + """ + It performes HTTP requests with request headers that vary capitalization + towards a backend. If we detect that the headers the backend received + matches the ones we have sent then we have detected tampering. + """ + name = "HTTP Header Field Manipulation" + author = "Arturo Filastò" + version = "0.1.3" + + randomizeUA = False + usageOptions = UsageOptions + + requiredOptions = ['backend'] + + def get_headers(self): + headers = {} + if self.localOptions['headers']: + try: + f = open(self.localOptions['headers']) + except IOError: + raise Exception("Specified input file does not exist") + content = ''.join(f.readlines()) + f.close() + headers = yaml.safe_load(content) + return headers + else: + # XXX generate these from a random choice taken from whatheaders.com + # http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip + headers = {"User-Agent": [random.choice(net.userAgents)[0]], + "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + "Accept-Encoding": ["gzip,deflate,sdch"], + "Accept-Language": ["en-US,en;q=0.8"], + "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], + "Host": [randomStr(15)+'.com'] + } + return headers + + def get_random_caps_headers(self): + headers = {} + normal_headers = self.get_headers() + for k, v in normal_headers.items(): + new_key = random_capitalization(k) + headers[new_key] = v + return headers + + def processInputs(self): + if self.localOptions['backend']: + self.url = self.localOptions['backend'] + else: + raise Exception("No backend specified") + + def processResponseBody(self, data): + self.check_for_tampering(data) + + def check_for_tampering(self, data): + """ + Here we do checks to verify if the request we made has been tampered + with. We have 3 categories of tampering: + + * **total** when the response is not a json object and therefore we were not + able to reach the ooniprobe test backend + + * **request_line_capitalization** when the HTTP Request line (e.x. GET / + HTTP/1.1) does not match the capitalization we set. + + * **header_field_number** when the number of headers we sent does not match + with the ones the backend received + + * **header_name_capitalization** when the header field names do not match + those that we sent. + + * **header_field_value** when the header field value does not match with the + one we transmitted. + """ + self.report['tampering'] = { + 'total': False, + 'request_line_capitalization': False, + 'header_name_capitalization': False, + 'header_field_value': False, + 'header_field_number': False + } + try: + response = json.loads(data) + except ValueError: + self.report['tampering']['total'] = True + return + + request_request_line = "%s / HTTP/1.1" % self.request_method + + try: + response_request_line = response['request_line'] + response_headers_dict = response['headers_dict'] + except KeyError: + self.report['tampering']['total'] = True + return + + if request_request_line != response_request_line: + self.report['tampering']['request_line_capitalization'] = True + + request_headers = TrueHeaders(self.request_headers) + diff = request_headers.getDiff(response_headers_dict, ignore=['Connection']) + if diff: + self.report['tampering']['header_field_name'] = True + else: + self.report['tampering']['header_field_name'] = False + self.report['tampering']['header_name_diff'] = list(diff) + + def test_get(self): + self.request_method = "GET" + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_get_random_capitalization(self): + self.request_method = random_capitalization("GET") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_post(self): + self.request_method = "POST" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_post_random_capitalization(self): + self.request_method = random_capitalization("POST") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_put(self): + self.request_method = "PUT" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + + def test_put_random_capitalization(self): + self.request_method = random_capitalization("PUT") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) + diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py deleted file mode 100644 index 5d67070..0000000 --- a/nettests/core/http_requests.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -import random -import json - -from twisted.python import usage - -from ooni.utils import log, net, randomStr -from ooni.templates import httpt - -def random_capitalization(string): - output = "" - original_string = string - string = string.swapcase() - for i in range(len(string)): - if random.randint(0, 1): - output += string[i].swapcase() - else: - output += string[i] - if original_string == output: - return random_capitalization(output) - else: - return output - -class UsageOptions(usage.Options): - optParameters = [ - ['backend', 'b', 'http://127.0.0.1:57001', - 'URL of the backend to use for sending the requests'], - ['headers', 'h', None, - 'Specify a yaml formatted file from which to read the request headers to send'] - ] - -class HTTPRequests(httpt.HTTPTest): - """ - This test is also known as Header Field manipulation. It performes HTTP - requests with variations in capitalization towards the backend. - """ - name = "HTTP Requests" - author = "Arturo Filastò" - version = "0.1.1" - - randomizeUA = False - usageOptions = UsageOptions - - requiredOptions = ['backend'] - - def processInputs(self): - if self.localOptions['backend']: - self.url = self.localOptions['backend'] - else: - raise Exception("No backend specified") - - def processResponseBody(self, data): - self.check_for_tampering(data) - - def check_for_tampering(self, data): - """ - Here we do checks to verify if the request we made has been tampered - with. We have 3 categories of tampering: - - * **total** when the response is not a json object and therefore we were not - able to reach the ooniprobe test backend - - * **request_line_capitalization** when the HTTP Request line (e.x. GET / - HTTP/1.1) does not match the capitalization we set. - - * **header_field_number** when the number of headers we sent does not match - with the ones the backend received - - * **header_name_capitalization** when the header field names do not match - those that we sent. - - * **header_field_value** when the header field value does not match with the - one we transmitted. - """ - self.report['tampering'] = {'total': False, - 'request_line_capitalization': False, - 'header_name_capitalization': False, - 'header_field_value': False, - 'header_field_number': False - } - - try: - response = json.loads(data) - except ValueError: - self.report['tampering']['total'] = True - return - - requestLine = "%s / HTTP/1.1" % self.request_method - if response['request_line'] != requestLine: - self.report['tampering']['request_line_capitalization'] = True - - # We compare against length -1 because the response will also contain - # the Connection: close header since we do not do persistent - # connections - if len(self.request_headers) != (len(response['headers_dict']) - 1): - self.report['tampering']['header_field_number'] = True - - for header, value in self.request_headers.items(): - # XXX this still needs some work - # in particular if the response headers are of different length or - # some extra headers get added in the response (so the lengths - # match), we will get header_name_capitalization set to true, while - # the actual tampering is the addition of an extraneous header - # field. - if header == "Connection": - # Ignore Connection header - continue - try: - response_value = response['headers_dict'][header] - if response_value != value[0]: - log.msg("Tampering detected because %s != %s" % (response_value, value[0])) - self.report['tampering']['header_field_value'] = True - except KeyError: - log.msg("Tampering detected because %s not in %s" % (header, response['headers_dict'])) - self.report['tampering']['header_name_capitalization'] = True - - def get_headers(self): - headers = {} - if self.localOptions['headers']: - # XXX test this code - try: - f = open(self.localOptions['headers']) - except IOError: - raise Exception("Specified input file does not exist") - content = ''.join(f.readlines()) - f.close() - headers = yaml.load(content) - return headers - else: - headers = {"User-Agent": [random.choice(net.userAgents)[0]], - "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], - "Accept-Encoding": ["gzip,deflate,sdch"], - "Accept-Language": ["en-US,en;q=0.8"], - "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], - "Host": [randomStr(15)+'.com'] - } - return headers - - def get_random_caps_headers(self): - headers = {} - normal_headers = self.get_headers() - for k, v in normal_headers.items(): - new_key = random_capitalization(k) - headers[new_key] = v - return headers - - def test_get(self): - self.request_method = "GET" - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_get_random_capitalization(self): - self.request_method = random_capitalization("GET") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_post(self): - self.request_method = "POST" - self.request_headers = self.get_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_post_random_capitalization(self): - self.request_method = random_capitalization("POST") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_put(self): - self.request_method = "PUT" - self.request_headers = self.get_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - def test_put_random_capitalization(self): - self.request_method = random_capitalization("PUT") - self.request_headers = self.get_random_caps_headers() - return self.doRequest(self.url, self.request_method, - headers=self.request_headers) - - diff --git a/ooni/reporter.py b/ooni/reporter.py index b79f27f..63f501e 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -184,7 +184,7 @@ class OReporter(object): test_input = test.input test_started = test._start_time - test_runtime = test_started - time.time() + test_runtime = time.time() - test_started report = {'input': test_input, 'test_name': test_name, diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 57c27e4..7b000fc 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -4,6 +4,8 @@ # :licence: see LICENSE import struct +import itertools +from copy import copy from zope.interface import implements from twisted.web import client, _newclient, http_headers @@ -137,6 +139,40 @@ class TrueHeaders(http_headers.Headers): self._rawHeaders[name.lower()]['name'] = name self._rawHeaders[name.lower()]['values'] = values + def getDiff(self, header_dict, ignore=[]): + """ + ignore: specify a list of header fields to ignore + + Returns a set containing the header names that are not present in + header_dict or not present in self. + """ + diff = set() + field_names = [] + + headers_a = copy(self) + headers_b = TrueHeaders(header_dict) + for name in ignore: + try: + del headers_a._rawHeaders[name.lower()] + except KeyError: + pass + try: + del headers_b._rawHeaders[name.lower()] + except KeyError: + pass + + for k, v in itertools.chain(headers_a.getAllRawHeaders(), \ + headers_b.getAllRawHeaders()): + field_names.append(k) + + for name in field_names: + if self.getRawHeaders(name) and \ + name in header_dict: + pass + else: + diff.add(name) + return diff + def getAllRawHeaders(self): for k, v in self._rawHeaders.iteritems(): yield v['name'], v['values'] diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index b384216..1fa0ccb 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -77,7 +77,10 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): def allHeadersReceived(self): headers_dict = {} for k, v in self.headers: - headers_dict[k] = v + if k not in headers_dict: + headers_dict[k] = [] + headers_dict[k].append(v) + response = {'request_headers': self.headers, 'request_line': self.requestLine, 'headers_dict': headers_dict diff --git a/tests/test_trueheaders.py b/tests/test_trueheaders.py new file mode 100644 index 0000000..33521b8 --- /dev/null +++ b/tests/test_trueheaders.py @@ -0,0 +1,41 @@ +from twisted.trial import unittest + +from ooni.utils.txagentwithsocks import TrueHeaders + +dummy_headers_dict = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'] +} + +dummy_headers_dict2 = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header3': ['ValueA', 'ValueB'], +} + +dummy_headers_dict3 = { + 'Header1': ['Value1', 'Value2'], + 'Header2': ['ValueA', 'ValueB'], + 'Header4': ['ValueA', 'ValueB'], +} + + +class TestTrueHeaders(unittest.TestCase): + def test_names_match(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict), set()) + + def test_names_not_match(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3'])) + + th = TrueHeaders(dummy_headers_dict3) + self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3', 'Header4'])) + + def test_names_match_expect_ignore(self): + th = TrueHeaders(dummy_headers_dict) + self.assertEqual(th.getDiff(dummy_headers_dict2, ignore=['Header3']), set()) + + + +

1 0

[ooni-probe/master] Update unittests for runner
by art＠torproject.org 25 Nov '12

25 Nov '12

commit df21761de0f0f887a2650a4fd97f0e3778bd4e6d Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 02:52:20 2012 +0100 Update unittests for runner * Remove superflous debug messages --- ooni/reporter.py | 8 -------- ooni/runner.py | 1 - tests/test_runner.py | 10 ++++++---- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 26c6e43..b79f27f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -193,14 +193,6 @@ class OReporter(object): 'report': test_report} return self.writeReportEntry(report) - def allDone(self): - log.debug("allDone: Finished running all tests") - try: - log.debug("Stopping the reactor") - except: - log.debug("Unable to stop the reactor") - return None - class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. diff --git a/ooni/runner.py b/ooni/runner.py index bc271a6..d7856e6 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -279,5 +279,4 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") - oreporter.allDone() diff --git a/tests/test_runner.py b/tests/test_runner.py index fac4cd7..9b37d71 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -36,8 +36,6 @@ class TestRunner(unittest.TestCase): self.assertEqual(test_cases[1][1], 'test_a') def test_run_testcase_with_input_unit(self): - oreporter = DummyReporter() - oreporter.dummy_report = [] def done(result): report = oreporter.dummy_report self.assertEqual(len(report), 5*2) @@ -50,6 +48,9 @@ class TestRunner(unittest.TestCase): input_unit = InputUnit([0,1,2,3,4]) cmd_line_options = {'collector': None} + oreporter = DummyReporter(cmd_line_options) + oreporter.dummy_report = [] + test_cases, options = loadTestsAndOptions([DummyTestCase], cmd_line_options) @@ -58,8 +59,6 @@ class TestRunner(unittest.TestCase): return d def test_with_post_processing(self): - oreporter = DummyReporter() - oreporter.dummy_report = [] def done(result): report = oreporter.dummy_report self.assertEqual(len(report), 3) @@ -70,6 +69,9 @@ class TestRunner(unittest.TestCase): input_unit = InputUnit([None]) cmd_line_options = {'collector': None} + oreporter = DummyReporter(cmd_line_options) + oreporter.dummy_report = [] + test_cases, options = loadTestsAndOptions([DummyTestCasePP], cmd_line_options)

1 0

[ooni-probe/master] Refactoring on runner, reporter and oonicli
by art＠torproject.org 25 Nov '12

25 Nov '12

commit eb46b078184500efd2024d403170047e64b272de Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 01:48:51 2012 +0100 Refactoring on runner, reporter and oonicli * Include the command line arguments in the test report --- ooni/config.py | 32 ++++++++++++++++++-------------- ooni/oonicli.py | 35 +++++++++++++++-------------------- ooni/reporter.py | 24 +++++++++++++++++++----- ooni/runner.py | 40 +++++++++++++++++----------------------- ooniprobe.conf | 2 +- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index ef44ee4..5168c34 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -11,6 +11,9 @@ from twisted.internet import reactor, threads from ooni.utils import otime from ooni.utils import Storage +reports = Storage() +basic = None +cmd_line_options = None def get_root_path(): this_directory = os.path.dirname(__file__) @@ -18,19 +21,6 @@ def get_root_path(): root = os.path.abspath(root) return root -def oreport_filenames(file_name): - """ - returns the filenames for the pcap file and the yamloo report - - returns - yamloo_filename, pcap_filename - """ - test_name = '.'.join(file_name.split(".")[:-1]) - base_filename = "%s_%s_"+otime.timestamp()+".%s" - yamloo_filename = base_filename % (test_name, "report", "yamloo") - pcap_filename = base_filename % (test_name, "packets", "pcap") - return yamloo_filename, pcap_filename - def loadConfigFile(): """ This is a helper function that makes sure that the configuration attributes @@ -62,7 +52,21 @@ def loadConfigFile(): advanced[k] = v return basic, privacy, advanced -basic = None +class TestFilenameNotSet(Exception): + pass + +def generateReportFilenames(): + try: + test_file_name = os.path.basename(cmd_line_options['test']) + except IndexError: + raise TestFilenameNotSet + + test_name = '.'.join(test_file_name.split(".")[:-1]) + base_filename = "%s_%s_"+otime.timestamp()+".%s" + print "Setting yamloo to %s" % base_filename + reports.yamloo = base_filename % (test_name, "report", "yamloo") + reports.pcap = base_filename % (test_name, "packets", "pcap") + if not basic: # Here we make sure that we instance the config file attributes only once basic, privacy, advanced = loadConfigFile() diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c8385ed..41a31f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -70,7 +70,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] - self['subArgs'] = args[1:] + self['subargs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -95,24 +95,19 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - test_file_name = os.path.basename(cmd_line_options['test']) - - yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) + config.cmd_line_options = cmd_line_options + config.generateReportFilenames() if cmd_line_options['reportfile']: - yamloo_filename = cmd_line_options['reportfile'] - pcap_filename = yamloo_filename+".pcap" - - if os.path.exists(yamloo_filename): - print "Report already exists with filename %s" % yamloo_filename - print "Renaming it to %s" % yamloo_filename+'.old' - os.rename(yamloo_filename, yamloo_filename+'.old') - if os.path.exists(pcap_filename): - print "Report PCAP already exists with filename %s" % pcap_filename - print "Renaming it to %s" % pcap_filename+'.old' - os.rename(pcap_filename, pcap_filename+'.old') - - classes = runner.findTestClassesFromConfig(cmd_line_options) + config.reports.yamloo = cmd_line_options['reportfile'] + config.reports.pcap = config.reports.yamloo+".pcap" + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming it to %s" % config.reports.pcap+'.old' + os.rename(config.reports.pcap, config.reports.pcap+'.old') + + classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: try: @@ -121,13 +116,13 @@ def run(): print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) + print "Starting sniffer" - net.capturePackets(pcap_filename) + net.capturePackets(config.reports.pcap) log.start(cmd_line_options['logfile']) - tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + tests_d = runner.runTestCases(test_cases, options, cmd_line_options) tests_d.addBoth(testsEnded) reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 1e2924d..99b7f7a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -10,6 +10,7 @@ import itertools import logging import sys +import os import time import yaml import json @@ -155,6 +156,9 @@ def getTestDetails(options): defer.returnValue(test_details) class OReporter(object): + def __init__(self, cmd_line_options): + self.cmd_line_options = dict(cmd_line_options) + def createReport(self, options): """ Override this with your own logic to implement tests. @@ -172,7 +176,6 @@ class OReporter(object): def testDone(self, test, test_name): log.msg("Finished running %s" % test_name) - log.debug("Writing report") test_report = dict(test.report) if isinstance(test.input, Packet): @@ -204,8 +207,15 @@ class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ - def __init__(self, stream): - self._stream = stream + def __init__(self, cmd_line_options): + if os.path.exists(config.reports.yamloo): + log.msg("Report already exists with filename %s" % config.reports.yamloo) + log.msg("Renaming it to %s" % config.reports.yamloo+'.old') + os.rename(config.reports.yamloo, config.reports.yamloo+'.old') + + log.debug("Creating %s" % config.reports.yamloo) + self._stream = open(config.reports.yamloo, 'w+') + OReporter.__init__(self, cmd_line_options) def _writeln(self, line): self._write("%s\n" % line) @@ -233,6 +243,7 @@ class YAMLReporter(OReporter): self._writeln("###########################################") test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options self.writeReportEntry(test_details) @@ -250,7 +261,9 @@ class OONIBTestDetailsLookupFailed(Exception): pass class OONIBReporter(OReporter): - def __init__(self, backend_url): + def __init__(self, cmd_line_options): + self.backend_url = cmd_line_options['collector'] + from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: @@ -259,7 +272,7 @@ class OONIBReporter(OReporter): except Exception, e: log.exception(e) - self.backend_url = backend_url + OReporter.__init__(self, cmd_line_options) @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -306,6 +319,7 @@ class OONIBReporter(OReporter): software_version = '0.0.1' test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options content = '---\n' content += safe_dump(test_details) diff --git a/ooni/runner.py b/ooni/runner.py index 92aa406..5e8536c 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -23,11 +23,11 @@ from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor -from ooni import reporter +from ooni import reporter, config from ooni.utils import log, checkForRoot, NotRootError -def processTest(obj, cmd_line_options): +def processTest(obj): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -57,7 +57,7 @@ def processTest(obj, cmd_line_options): options = obj.usageOptions() - options.parseOptions(cmd_line_options['subArgs']) + options.parseOptions(config.cmd_line_options['subargs']) obj.localOptions = options if obj.inputFile: @@ -73,7 +73,7 @@ def processTest(obj, cmd_line_options): log.err("There was an error in running %s!" % test_name) log.err("%s" % e) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) + raise usage.UsageError("Error in parsing command line args for %s" % test_name) if obj.requiresRoot: try: @@ -90,28 +90,23 @@ def isTestCase(obj): except TypeError: return False -def findTestClassesFromConfig(cmd_line_options): +def findTestClassesFromFile(filename): """ Takes as input the command line config parameters and returns the test case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - :param cmd_line_options: - A configured and instantiated :class:`twisted.python.usage.Options` - class. + :param filename: + the absolute path to the file containing the ooniprobe test classes + :return: A list of class objects found in a file or module given on the commandline. """ - - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) + classes.append(processTest(val)) return classes def makeTestCases(klass, tests, method_prefix): @@ -227,8 +222,10 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): +def runTestCases(test_cases, options, cmd_line_options): + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -247,15 +244,12 @@ def runTestCases(test_cases, options, log.msg("options[0] = %s" % first) test_inputs = [None] - log.debug("Creating %s" % yamloo_filename) - if cmd_line_options['collector']: - log.debug("Using remote collector %s" % cmd_line_options['collector']) - oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + log.debug("Using remote collector") + oreporter = reporter.OONIBReporter(cmd_line_options) else: - reportFile = open(yamloo_filename, 'w+') - log.debug("Reporting to file %s" % reportFile) - oreporter = reporter.YAMLReporter(reportFile) + log.debug("Reporting to file %s" % config.reports.yamloo) + oreporter = reporter.YAMLReporter(cmd_line_options) try: input_unit_factory = InputUnitFactory(test_inputs) diff --git a/ooniprobe.conf b/ooniprobe.conf index ad761e1..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: false + debug: true threadpool_size: 10 tor_socksport: 9050 # For auto detection

1 0

[ooni-probe/master] Implement test deck system for automating the specification of command line arguments for tests
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 263568f28bcac776da7f9936a0e80958c4027f46 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 02:43:10 2012 +0100 Implement test deck system for automating the specification of command line arguments for tests * Change before_i_commit script to use test deck system --- before_i_commit.sh | 13 ++------- before_i_commit.testdeck | 40 ++++++++++++++++++++++++++++ ooni/oonicli.py | 65 +++++++++++++++++++++++++++++----------------- ooni/reporter.py | 2 - ooni/runner.py | 2 - 5 files changed, 84 insertions(+), 38 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 5147149..8b8180f 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -14,15 +14,7 @@ rm before_i_commit.log find . -type f -name "*.py[co]" -delete -./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt - -./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt - -./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py - -./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt - -./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt +./bin/ooniprobe -i before_i_commit.testdeck echo "Below you should not see anything" echo "---------------------------------" @@ -32,5 +24,6 @@ echo "If you do, it means something is wrong." echo "Read through the log file and fix it." echo "If you are having some problems fixing some things that have to do with" echo "the core of OONI, let's first discuss it on IRC, or open a ticket" -less *yamloo +read +cat *yamloo | less rm -f *yamloo diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck new file mode 100644 index 0000000..e0b30ea --- /dev/null +++ b/before_i_commit.testdeck @@ -0,0 +1,40 @@ +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: captive_portal_test.yamloo + subargs: [] + test: nettests/core/captiveportal.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: dns_tamper_test.yamloo + subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt] + test: nettests/core/dnstamper.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_host.yamloo + subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt] + test: nettests/core/http_host.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_keyword_filtering.yamloo + subargs: [-b, 'http://ooni.nu/test/', -f, test_inputs/keyword_filtering_file.txt] + test: nettests/core/http_keyword_filtering.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_url_lists.yamloo + subargs: [-f, test_inputs/url_lists_file.txt] + test: nettests/core/http_url_list.py diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 41a31f6..5c582b2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -13,6 +13,7 @@ import sys import os import random import time +import yaml from twisted.internet import defer, reactor from twisted.application import app @@ -27,7 +28,7 @@ from ooni.utils import net from ooni.utils import checkForRoot, NotRootError from ooni.utils import log -class Options(usage.Options, app.ReactorSelectionMixin): +class Options(usage.Options): synopsis = """%s [options] [path to test].py """ % (os.path.basename(sys.argv[0]),) @@ -35,12 +36,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): " network tests. These are loaded from modules, packages and" " files listed on the command line") - optFlags = [["help", "h"], - ['debug-stacktraces', 'B', - 'Report deferred creation and callback stack traces'],] + optFlags = [["help", "h"]] optParameters = [["reportfile", "o", None, "report file name"], - ["collector", "c", None, + ["testdeck", "i", None, + "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], + ["collector", "c", None, "Address of the collector of test results. (example: http://127.0.0.1:8888)"], ["logfile", "l", None, "log file name"], ["pcapfile", "p", None, "pcap file name"]] @@ -68,6 +69,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): sys.settrace(spewer) def parseArgs(self, *args): + if self['testdeck']: + return try: self['test'] = args[0] self['subargs'] = args[1:] @@ -79,22 +82,9 @@ def testsEnded(*arg, **kw): You can place here all the post shutdown tasks. """ log.debug("testsEnded: Finished running all tests") + reactor.stop() -def run(): - """ - Call me to begin testing from a file. - """ - cmd_line_options = Options() - if len(sys.argv) == 1: - cmd_line_options.getUsage() - try: - cmd_line_options.parseOptions() - except usage.UsageError, ue: - raise SystemExit, "%s: %s" % (sys.argv[0], ue) - - if cmd_line_options['debug-stacktraces']: - defer.setDebugging(True) - +def runTest(cmd_line_options): config.cmd_line_options = cmd_line_options config.generateReportFilenames() @@ -120,10 +110,37 @@ def run(): print "Starting sniffer" net.capturePackets(config.reports.pcap) - log.start(cmd_line_options['logfile']) + return runner.runTestCases(test_cases, options, cmd_line_options) - tests_d = runner.runTestCases(test_cases, options, cmd_line_options) - tests_d.addBoth(testsEnded) +def run(): + """ + Call me to begin testing from a file. + """ + cmd_line_options = Options() + if len(sys.argv) == 1: + cmd_line_options.getUsage() + try: + cmd_line_options.parseOptions() + except usage.UsageError, ue: + raise SystemExit, "%s: %s" % (sys.argv[0], ue) - reactor.run() + deck_dl = [] + log.start(cmd_line_options['logfile']) + if cmd_line_options['testdeck']: + test_deck = yaml.safe_load(open(cmd_line_options['testdeck'])) + for test in test_deck: + del cmd_line_options + cmd_line_options = test['options'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + else: + log.msg("No test deck detected") + del cmd_line_options['testdeck'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + + d2 = defer.DeferredList(deck_dl) + d2.addCallback(testsEnded) + + reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 99b7f7a..26c6e43 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -197,10 +197,8 @@ class OReporter(object): log.debug("allDone: Finished running all tests") try: log.debug("Stopping the reactor") - reactor.stop() except: log.debug("Unable to stop the reactor") - pass return None class YAMLReporter(OReporter): diff --git a/ooni/runner.py b/ooni/runner.py index 5e8536c..bc271a6 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -262,7 +262,6 @@ def runTestCases(test_cases, options, cmd_line_options): yield oreporter.createReport(options) except reporter.OONIBReportCreationFailed: log.err("Error in creating new report") - reactor.stop() raise except Exception, e: log.exception(e) @@ -280,6 +279,5 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") - reactor.stop() oreporter.allDone()

1 0

[ooni-probe/master] Update before_i_commit script with command line option of DNS tamper
by art＠torproject.org 24 Nov '12

24 Nov '12

commit ad22e3182df140e05d86f15b43b14dd4f26e1873 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 23:08:20 2012 +0100 Update before_i_commit script with command line option of DNS tamper --- before_i_commit.sh | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 3e01366..5147149 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -16,7 +16,7 @@ find . -type f -name "*.py[co]" -delete ./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt -./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt +./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt ./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py

1 0

[ooni-probe/master] If libpcap and libdnet is present we will use them. If not use raw sockets.
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 3d100e9495bf2459e13bba73a2eaa8f1cd58d521 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 23:04:40 2012 +0100 If libpcap and libdnet is present we will use them. If not use raw sockets. --- ooni/utils/net.py | 1 - ooni/utils/txscapy.py | 32 ++++++++++++++++++++++++-------- ooniprobe.conf | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index e828977..649dc64 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -15,7 +15,6 @@ from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor from twisted.web.iweb import IBodyProducer -from scapy.all import utils from ooni.utils import log, txscapy diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 432a987..679eb42 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -17,13 +17,30 @@ from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract from zope.interface import implements -from scapy.all import PcapWriter, MTU -from scapy.all import BasePacketList, conf, PcapReader - -from scapy.all import conf, Gen, SetGen -from scapy.arch import pcapdnet +from scapy.config import conf from ooni.utils import log +from ooni import config + +try: + conf.use_pcap = True + conf.use_dnet = True + + from scapy.all import PcapWriter + from scapy.arch import pcapdnet + + config.pcap_dnet = True + +except ImportError, e: + log.err("pypcap or dnet not installed. Certain tests may not work.") + config.pcap_dnet = False + conf.use_pcap = False + conf.use_dnet = False + + from scapy.all import PcapWriter + +from scapy.all import BasePacketList, conf, PcapReader +from scapy.all import conf, Gen, SetGen, MTU def getNetworksFromRoutes(): from scapy.all import conf, ltoa, read_routes @@ -58,9 +75,8 @@ class ScapyProtocol(abstract.FileDescriptor): def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) if not super_socket: - # XXX this is what we would want - #super_socket = pcapdnet.L3dnetSocket(iface=interface) - super_socket = conf.L3Socket(iface=interface) + super_socket = conf.L3socket(iface=interface, promisc=True, filter='') + #super_socket = conf.L2socket(iface=interface) fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket diff --git a/ooniprobe.conf b/ooniprobe.conf index e9f208f..ad761e1 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: true + debug: false threadpool_size: 10 tor_socksport: 9050 # For auto detection

1 0

[ooni-probe/master] Comment out what we would want to have
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 17e73e2253cf27e6c2d21d11bf33bcedeb82d7a1 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 21:39:35 2012 +0100 Comment out what we would want to have * Uncomment set close-on-exec on the socket file descriptor. Would not be useful in our case since we are not forking, but it's does not hurt to be safe. * This functionality does not work on windows based machines, but twisteds wrapper for it takes care of handling that. --- ooni/utils/txscapy.py | 12 ++++-------- 1 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index e41d649..432a987 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -25,9 +25,6 @@ from scapy.arch import pcapdnet from ooni.utils import log -conf.use_pcap = True -conf.use_dnet = True - def getNetworksFromRoutes(): from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress @@ -60,13 +57,12 @@ class TXPcapWriter(PcapWriter): class ScapyProtocol(abstract.FileDescriptor): def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) - # By default we use the conf.L3socket if not super_socket: - super_socket = pcapdnet.L3dnetSocket(iface=interface) - print super_socket - log.msg("Creating layer 3 socket with interface %s" % interface) + # XXX this is what we would want + #super_socket = pcapdnet.L3dnetSocket(iface=interface) + super_socket = conf.L3Socket(iface=interface) - #fdesc._setCloseOnExec(super_socket.ins.fileno()) + fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket self.interface = interface

1 0

[ooni-probe/master] Fix bug in tests that do not have usageOptions
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 18a6cde20774311de552fcae966d847b5eb9f187 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 20:43:23 2012 +0100 Fix bug in tests that do not have usageOptions --- nettests/core/traceroute.py | 1 + ooni/runner.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index 6a443d9..45bbb67 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -19,6 +19,7 @@ class UsageOptions(usage.Options): ['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'], ['srcport', 'p', None, 'Set the source port to a specific value (only applies to TCP and UDP)'] ] + optFlags = [['randomize','r', 'Randomize the source port']] class TracerouteTest(scapyt.BaseScapyTest): diff --git a/ooni/runner.py b/ooni/runner.py index f5ab651..92aa406 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -39,12 +39,13 @@ def processTest(obj, cmd_line_options): A configured and instantiated :class:`twisted.python.usage.Options` class. """ + if not hasattr(obj.usageOptions, 'optParameters'): + obj.usageOptions.optParameters = [] + if obj.inputFile: obj.usageOptions.optParameters.append(obj.inputFile) if obj.baseParameters: - if not hasattr(obj.usageOptions, 'optParameters'): - obj.usageOptions.optParameters = [] for parameter in obj.baseParameters: obj.usageOptions.optParameters.append(parameter)

1 0