[tor-commits] [ooni-probe/master] Start porting tests to new twisted based pattern

commit 7a60f4cd618540ff2ae64244867d9d94798616db Author: Arturo Filastò hellais@torproject.org Date: Wed May 23 21:45:38 2012 +0200

Start porting tests to new twisted based pattern --- lib/Makefile | 14 ++ lib/traceroute.py | 32 ---- lib/txtraceroute.py | 389 -------------------------------------------------- oonicli.py | 3 - plugins/bridget.py | 36 +++++ plugins/dropin.cache | 28 ++++ plugins/skel.py | 3 +- plugoo/tests.py | 32 ++++- tests/bridget.py | 9 +- 9 files changed, 110 insertions(+), 436 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile new file mode 100644 index 0000000..dfb2449 --- /dev/null +++ b/lib/Makefile @@ -0,0 +1,14 @@ +all: txtorcon txtraceroute + +txtraceroute: + echo "Processing dependency txtraceroute..." + git clone https://github.com/hellais/txtraceroute.git txtraceroute.git + mv txtraceroute.git/txtraceroute.py txtraceroute.py + rm -rf txtraceroute.git + +txtorcon: + echo "Processing dependency txtorcon..." + git clone https://github.com/meejah/txtorcon.git txtorcon.git + mv txtorcon.git/txtorcon txtorcon + rm -rf txtorcon.git + diff --git a/lib/traceroute.py b/lib/traceroute.py deleted file mode 100644 index c02722a..0000000 --- a/lib/traceroute.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys -import socket -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import threads - -from txtraceroute import traceroute - -def run(target, src_port, dst_port): - res = [] - @defer.inlineCallbacks - def start_trace(target, **settings): - hops = yield traceroute(target, **settings) - for hop in hops: - res.append(hop.get()) - reactor.stop() - - settings = dict(hop_callback=None, - timeout=2, - max_tries=3, - max_hops=30, proto="tcp") - try: - target = socket.gethostbyname(target) - except Exception, e: - print("could not resolve '%s': %s" % (target, str(e))) - sys.exit(1) - - reactor.callWhenRunning(start_trace, target, **settings) - reactor.run() - return res - -print run("8.8.8.8", 80, 80) diff --git a/lib/txtraceroute.py b/lib/txtraceroute.py deleted file mode 100644 index e18b558..0000000 --- a/lib/txtraceroute.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -# -# Copyright (c) 2012 Alexandre Fiori -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import json -import operator -import os -import socket -import struct -import sys -import time - -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import threads -from twisted.python import usage -from twisted.web.client import getPage - - -class iphdr(object): - def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None): - self.version = 4 - self.hlen = 5 - self.tos = 0 - self.length = 20 - self.id = os.getpid() - self.frag = 0 - self.ttl = 255 - self.proto = proto - self.cksum = 0 - self.src = src - self.saddr = socket.inet_aton(src) - self.dst = dst or "0.0.0.0" - self.daddr = socket.inet_aton(self.dst) - self.data = "" - - def assemble(self): - header = struct.pack('BBHHHBB', - (self.version & 0x0f) << 4 | (self.hlen & 0x0f), - self.tos, self.length + len(self.data), - socket.htons(self.id), self.frag, - self.ttl, self.proto) - return header + "\000\000" + self.saddr + self.daddr + self.data - - @classmethod - def disassemble(self, data): - ip = iphdr() - pkt = struct.unpack('!BBHHHBBH', data[:12]) - ip.version = (pkt[0] >> 4 & 0x0f) - ip.hlen = (pkt[0] & 0x0f) - ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:] - ip.saddr = data[12:16] - ip.daddr = data[16:20] - ip.src = socket.inet_ntoa(ip.saddr) - ip.dst = socket.inet_ntoa(ip.daddr) - return ip - - def __repr__(self): - return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \ - "%s -> %s" % \ - (self.tos, self.ttl, self.id, self.frag, self.proto, - self.length, self.src, self.dst) - -class tcphdr(object): - def __init__(self, data="", dport=4242, sport=4242): - self.seq = 123132 - self.hlen = 44 - self.flags = 2 - self.wsize = 200 - self.cksum = 123 - self.options = 0 - self.mss = 1460 - self.dport = dport - self.sport = sport - - def assemble(self): - header = struct.pack("!HHL", self.sport, self.dport, self.seq) - header += '\00\00\00\00' - header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags & - 0xff), self.wsize, self.cksum) - # XXX There is something wrong here fixme - options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1) - options += struct.pack("!BBL", 8, 10, 1209452188) - options += '\00'*4 - options += struct.pack("!BB", 4, 2) - options += '\00' - return header+options - - @classmethod - def checksum(self, data): - pass - - def disassemble(self, data): - tcp = tcphdr() - pkt = struct.unpack("!HHLH", data[:20]) - tcp.sport, tcp.dport, tcp.seq = pkt[:3] - tcp.hlen = (pkt[4] >> 10 ) & 0xff - tcp.flags = pkf[4] & 0xff - tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:28]) - return tcp - -class udphdr(object): - def __init__(self, data="", dport=4242, sport=4242): - self.dport = dport - self.sport = sport - self.cksum = 0 - self.length = 0 - self.data = data - - def assemble(self): - self.length = len(self.data) + 8 - part1 = struct.pack("!HHH", self.sport, self.dport, self.length) - cksum = self.checksum(self.data) - cksum = struct.pack("!H", cksum) - return part1 + cksum + self.data - - @classmethod - def checksum(self, data): - # XXX implement proper checksum - cksum = 0 - return cksum - - def disassemble(self, data): - udp = udphdr() - pkt = struct.unpack("!HHHH", data) - udp.src_port, udp.dst_port, udp.length, udp.cksum = pkt - return udp - -class icmphdr(object): - def __init__(self, data=""): - self.type = 8 - self.code = 0 - self.cksum = 0 - self.id = os.getpid() - self.sequence = 0 - self.data = data - - def assemble(self): - part1 = struct.pack("BB", self.type, self.code) - part2 = struct.pack("!HH", self.id, self.sequence) - cksum = self.checksum(part1 + "\000\000" + part2 + self.data) - cksum = struct.pack("!H", cksum) - return part1 + cksum + part2 + self.data - - @classmethod - def checksum(self, data): - if len(data) & 1: - data += "\0" - cksum = reduce(operator.add, - struct.unpack('!%dH' % (len(data) >> 1), data)) - cksum = (cksum >> 16) + (cksum & 0xffff) - cksum += (cksum >> 16) - cksum = (cksum & 0xffff) ^ 0xffff - return cksum - - @classmethod - def disassemble(self, data): - icmp = icmphdr() - pkt = struct.unpack("!BBHHH", data) - icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt - return icmp - - def __repr__(self): - return "ICMP (type %s, code %s, id %s, sequence %s)" % \ - (self.type, self.code, self.id, self.sequence) - - -@defer.inlineCallbacks -def geoip_lookup(ip): - try: - r = yield getPage("http://freegeoip.net/json/%s" % ip) - d = json.loads(r) - items = [d["country_name"], d["region_name"], d["city"]] - text = ", ".join([s for s in items if s]) - defer.returnValue(text.encode("utf-8")) - except Exception: - defer.returnValue("Unknown location") - - -@defer.inlineCallbacks -def reverse_lookup(ip): - try: - r = yield threads.deferToThread(socket.gethostbyaddr, ip) - defer.returnValue(r[0]) - except Exception: - defer.returnValue(None) - - -class Hop(object): - def __init__(self, target, ttl, proto="icmp"): - self.proto = proto - self.found = False - self.tries = 0 - self.last_try = 0 - self.remote_ip = None - self.remote_icmp = None - self.remote_host = None - self.location = "" - - self.ttl = ttl - self.ip = iphdr(dst=target) - self.ip.ttl = ttl - self.ip.id += ttl - - if proto is "icmp": - self.icmp = icmphdr("traceroute") - self.icmp.id = self.ip.id - self.ip.data = self.icmp.assemble() - elif proto is "udp": - self.udp = udphdr("blabla") - self.ip.data = self.udp.assemble() - self.ip.proto = socket.IPPROTO_UDP - elif proto is "tcp": - self.tcp = tcphdr() - self.ip.data = self.tcp.assemble() - self.ip.proto = socket.IPPROTO_TCP - - self._pkt = self.ip.assemble() - - @property - def pkt(self): - self.tries += 1 - self.last_try = time.time() - return self._pkt - - def get(self): - if self.found: - if self.remote_host: - ip = self.remote_host - else: - ip = self.remote_ip.src - ping = self.found - self.last_try - else: - ip = None - ping = None - - location = self.location if self.location else None - return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location} - - def __repr__(self): - if self.found: - if self.remote_host: - ip = ":: %s" % self.remote_host - else: - ip = ":: %s" % self.remote_ip.src - ping = "%0.3fs" % (self.found - self.last_try) - else: - ip = "??" - ping = "-" - - location = ":: %s" % self.location if self.location else "" - return "%02d. %s %s %s" % (self.ttl, ping, ip, location) - - -class TracerouteProtocol(object): - def __init__(self, target, **settings): - self.target = target - self.settings = settings - self.fd = socket.socket(socket.AF_INET, socket.SOCK_RAW, - socket.IPPROTO_ICMP) - self.fd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) - - self.hops = [] - self.out_queue = [] - self.waiting = True - self.deferred = defer.Deferred() - - reactor.addReader(self) - reactor.addWriter(self) - - # send 1st probe packet - self.out_queue.append(Hop(self.target, 1, settings.get("proto"))) - - def logPrefix(self): - return "TracerouteProtocol(%s)" % self.target - - def fileno(self): - return self.fd.fileno() - - @defer.inlineCallbacks - def hopFound(self, hop, ip, icmp): - hop.remote_ip = ip - hop.remote_icmp = icmp - - if (ip and icmp): - hop.found = time.time() - if self.settings.get("geoip_lookup") is True: - hop.location = yield geoip_lookup(ip.src) - - if self.settings.get("reverse_lookup") is True: - hop.remote_host = yield reverse_lookup(ip.src) - - ttl = hop.ttl + 1 - last = self.hops[-2:] - if len(last) == 2 and last[0].remote_ip == ip or \ - (ttl > (self.settings.get("max_hops", 30) + 1)): - done = True - else: - done = False - - if not done: - cb = self.settings.get("hop_callback") - if callable(cb): - yield defer.maybeDeferred(cb, hop) - - if not self.waiting: - if self.deferred: - self.deferred.callback(self.hops) - self.deferred = None - else: - self.out_queue.append(Hop(self.target, ttl, self.settings.get("proto"))) - - def doRead(self): - if not self.waiting or not self.hops: - return - - pkt = self.fd.recv(4096) - - # disassemble ip header - ip = iphdr.disassemble(pkt[:20]) - if ip.proto != socket.IPPROTO_ICMP: - return - - found = False - - # disassemble icmp header - icmp = icmphdr.disassemble(pkt[20:28]) - if icmp.type == 0 and icmp.id == self.hops[-1].icmp.id: - found = True - elif icmp.type == 11: - # disassemble referenced ip header - ref = iphdr.disassemble(pkt[28:48]) - if ref.dst == self.target: - found = True - - if ip.src == self.target: - self.waiting = False - - if found: - self.hopFound(self.hops[-1], ip, icmp) - - def hopTimeout(self, *ign): - hop = self.hops[-1] - if not hop.found: - if hop.tries < self.settings.get("max_tries", 3): - # retry - self.out_queue.append(hop) - else: - # give up and move forward - self.hopFound(hop, None, None) - - def doWrite(self): - if self.waiting and self.out_queue: - hop = self.out_queue.pop(0) - pkt = hop.pkt - if not self.hops or (self.hops and hop.ttl != self.hops[-1].ttl): - self.hops.append(hop) - self.fd.sendto(pkt, (hop.ip.dst, 0)) - - timeout = self.settings.get("timeout", 1) - reactor.callLater(timeout, self.hopTimeout) - - def connectionLost(self, why): - pass - - -def traceroute(target, **settings): - tr = TracerouteProtocol(target, **settings) - return tr.deferred diff --git a/oonicli.py b/oonicli.py index 68384f8..d290e68 100755 --- a/oonicli.py +++ b/oonicli.py @@ -40,9 +40,6 @@ def retrieve_plugoo(): print "Plugin Broken" print bi error = True - except BrokenMethodImplementation, bmi: - print "Plugin Broken" - error = True if error != False: print "Plugin Loaded!" return d diff --git a/plugins/bridget.py b/plugins/bridget.py new file mode 100644 index 0000000..7271468 --- /dev/null +++ b/plugins/bridget.py @@ -0,0 +1,36 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, TwistedTest +from twisted.internet import threads + +from tests.bridget import BridgeT as BridgeTlegacy +from tests.bridget import BridgeTAsset as BridgeTAsset +from ooniprobe import ooni + +o = ooni() + +class BridgetArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['bridge', 'b', None, 'Specify a single bridge']] + +class BridgeT(TwistedTest): + implements(IPlugin, ITest) + + shortName = "bridget" + description = "Bridget plugin" + requirements = None + options = BridgetArgs + + def experiment(self): + bridget = BridgeTlegacy(o) + o.logger.info("Starting bridget test") + print "ASSET:%s " % self.asset + d = threads.deferToThread(bridget.connect, self.asset) + d.addCallback(self.d_experiment.callback, None) + return d + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +bridget = BridgeT(None, None) diff --git a/plugins/dropin.cache b/plugins/dropin.cache index 7bc3990..9258b18 100755 --- a/plugins/dropin.cache +++ b/plugins/dropin.cache @@ -45,4 +45,32 @@ p21 S'skel' p22 sg10 +NsbasbsS'bridget' +p23 +g3 +(g4 +g5 +NtRp24 +(dp25 +g8 +S'plugins.bridget' +p26 +sg10 +Nsg11 +(lp27 +g3 +(g13 +g5 +NtRp28 +(dp29 +g16 +(lp30 +g18 +ag19 +asg20 +g24 +sg21 +S'bridget' +p31 +sg10 Nsbasbs. \ No newline at end of file diff --git a/plugins/skel.py b/plugins/skel.py index 0427239..93bc1bb 100644 --- a/plugins/skel.py +++ b/plugins/skel.py @@ -5,7 +5,8 @@ from plugoo.tests import ITest, TwistedTest

class SkelArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index']] + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']]

class SkelTest(TwistedTest): implements(IPlugin, ITest) diff --git a/plugoo/tests.py b/plugoo/tests.py index e74586c..39adfa2 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -81,7 +81,7 @@ class Test: if assets: self.logger.debug("Running through tests")

- if extradata['index']: + if extradata and 'index' in extradata: index = extradata['index'] else: index = None @@ -168,11 +168,17 @@ class TwistedTest(object): self.asset = asset self.arguments = arguments self.start_time = datetime.now() + self._parse_arguments() #self.ooninet = ooninet

def __repr__(self): return "<TwistedTest %s %s>" % (self.arguments, self.asset)

+ def _parse_arguments(self): + print self.arguments + if self.arguments and 'test' in self.arguments: + self.test = self.arguments['test'] + def finished(self, result): #self.ooninet.report(result) print "FINIHSED" @@ -182,12 +188,30 @@ class TwistedTest(object): result['run_time'] = self.end_time - self.start_time return self.d.callback(result)

+ def _do_experiment(self): + self.d_experiment = defer.Deferred() + self.d_experiment.addCallback(self._do_control) + self.experiment() + return self.d + + def _do_control(self, exp): + self.control(exp) + self.finished(dict()) + + def control(self, exp): + print "Doing control..." + + def experiment(self): + print "Doing experiment" + self.d_experiment.callback(None) + def startTest(self): - print "Starting test" + print "Starting test %s" % repr(self) self.d = defer.Deferred() result = {} - reactor.callLater(2.0, self.finished, result) - return self.d + #reactor.callLater(2.0, self.finished, result) + # Start experiment + return self._do_experiment()

class TwistedTestFactory(object):

diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/bridget.py b/tests/bridget.py index da014fd..a613f61 100644 --- a/tests/bridget.py +++ b/tests/bridget.py @@ -241,13 +241,6 @@ Log info file %s except: self.logger.error("Error in starting Tor (do you have tor installed?)")

- # XXX this only works on UNIX (do we care?) - # Make file reading non blocking - try: - fcntl.fcntl(p.stdout, fcntl.F_SETFL, os.O_NONBLOCK) - except: - self.logger.error("Unable to set file descriptor to non blocking") - self.logger.info("Testing bridge: %s" % bridge) while True: o = "" @@ -308,6 +301,7 @@ Log info file %s self.logger.error("Error IOError: EAGAIN") raise sys.exc_clear() + print "In this exception 1"

try: # Set the timeout for the socket wait @@ -376,3 +370,4 @@ def run(ooni, assets=None): bridget.print_failures() bridget.clean() ooni.logger.info("Testing completed!") +