[tor-commits] [ooni-probe/master] Start porting tests to new twisted based pattern

art at torproject.org art at torproject.org
Thu May 31 03:01:42 UTC 2012


commit 7a60f4cd618540ff2ae64244867d9d94798616db
Author: Arturo Filastò <hellais at torproject.org>
Date:   Wed May 23 21:45:38 2012 +0200

    Start porting tests to new twisted based pattern
---
 lib/Makefile         |   14 ++
 lib/traceroute.py    |   32 ----
 lib/txtraceroute.py  |  389 --------------------------------------------------
 oonicli.py           |    3 -
 plugins/bridget.py   |   36 +++++
 plugins/dropin.cache |   28 ++++
 plugins/skel.py      |    3 +-
 plugoo/tests.py      |   32 ++++-
 tests/bridget.py     |    9 +-
 9 files changed, 110 insertions(+), 436 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
new file mode 100644
index 0000000..dfb2449
--- /dev/null
+++ b/lib/Makefile
@@ -0,0 +1,14 @@
+all: txtorcon txtraceroute
+
+txtraceroute:
+	echo "Processing dependency txtraceroute..."
+	git clone https://github.com/hellais/txtraceroute.git  txtraceroute.git
+	mv txtraceroute.git/txtraceroute.py txtraceroute.py
+	rm -rf txtraceroute.git
+
+txtorcon:
+	echo "Processing dependency txtorcon..."
+	git clone https://github.com/meejah/txtorcon.git txtorcon.git
+	mv txtorcon.git/txtorcon txtorcon
+	rm -rf txtorcon.git
+
diff --git a/lib/traceroute.py b/lib/traceroute.py
deleted file mode 100644
index c02722a..0000000
--- a/lib/traceroute.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import sys
-import socket
-from twisted.internet import defer
-from twisted.internet import reactor
-from twisted.internet import threads
-
-from txtraceroute import traceroute
-
-def run(target, src_port, dst_port):
-    res = []
-    @defer.inlineCallbacks
-    def start_trace(target, **settings):
-        hops = yield traceroute(target, **settings)
-        for hop in hops:
-            res.append(hop.get())
-        reactor.stop()
-
-    settings = dict(hop_callback=None,
-                    timeout=2,
-                    max_tries=3,
-                    max_hops=30, proto="tcp")
-    try:
-        target = socket.gethostbyname(target)
-    except Exception, e:
-        print("could not resolve '%s': %s" % (target, str(e)))
-        sys.exit(1)
-
-    reactor.callWhenRunning(start_trace, target, **settings)
-    reactor.run()
-    return res
-
-print run("8.8.8.8", 80, 80)
diff --git a/lib/txtraceroute.py b/lib/txtraceroute.py
deleted file mode 100644
index e18b558..0000000
--- a/lib/txtraceroute.py
+++ /dev/null
@@ -1,389 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-#
-# Copyright (c) 2012 Alexandre Fiori
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-import json
-import operator
-import os
-import socket
-import struct
-import sys
-import time
-
-from twisted.internet import defer
-from twisted.internet import reactor
-from twisted.internet import threads
-from twisted.python import usage
-from twisted.web.client import getPage
-
-
-class iphdr(object):
-    def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None):
-        self.version = 4
-        self.hlen = 5
-        self.tos = 0
-        self.length = 20
-        self.id = os.getpid()
-        self.frag = 0
-        self.ttl = 255
-        self.proto = proto
-        self.cksum = 0
-        self.src = src
-        self.saddr = socket.inet_aton(src)
-        self.dst = dst or "0.0.0.0"
-        self.daddr = socket.inet_aton(self.dst)
-        self.data = ""
-
-    def assemble(self):
-        header = struct.pack('BBHHHBB',
-                             (self.version & 0x0f) << 4 | (self.hlen & 0x0f),
-                             self.tos, self.length + len(self.data),
-                             socket.htons(self.id), self.frag,
-                             self.ttl, self.proto)
-        return header + "\000\000" + self.saddr + self.daddr + self.data
-
-    @classmethod
-    def disassemble(self, data):
-        ip = iphdr()
-        pkt = struct.unpack('!BBHHHBBH', data[:12])
-        ip.version = (pkt[0] >> 4 & 0x0f)
-        ip.hlen = (pkt[0] & 0x0f)
-        ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:]
-        ip.saddr = data[12:16]
-        ip.daddr = data[16:20]
-        ip.src = socket.inet_ntoa(ip.saddr)
-        ip.dst = socket.inet_ntoa(ip.daddr)
-        return ip
-
-    def __repr__(self):
-        return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \
-               "%s -> %s" % \
-               (self.tos, self.ttl, self.id, self.frag, self.proto,
-                self.length, self.src, self.dst)
-
-class tcphdr(object):
-    def __init__(self, data="", dport=4242, sport=4242):
-        self.seq = 123132
-        self.hlen = 44
-        self.flags = 2
-        self.wsize = 200
-        self.cksum = 123
-        self.options = 0
-        self.mss = 1460
-        self.dport = dport
-        self.sport = sport
-
-    def assemble(self):
-        header = struct.pack("!HHL", self.sport, self.dport, self.seq)
-        header += '\00\00\00\00'
-        header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags &
-            0xff), self.wsize, self.cksum)
-        # XXX There is something wrong here fixme
-        options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1)
-        options += struct.pack("!BBL", 8, 10, 1209452188)
-        options += '\00'*4
-        options += struct.pack("!BB", 4, 2)
-        options += '\00'
-        return header+options
-
-    @classmethod
-    def checksum(self, data):
-        pass
-
-    def disassemble(self, data):
-        tcp = tcphdr()
-        pkt = struct.unpack("!HHLH", data[:20])
-        tcp.sport, tcp.dport, tcp.seq = pkt[:3]
-        tcp.hlen = (pkt[4] >> 10 ) & 0xff
-        tcp.flags = pkf[4] & 0xff
-        tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:28])
-        return tcp
-
-class udphdr(object):
-    def __init__(self, data="", dport=4242, sport=4242):
-        self.dport = dport
-        self.sport = sport
-        self.cksum = 0
-        self.length = 0
-        self.data = data
-
-    def assemble(self):
-        self.length = len(self.data) + 8
-        part1 = struct.pack("!HHH", self.sport, self.dport, self.length)
-        cksum = self.checksum(self.data)
-        cksum = struct.pack("!H", cksum)
-        return part1 + cksum + self.data
-
-    @classmethod
-    def checksum(self, data):
-        # XXX implement proper checksum
-        cksum = 0
-        return cksum
-
-    def disassemble(self, data):
-        udp = udphdr()
-        pkt = struct.unpack("!HHHH", data)
-        udp.src_port, udp.dst_port, udp.length, udp.cksum = pkt
-        return udp
-
-class icmphdr(object):
-    def __init__(self, data=""):
-        self.type = 8
-        self.code = 0
-        self.cksum = 0
-        self.id = os.getpid()
-        self.sequence = 0
-        self.data = data
-
-    def assemble(self):
-        part1 = struct.pack("BB", self.type, self.code)
-        part2 = struct.pack("!HH", self.id, self.sequence)
-        cksum = self.checksum(part1 + "\000\000" + part2 + self.data)
-        cksum = struct.pack("!H", cksum)
-        return part1 + cksum + part2 + self.data
-
-    @classmethod
-    def checksum(self, data):
-        if len(data) & 1:
-            data += "\0"
-        cksum = reduce(operator.add,
-                       struct.unpack('!%dH' % (len(data) >> 1), data))
-        cksum = (cksum >> 16) + (cksum & 0xffff)
-        cksum += (cksum >> 16)
-        cksum = (cksum & 0xffff) ^ 0xffff
-        return cksum
-
-    @classmethod
-    def disassemble(self, data):
-        icmp = icmphdr()
-        pkt = struct.unpack("!BBHHH", data)
-        icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt
-        return icmp
-
-    def __repr__(self):
-        return "ICMP (type %s, code %s, id %s, sequence %s)" % \
-               (self.type, self.code, self.id, self.sequence)
-
-
- at defer.inlineCallbacks
-def geoip_lookup(ip):
-    try:
-        r = yield getPage("http://freegeoip.net/json/%s" % ip)
-        d = json.loads(r)
-        items = [d["country_name"], d["region_name"], d["city"]]
-        text = ", ".join([s for s in items if s])
-        defer.returnValue(text.encode("utf-8"))
-    except Exception:
-        defer.returnValue("Unknown location")
-
-
- at defer.inlineCallbacks
-def reverse_lookup(ip):
-    try:
-        r = yield threads.deferToThread(socket.gethostbyaddr, ip)
-        defer.returnValue(r[0])
-    except Exception:
-        defer.returnValue(None)
-
-
-class Hop(object):
-    def __init__(self, target, ttl, proto="icmp"):
-        self.proto = proto
-        self.found = False
-        self.tries = 0
-        self.last_try = 0
-        self.remote_ip = None
-        self.remote_icmp = None
-        self.remote_host = None
-        self.location = ""
-
-        self.ttl = ttl
-        self.ip = iphdr(dst=target)
-        self.ip.ttl = ttl
-        self.ip.id += ttl
-
-        if proto is "icmp":
-            self.icmp = icmphdr("traceroute")
-            self.icmp.id = self.ip.id
-            self.ip.data = self.icmp.assemble()
-        elif proto is "udp":
-            self.udp = udphdr("blabla")
-            self.ip.data = self.udp.assemble()
-            self.ip.proto = socket.IPPROTO_UDP
-        elif proto is "tcp":
-            self.tcp = tcphdr()
-            self.ip.data = self.tcp.assemble()
-            self.ip.proto = socket.IPPROTO_TCP
-
-        self._pkt = self.ip.assemble()
-
-    @property
-    def pkt(self):
-        self.tries += 1
-        self.last_try = time.time()
-        return self._pkt
-
-    def get(self):
-        if self.found:
-            if self.remote_host:
-                ip = self.remote_host
-            else:
-                ip = self.remote_ip.src
-            ping = self.found - self.last_try
-        else:
-            ip = None
-            ping = None
-
-        location = self.location if self.location else None
-        return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location}
-
-    def __repr__(self):
-        if self.found:
-            if self.remote_host:
-                ip = ":: %s" % self.remote_host
-            else:
-                ip = ":: %s" % self.remote_ip.src
-            ping = "%0.3fs" % (self.found - self.last_try)
-        else:
-            ip = "??"
-            ping = "-"
-
-        location = ":: %s" % self.location if self.location else ""
-        return "%02d. %s %s %s" % (self.ttl, ping, ip, location)
-
-
-class TracerouteProtocol(object):
-    def __init__(self, target, **settings):
-        self.target = target
-        self.settings = settings
-        self.fd = socket.socket(socket.AF_INET, socket.SOCK_RAW,
-                                socket.IPPROTO_ICMP)
-        self.fd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
-
-        self.hops = []
-        self.out_queue = []
-        self.waiting = True
-        self.deferred = defer.Deferred()
-
-        reactor.addReader(self)
-        reactor.addWriter(self)
-
-        # send 1st probe packet
-        self.out_queue.append(Hop(self.target, 1, settings.get("proto")))
-
-    def logPrefix(self):
-        return "TracerouteProtocol(%s)" % self.target
-
-    def fileno(self):
-        return self.fd.fileno()
-
-    @defer.inlineCallbacks
-    def hopFound(self, hop, ip, icmp):
-        hop.remote_ip = ip
-        hop.remote_icmp = icmp
-
-        if (ip and icmp):
-            hop.found = time.time()
-            if self.settings.get("geoip_lookup") is True:
-                hop.location = yield geoip_lookup(ip.src)
-
-            if self.settings.get("reverse_lookup") is True:
-                hop.remote_host = yield reverse_lookup(ip.src)
-
-        ttl = hop.ttl + 1
-        last = self.hops[-2:]
-        if len(last) == 2 and last[0].remote_ip == ip or \
-           (ttl > (self.settings.get("max_hops", 30) + 1)):
-            done = True
-        else:
-            done = False
-
-        if not done:
-            cb = self.settings.get("hop_callback")
-            if callable(cb):
-                yield defer.maybeDeferred(cb, hop)
-
-        if not self.waiting:
-            if self.deferred:
-                self.deferred.callback(self.hops)
-                self.deferred = None
-        else:
-            self.out_queue.append(Hop(self.target, ttl, self.settings.get("proto")))
-
-    def doRead(self):
-        if not self.waiting or not self.hops:
-            return
-
-        pkt = self.fd.recv(4096)
-
-        # disassemble ip header
-        ip = iphdr.disassemble(pkt[:20])
-        if ip.proto != socket.IPPROTO_ICMP:
-            return
-
-        found = False
-
-        # disassemble icmp header
-        icmp = icmphdr.disassemble(pkt[20:28])
-        if icmp.type == 0 and icmp.id == self.hops[-1].icmp.id:
-            found = True
-        elif icmp.type == 11:
-            # disassemble referenced ip header
-            ref = iphdr.disassemble(pkt[28:48])
-            if ref.dst == self.target:
-                found = True
-
-        if ip.src == self.target:
-            self.waiting = False
-
-        if found:
-            self.hopFound(self.hops[-1], ip, icmp)
-
-    def hopTimeout(self, *ign):
-        hop = self.hops[-1]
-        if not hop.found:
-            if hop.tries < self.settings.get("max_tries", 3):
-                # retry
-                self.out_queue.append(hop)
-            else:
-                # give up and move forward
-                self.hopFound(hop, None, None)
-
-    def doWrite(self):
-        if self.waiting and self.out_queue:
-            hop = self.out_queue.pop(0)
-            pkt = hop.pkt
-            if not self.hops or (self.hops and hop.ttl != self.hops[-1].ttl):
-                self.hops.append(hop)
-            self.fd.sendto(pkt, (hop.ip.dst, 0))
-
-            timeout = self.settings.get("timeout", 1)
-            reactor.callLater(timeout, self.hopTimeout)
-
-    def connectionLost(self, why):
-        pass
-
-
-def traceroute(target, **settings):
-    tr = TracerouteProtocol(target, **settings)
-    return tr.deferred
diff --git a/oonicli.py b/oonicli.py
index 68384f8..d290e68 100755
--- a/oonicli.py
+++ b/oonicli.py
@@ -40,9 +40,6 @@ def retrieve_plugoo():
             print "Plugin Broken"
             print bi
             error = True
-        except BrokenMethodImplementation, bmi:
-            print "Plugin Broken"
-            error = True
     if error != False:
         print "Plugin Loaded!"
     return d
diff --git a/plugins/bridget.py b/plugins/bridget.py
new file mode 100644
index 0000000..7271468
--- /dev/null
+++ b/plugins/bridget.py
@@ -0,0 +1,36 @@
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from plugoo.tests import ITest, TwistedTest
+from twisted.internet import threads
+
+from tests.bridget import BridgeT as BridgeTlegacy
+from tests.bridget import BridgeTAsset as BridgeTAsset
+from ooniprobe import ooni
+
+o = ooni()
+
+class BridgetArgs(usage.Options):
+    optParameters = [['asset', 'a', None, 'Asset file'],
+                     ['resume', 'r', 0, 'Resume at this index'],
+                     ['bridge', 'b', None, 'Specify a single bridge']]
+
+class BridgeT(TwistedTest):
+    implements(IPlugin, ITest)
+
+    shortName = "bridget"
+    description = "Bridget plugin"
+    requirements = None
+    options = BridgetArgs
+
+    def experiment(self):
+        bridget = BridgeTlegacy(o)
+        o.logger.info("Starting bridget test")
+        print "ASSET:%s " % self.asset
+        d = threads.deferToThread(bridget.connect, self.asset)
+        d.addCallback(self.d_experiment.callback, None)
+        return d
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+bridget = BridgeT(None, None)
diff --git a/plugins/dropin.cache b/plugins/dropin.cache
index 7bc3990..9258b18 100755
--- a/plugins/dropin.cache
+++ b/plugins/dropin.cache
@@ -45,4 +45,32 @@ p21
 S'skel'
 p22
 sg10
+NsbasbsS'bridget'
+p23
+g3
+(g4
+g5
+NtRp24
+(dp25
+g8
+S'plugins.bridget'
+p26
+sg10
+Nsg11
+(lp27
+g3
+(g13
+g5
+NtRp28
+(dp29
+g16
+(lp30
+g18
+ag19
+asg20
+g24
+sg21
+S'bridget'
+p31
+sg10
 Nsbasbs.
\ No newline at end of file
diff --git a/plugins/skel.py b/plugins/skel.py
index 0427239..93bc1bb 100644
--- a/plugins/skel.py
+++ b/plugins/skel.py
@@ -5,7 +5,8 @@ from plugoo.tests import ITest, TwistedTest
 
 class SkelArgs(usage.Options):
     optParameters = [['asset', 'a', None, 'Asset file'],
-                     ['resume', 'r', 0, 'Resume at this index']]
+                     ['resume', 'r', 0, 'Resume at this index'],
+                     ['other', 'o', None, 'Other arguments']]
 
 class SkelTest(TwistedTest):
     implements(IPlugin, ITest)
diff --git a/plugoo/tests.py b/plugoo/tests.py
index e74586c..39adfa2 100644
--- a/plugoo/tests.py
+++ b/plugoo/tests.py
@@ -81,7 +81,7 @@ class Test:
         if assets:
             self.logger.debug("Running through tests")
 
-            if extradata['index']:
+            if extradata and 'index' in extradata:
                 index = extradata['index']
             else:
                 index = None
@@ -168,11 +168,17 @@ class TwistedTest(object):
         self.asset = asset
         self.arguments = arguments
         self.start_time = datetime.now()
+        self._parse_arguments()
         #self.ooninet = ooninet
 
     def __repr__(self):
         return "<TwistedTest %s %s>" % (self.arguments, self.asset)
 
+    def _parse_arguments(self):
+        print self.arguments
+        if self.arguments and 'test' in self.arguments:
+            self.test = self.arguments['test']
+
     def finished(self, result):
         #self.ooninet.report(result)
         print "FINIHSED"
@@ -182,12 +188,30 @@ class TwistedTest(object):
         result['run_time'] = self.end_time - self.start_time
         return self.d.callback(result)
 
+    def _do_experiment(self):
+        self.d_experiment = defer.Deferred()
+        self.d_experiment.addCallback(self._do_control)
+        self.experiment()
+        return self.d
+
+    def _do_control(self, exp):
+        self.control(exp)
+        self.finished(dict())
+
+    def control(self, exp):
+        print "Doing control..."
+
+    def experiment(self):
+        print "Doing experiment"
+        self.d_experiment.callback(None)
+
     def startTest(self):
-        print "Starting test"
+        print "Starting test %s" % repr(self)
         self.d = defer.Deferred()
         result = {}
-        reactor.callLater(2.0, self.finished, result)
-        return self.d
+        #reactor.callLater(2.0, self.finished, result)
+        # Start experiment
+        return self._do_experiment()
 
 class TwistedTestFactory(object):
 
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/bridget.py b/tests/bridget.py
index da014fd..a613f61 100644
--- a/tests/bridget.py
+++ b/tests/bridget.py
@@ -241,13 +241,6 @@ Log info file %s
         except:
             self.logger.error("Error in starting Tor (do you have tor installed?)")
 
-        # XXX this only works on UNIX (do we care?)
-        # Make file reading non blocking
-        try:
-            fcntl.fcntl(p.stdout, fcntl.F_SETFL, os.O_NONBLOCK)
-        except:
-            self.logger.error("Unable to set file descriptor to non blocking")
-
         self.logger.info("Testing bridge: %s" % bridge)
         while True:
             o = ""
@@ -308,6 +301,7 @@ Log info file %s
                     self.logger.error("Error IOError: EAGAIN")
                     raise
                 sys.exc_clear()
+                print "In this exception 1"
 
             try:
                 # Set the timeout for the socket wait
@@ -376,3 +370,4 @@ def run(ooni, assets=None):
     bridget.print_failures()
     bridget.clean()
     ooni.logger.info("Testing completed!")
+





More information about the tor-commits mailing list