[tor-commits] [ooni-probe/master] Fixed imports.
isis at torproject.org
isis at torproject.org
Fri Sep 14 02:22:56 UTC 2012
commit cf5e13c241389f8192a587c0601a8cf561329ae7
Author: Isis Lovecruft <isis at torproject.org>
Date: Fri Sep 14 02:20:52 2012 +0000
Fixed imports.
---
ooni/lib/Makefile | 30 ++
ooni/lib/__init__.py | 43 +---
ooni/lib/txscapy | 1 -
ooni/lib/txscapy.py | 348 ++++++++++++++++++++
ooni/lib/txtraceroute | 1 -
ooni/lib/txtraceroute.py | 752 ++++++++++++++++++++++++++++++++++++++++++++
ooni/ooniprobe.py | 10 +-
ooni/plugins/blocking.py | 2 +-
ooni/plugins/dnstamper.py | 6 +-
ooni/plugins/httphost.py | 2 +-
ooni/plugins/tcpconnect.py | 5 +-
ooni/plugoo/nodes.py | 6 +-
ooni/plugoo/reports.py | 2 +-
ooni/plugoo/tests.py | 3 +-
ooni/utils/log.py | 1 -
15 files changed, 1152 insertions(+), 60 deletions(-)
diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile
new file mode 100644
index 0000000..3b0c922
--- /dev/null
+++ b/ooni/lib/Makefile
@@ -0,0 +1,30 @@
+all: txtorcon txtraceroute
+
+txtraceroute:
+ echo "Processing dependency txtraceroute..."
+ git clone https://github.com/hellais/txtraceroute.git txtraceroute.git
+ mv txtraceroute.git/txtraceroute.py txtraceroute.py
+ rm -rf txtraceroute.git
+
+txtorcon:
+ echo "Processing dependency txtorcon..."
+ git clone https://github.com/meejah/txtorcon.git txtorcon.git
+ mv txtorcon.git/txtorcon txtorcon
+ rm -rf txtorcon.git
+
+clean:
+ rm -rf txtorcon
+ rm -rf txtraceroute.py
+
+#txscapy:
+# echo "Processing dependency txscapy"
+# git clone https://github.com/hellais/txscapy.git txscapy.git
+# mv txscapy.git/txscapy.py txscapy.py
+# rm -rf txscapy.git
+
+#rfc3339:
+# echo "Processing RFC3339 dependency"
+# hg clone https://bitbucket.org/henry/rfc3339 rfc3339
+# mv rfc3339/rfc3339.py rfc3339.py
+# rm -rf rfc3339
+
diff --git a/ooni/lib/__init__.py b/ooni/lib/__init__.py
index 0fd36c5..611d50c 100644
--- a/ooni/lib/__init__.py
+++ b/ooni/lib/__init__.py
@@ -1,40 +1,5 @@
-import pkgutil
-import sys
-from os import listdir, path
+from sys import path as syspath
+from os import path as ospath
-__all__ = ['txtorcon', 'txscapy', 'txtraceroute']
-
-__sub_modules__ = [ ]
-
-def callback(arg, directory, files):
- for file in listdir(directory):
- fullpath = path.abspath(file)
- if path.isdir(fullpath) and not path.islink(fullpath):
- __sub_modules__.append(fullpath)
- sys.path.append(fullpath)
-
-path.walk(".", callback, None)
-
-def load_submodules(init, list):
- for subdir in list:
- contents=[x for x in pkgutil.iter_modules(path=subdir,
- prefix='ooni.lib.')]
- for loader, module_name, ispkg in contents:
- init_dot_module = init + "." + module_name
- if init_dot_module in sys.modules:
- module = sys.modules[module_name]
- else:
- if module_name in __all__:
- grep = loader.find_module(module_name)
- module = grep.load_module(module_name)
- else:
- module = None
-
- if module is not None:
- globals()[module_name] = module
-
-load_submodules(__name__, __sub_modules__)
-
-print "system paths are: %s" % sys.path
-print "globals are: %s" % globals()
-print "system modules are: %s" % sys.modules
+pwd = ospath.dirname(__file__)
+syspath.append(pwd)
diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy
deleted file mode 160000
index 19fb281..0000000
--- a/ooni/lib/txscapy
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf
diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py
new file mode 100644
index 0000000..4d83dce
--- /dev/null
+++ b/ooni/lib/txscapy.py
@@ -0,0 +1,348 @@
+# -*- coding:utf8 -*-
+"""
+ txscapy
+ ******
+ (c) 2012 Arturo Filastò
+ a twisted wrapper for scapys send and receive functions.
+
+ This software has been written to be part of OONI, the Open Observatory of
+ Network Interference. More information on that here: http://ooni.nu/
+
+"""
+
+import struct
+import socket
+import os
+import sys
+import time
+
+from twisted.internet import protocol, base, fdesc, error, defer
+from twisted.internet import reactor, threads
+from twisted.python import log
+from zope.interface import implements
+
+from scapy.all import Gen
+from scapy.all import SetGen
+
+LINUX=sys.platform.startswith("linux")
+OPENBSD=sys.platform.startswith("openbsd")
+FREEBSD=sys.platform.startswith("freebsd")
+NETBSD=sys.platform.startswith("netbsd")
+DARWIN=sys.platform.startswith("darwin")
+SOLARIS=sys.platform.startswith("sunos")
+WINDOWS=sys.platform.startswith("win32")
+
+from scapy.all import RawPcapWriter, MTU, BasePacketList, conf
+class PcapWriter(RawPcapWriter):
+ def __init__(self, filename, linktype=None, gz=False, endianness="",
+ append=False, sync=False):
+ RawPcapWriter.__init__(self, filename, linktype=None, gz=False,
+ endianness="", append=False, sync=False)
+ fdesc.setNonBlocking(self.f)
+
+ def _write_header(self, pkt):
+ if self.linktype == None:
+ if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList):
+ pkt = pkt[0]
+ try:
+ self.linktype = conf.l2types[pkt.__class__]
+ except KeyError:
+ self.linktype = 1
+ RawPcapWriter._write_header(self, pkt)
+
+ def _write_packet(self, packet):
+ sec = int(packet.time)
+ usec = int(round((packet.time-sec)*1000000))
+ s = str(packet)
+ caplen = len(s)
+ RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen)
+
+class ScapySocket(object):
+ MTU = 1500
+ def __init__(self, filter=None, iface=None, nofilter=None):
+ from scapy.all import conf
+ self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter)
+
+ def fileno(self):
+ return self.ssocket.ins.fileno()
+
+ def send(self, data):
+ return self.ssocket.send(data)
+
+ def recv(self):
+ if FREEBSD or DARWIN:
+ return self.ssocket.nonblock_recv()
+ else:
+ return self.ssocket.recv(self.MTU)
+
+class Scapy(object):
+ """
+ A twisted based wrapper for scapy send and receive functionality.
+
+ It sends packets inside of a threadpool and receives packets using the
+ libdnet receive non blocking file descriptor.
+ """
+ min = 2
+ max = 6
+ debug = True
+ write_only_answers = False
+ pcapwriter = None
+ recv = False
+
+ def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None,
+ iface=None, nofilter=None, pcapfile=None):
+
+ if self.debug:
+ log.startLogging(sys.stdout)
+
+ self.maxPacketSize = maxPacketSize
+ if not reactor:
+ from twisted.internet import reactor
+
+ self._reactor = reactor
+
+ if pkts:
+ self._buildPacketQueues(pkts)
+ self._buildSocket()
+
+ self.cthreads = 0
+ self.mthreads = 80
+
+ self.running = False
+ self.done = False
+ self.finished = False
+
+ import thread
+ from twisted.python import threadpool
+ self.threadID = thread.get_ident
+ self.threadpool = threadpool.ThreadPool(self.min, self.max)
+ self.startID = self._reactor.callWhenRunning(self._start)
+
+ self.deferred = defer.Deferred()
+
+ if pcapfile:
+ self.pcapwriter = PcapWriter(pcapfile)
+
+ def _buildSocket(self, filter=None, iface=None, nofilter=None):
+ self.socket = ScapySocket(filter, iface, nofilter)
+ if self.recv:
+ self._reactor.addReader(self)
+
+ def _buildPacketQueues(self, pkts):
+ """
+ Converts the list of packets to a Scapy generator and sets up all the
+ necessary attributes for understanding if all the needed responses have
+ been received.
+ """
+ if not isinstance(pkts, Gen):
+ self.pkts = SetGen(pkts)
+
+ self.outqueue = [p for p in pkts]
+
+ self.total_count = len(self.outqueue)
+ self.answer_count = 0
+ self.out_count = 0
+
+ self.hsent = {}
+ for p in self.outqueue:
+ h = p.hashret()
+ if h in self.hsent:
+ self.hsent[h].append(p)
+ else:
+ self.hsent[h] = [p]
+
+
+ def gotAnswer(self, answer, question):
+ """
+ Got a packet that has been identified as an answer to one of the sent
+ out packets.
+
+ If the answer count matches the sent count the finish callback is
+ fired.
+
+ @param answer: the packet received on the wire.
+
+ @param question: the sent packet that matches that response.
+
+ """
+
+ if self.pcapwriter and self.write_only_answers:
+ self.pcapwriter.write(question)
+ self.pcapwriter.write(answer)
+ self.answer_count += 1
+ if self.answer_count >= self.total_count:
+ print "Got all the answers I need"
+ self.deferred.callback(None)
+
+ def processAnswer(self, pkt, hlst):
+ """
+ Checks if the potential answer is in fact an answer to one of the
+ matched sent packets. Uses the scapy .answers() function to verify
+ this.
+
+ @param pkt: The packet to be tested if is the answer to a sent packet.
+
+ @param hlst: a list of packets that match the hash for an answer to
+ pkt.
+ """
+ for i in range(len(hlst)):
+ if pkt.answers(hlst[i]):
+ self.gotAnswer(pkt, hlst[i])
+
+ def fileno(self):
+ """
+ Returns a fileno for use by twisteds Reader.
+ """
+ return self.socket.fileno()
+
+ def processPacket(self, pkt):
+ """
+ Override this method to process your packets.
+
+ @param pkt: the packet that has been received.
+ """
+ #pkt.show()
+
+
+ def doRead(self):
+ """
+ There is something to be read on the wire. Do all the processing on the
+ received packet.
+ """
+ pkt = self.socket.recv()
+ if self.pcapwriter and not self.write_only_answers:
+ self.pcapwriter.write(pkt)
+ self.processPacket(pkt)
+
+ h = pkt.hashret()
+ if h in self.hsent:
+ hlst = self.hsent[h]
+ self.processAnswer(pkt, hlst)
+
+ def logPrefix(self):
+ """
+ The prefix to be prepended in logging.
+ """
+ return "txScapy"
+
+ def _start(self):
+ """
+ Start the twisted thread pool.
+ """
+ self.startID = None
+ return self.start()
+
+ def start(self):
+ """
+ Actually start the thread pool.
+ """
+ if not self.running:
+ self.threadpool.start()
+ self.shutdownID = self._reactor.addSystemEventTrigger(
+ 'during', 'shutdown', self.finalClose)
+ self.running = True
+
+ def sendPkt(self, pkt):
+ """
+ Send a packet to the wire.
+
+ @param pkt: The packet to be sent.
+ """
+ self.socket.send(pkt)
+
+ def sr(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw):
+ """
+ Wraps the scapy sr function.
+
+ @param nofilter: put 1 to avoid use of bpf filters
+
+ @param retry: if positive, how many times to resend unanswered packets
+ if negative, how many times to retry when no more packets are
+ answered (XXX to be implemented)
+
+ @param timeout: how much time to wait after the last packet has
+ been sent (XXX to be implemented)
+
+ @param multi: whether to accept multiple answers for the same
+ stimulus (XXX to be implemented)
+
+ @param filter: provide a BPF filter
+ @param iface: listen answers only on the given interface
+ """
+ self.recv = True
+ self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
+
+ def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw):
+ """
+ Wraps the scapy send function. Its the same as send and receive, except
+ it does not receive. Who would have ever guessed? ;)
+
+ @param nofilter: put 1 to avoid use of bpf filters
+
+ @param retry: if positive, how many times to resend unanswered packets
+ if negative, how many times to retry when no more packets are
+ answered (XXX to be implemented)
+
+ @param timeout: how much time to wait after the last packet has
+ been sent (XXX to be implemented)
+
+ @param multi: whether to accept multiple answers for the same
+ stimulus (XXX to be implemented)
+
+ @param filter: provide a BPF filter
+ @param iface: listen answers only on the given interface
+ """
+ self.recv = False
+ self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
+
+ def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
+ self._buildSocket(filter, iface, nofilter)
+ self._buildPacketQueues(pkts)
+ def sent(cb):
+ if self.cthreads < self.mthreads and not self.done:
+ pkt = None
+ try:
+ pkt = self.outqueue.pop()
+ except:
+ self.done = True
+ if not self.recv:
+ self.deferred.callback(None)
+ return
+ d = threads.deferToThreadPool(reactor, self.threadpool,
+ self.sendPkt, pkt)
+ d.addCallback(sent)
+ return d
+
+ for x in range(self.mthreads):
+ try:
+ pkt = self.outqueue.pop()
+ except:
+ self.done = True
+ return
+ if self.cthreads >= self.mthreads and self.done:
+ return
+ d = threads.deferToThreadPool(reactor, self.threadpool,
+ self.sendPkt, pkt)
+ d.addCallback(sent)
+ return d
+
+ def connectionLost(self, why):
+ pass
+
+ def finalClose(self):
+ """
+ Clean all the thread related stuff up.
+ """
+ self.shutdownID = None
+ self.threadpool.stop()
+ self.running = False
+
+def txsr(*args, **kw):
+ tr = Scapy(*args, **kw)
+ tr.sr(*args, **kw)
+ return tr.deferred
+
+def txsend(*arg, **kw):
+ tr = Scapy(*arg, **kw)
+ tr.send(*arg, **kw)
+ return tr.deferred
diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute
deleted file mode 160000
index 067a260..0000000
--- a/ooni/lib/txtraceroute
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 067a2609390e77bf9187275638cf8786efef7e13
diff --git a/ooni/lib/txtraceroute.py b/ooni/lib/txtraceroute.py
new file mode 100644
index 0000000..8182b34
--- /dev/null
+++ b/ooni/lib/txtraceroute.py
@@ -0,0 +1,752 @@
+#!/usr/bin/env python
+# coding: utf-8
+#
+# Copyright (c) 2012 Alexandre Fiori
+# Arturo Filastò
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import json
+import operator
+import os
+import socket
+import struct
+import sys
+import time
+import random
+import itertools
+from pprint import pprint
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.internet import threads
+from twisted.python import usage
+from twisted.web.client import getPage
+
+class iphdr(object):
+ """
+ This represents an IP packet header.
+
+ XXX enable IP_TIMESTAMP in setsockopt
+ to get the timestamp of when the router says it has gotten an ICMP
+ timeout.
+
+ @assemble packages the packet
+ @disassemble disassembles the packet
+ """
+ def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None):
+ self.version = 4
+ self.hlen = 5
+ self.tos = 0
+ self.length = 20
+ self.id = os.getpid()
+ self.frag = 0
+ self.ttl = 255
+ self.proto = proto
+ self.cksum = 0
+ self.src = src
+ self.saddr = socket.inet_aton(src)
+ self.dst = dst or "0.0.0.0"
+ self.daddr = socket.inet_aton(self.dst)
+ self.data = ""
+
+ def assemble(self):
+ header = struct.pack('BBHHHBB',
+ (self.version & 0x0f) << 4 | (self.hlen & 0x0f),
+ self.tos, self.length + len(self.data),
+ socket.htons(self.id), self.frag,
+ self.ttl, self.proto)
+ self._raw = header + "\x00\x00" + self.saddr + self.daddr + self.data
+ return self._raw
+
+ @classmethod
+ def disassemble(self, data):
+ self._raw = data
+ ip = iphdr()
+ pkt = struct.unpack('!BBHHHBBH', data[:12])
+ ip.version = (pkt[0] >> 4 & 0x0f)
+ ip.hlen = (pkt[0] & 0x0f)
+ ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:]
+ ip.saddr = data[12:16]
+ ip.daddr = data[16:20]
+ ip.src = socket.inet_ntoa(ip.saddr)
+ ip.dst = socket.inet_ntoa(ip.daddr)
+ return ip
+
+ def __repr__(self):
+ return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \
+ "%s -> %s" % \
+ (self.tos, self.ttl, self.id, self.frag, self.proto,
+ self.length, self.src, self.dst)
+
+class tcphdr(object):
+ def __init__(self, data="", sport=4242, dport=4242):
+ self.seq = 0
+ self.hlen = 44
+ self.flags = 2
+ self.wsize = 200
+ self.cksum = 123
+ self.options = 0
+ self.mss = 1460
+ self.dport = dport
+ self.sport = sport
+
+ def assemble(self):
+ header = struct.pack("!HHL", self.sport, self.dport, self.seq)
+ header += '\x00\x00\x00\x00'
+ header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags &
+ 0xff), self.wsize, self.cksum)
+ header += "\x00\x00"
+ options = '\x02\x04\x05\xb4\x01\x03\x03\x01\x01\x01\x08\x0a'
+ options += '\x4d\xcf\x52\x33\x00\x00\x00\x00\x04\x02\x00\x00'
+ # XXX There is something wrong here fixme
+ # options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1)
+ # options += struct.pack("!BBL", 8, 10, 1209452188)
+ # options += '\00'*4
+ # options += struct.pack("!BB", 4, 2)
+ # options += '\00'
+ self._raw = header+options
+ return self._raw
+
+ @classmethod
+ def checksum(self, data):
+ pass
+
+ def __repr__(self):
+ return "<TCPPacket (sport: %s dport: %s seq: %s) " %\
+ (self.sport, self.dport, self.seq)
+
+ @classmethod
+ def disassemble(self, data):
+ self._raw = data
+ tcp = tcphdr()
+ pkt = struct.unpack("!HHL", data[:8])
+ tcp.sport, tcp.dport, tcp.seq = pkt
+ if len(data) > 10:
+ pkt = struct.unpack("!H", data[8:10])
+ tcp.hlen = (pkt[0] >> 10 ) & 0xff
+ tcp.flags = pkt[0] & 0xff
+ tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:24])
+ return tcp
+
+class udphdr(object):
+ def __init__(self, data="", sport=4242, dport=4242):
+ self.dport = dport
+ self.sport = sport
+ self.cksum = 0
+ self.length = 0
+ self.data = data
+
+ def assemble(self):
+ self.length = len(self.data) + 8
+ part1 = struct.pack("!HHH", self.sport, self.dport, self.length)
+ cksum = self.checksum(self.data)
+ cksum = struct.pack("!H", cksum)
+
+ self._raw = part1 + cksum + self.data
+ return self._raw
+
+ @classmethod
+ def checksum(self, data):
+ # XXX implement proper checksum
+ cksum = 0
+ return cksum
+
+ def __repr__(self):
+ return "<UDPPacket (sport %s, dport %s, length %s, data %s)>" % \
+ (self.sport, self.dport, self.length, self.data)
+
+ @classmethod
+ def disassemble(self, data):
+ self._raw = data
+ udp = udphdr()
+ pkt = struct.unpack("!HHHH", data[:8])
+ udp.sport, udp.dport, udp.length, udp.cksum = pkt
+ udp.data = data[8:]
+ return udp
+
+class icmphdr(object):
+ def __init__(self, data=""):
+ self.type = 8
+ self.code = 0
+ self.cksum = 0
+ self.id = os.getpid()
+ self.sequence = 0
+ self.data = data
+
+ def assemble(self):
+ part1 = struct.pack("BB", self.type, self.code)
+ part2 = struct.pack("!HH", self.id, self.sequence)
+ cksum = self.checksum(part1 + "\x00\x00" + part2 + self.data)
+ cksum = struct.pack("!H", cksum)
+ self._raw = part1 + cksum + part2 + self.data
+ return self._raw
+
+ @classmethod
+ def checksum(self, data):
+ if len(data) & 1:
+ data += "\x00"
+ cksum = reduce(operator.add,
+ struct.unpack('!%dH' % (len(data) >> 1), data))
+ cksum = (cksum >> 16) + (cksum & 0xffff)
+ cksum += (cksum >> 16)
+ cksum = (cksum & 0xffff) ^ 0xffff
+ return cksum
+
+ @classmethod
+ def disassemble(self, data):
+ self._raw = data
+ icmp = icmphdr()
+ pkt = struct.unpack("!BBHHH", data)
+ icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt
+ return icmp
+
+ def __repr__(self):
+ return "ICMP (type %s, code %s, id %s, sequence %s)" % \
+ (self.type, self.code, self.id, self.sequence)
+
+
+def pprintp(packet):
+ """
+ Used to pretty print packets.
+ """
+ lines = []
+ line = []
+ for i, byte in enumerate(packet):
+ line.append(("%.2x" % ord(byte), byte))
+ if (i + 1) % 8 == 0:
+ lines.append(line)
+ line = []
+
+ lines.append(line)
+
+ for row in lines:
+ left = ""
+ right = " " * (8 - len(row))
+ for y in row:
+ left += "%s " % y[0]
+ right += "%s" % y[1]
+
+ print left + " " + right
+
+ at defer.inlineCallbacks
+def geoip_lookup(ip):
+ try:
+ r = yield getPage("http://freegeoip.net/json/%s" % ip)
+ d = json.loads(r)
+ items = [d["country_name"], d["region_name"], d["city"]]
+ text = ", ".join([s for s in items if s])
+ defer.returnValue(text.encode("utf-8"))
+ except Exception:
+ defer.returnValue("Unknown location")
+
+
+ at defer.inlineCallbacks
+def reverse_lookup(ip):
+ try:
+ r = yield threads.deferToThread(socket.gethostbyaddr, ip)
+ defer.returnValue(r[0])
+ except Exception:
+ defer.returnValue(None)
+
+
+class Hop(object):
+ def __init__(self, target, ttl, proto, sport=None, dport=None):
+ self.proto = proto
+ self.dport = dport
+ self.sport = sport
+
+ self.found = False
+ self.tries = 0
+ self.last_try = 0
+ self.remote_ip = None
+ self.remote_icmp = None
+ self.remote_host = None
+ self.location = ""
+
+ self.ttl = ttl
+ self.ip = iphdr(dst=target)
+ self.ip.ttl = ttl
+ self.ip.id += ttl
+ if self.proto == "icmp":
+ self.icmp = icmphdr('\x00'*20)
+ self.icmp.id = self.ip.id
+ self.ip.data = self.icmp.assemble()
+ elif self.proto == "udp":
+ self.udp = udphdr('\x00'*20, self.sport, self.dport)
+ self.ip.data = self.udp.assemble()
+ self.ip.proto = socket.IPPROTO_UDP
+ else:
+ self.tcp = tcphdr('\x42'*20, self.sport, self.dport)
+ self.ip.data = self.tcp.assemble()
+ self.ip.proto = socket.IPPROTO_TCP
+
+ self._pkt = self.ip.assemble()
+
+ @property
+ def pkt(self):
+ self.tries += 1
+ self.last_try = time.time()
+ return self._pkt
+
+ def get(self):
+ if self.found:
+ if self.remote_host:
+ ip = self.remote_host
+ else:
+ ip = self.remote_ip.src
+ ping = self.found - self.last_try
+ else:
+ ip = None
+ ping = None
+
+ location = self.location if self.location else None
+ return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location,
+ 'proto': self.proto, 'dport': self.dport, 'sport': self.sport}
+
+ def __repr__(self):
+ if self.found:
+ if self.remote_host:
+ ip = ":: %s" % self.remote_host
+ else:
+ ip = ":: %s" % self.remote_ip.src
+ ping = "%0.3fs" % (self.found - self.last_try)
+ else:
+ ip = "??"
+ ping = "-"
+
+ location = ":: %s" % self.location if self.location else ""
+ return "%02d. %s %s %s (%s, sport: %s dport: %s)" % (self.ttl, ping, ip, location, self.proto, self.sport, self.dport)
+
+class TracerouteResult(object):
+ """
+ Used to store the results of a Traceroute.
+ """
+ #src_ports = [0, 9090]
+ #dst_ports = [0, 21, 123, 80, 443]
+ src_ports = [0, 80]
+ dst_ports = [0, 80]
+ hops = []
+ done = False
+
+ def __init__(self, protocol):
+ self.protocol = protocol
+ self.probes = {}
+
+ if protocol == "icmp":
+ self.current = None
+ else:
+ self.current = {}
+ for src, dst in itertools.product(self.src_ports,
+ self.dst_ports):
+ if src not in self.probes:
+ self.probes[src] = {}
+ self.probes[src][dst] = []
+
+ if src not in self.current:
+ self.current[src] = {}
+ self.current[src][dst] = None
+
+ def get_current_probes(self):
+ if self.protocol == "icmp":
+ return self.current
+
+ def add_to_current_probes(self, probe):
+ if self.protocol == "icmp":
+ self.current = probe
+ else:
+ self.current[probe.sport][probe.dport] = probe
+
+ def is_in_progress(self):
+ if self.protocol == "icmp":
+ progress = self.current
+ else:
+ progress = None
+ for x in self.current:
+ for y in self.current[x]:
+ if self.current[x][y] != None:
+ progress = True
+ if progress is None:
+ return False
+ else:
+ return True
+
+ def get(self, src=None, dst=None):
+ if self.protocol == "icmp":
+ return self.probes
+ else:
+ return self.probes[src][dst]
+
+ def append(self, probe, src=None, dst=None):
+ if self.protocol == "icmp":
+ self.probes.append(hop)
+ else:
+ self.probes[src][dst].append(probe)
+
+ def pop(self, src=None, dst=None):
+ if self.protocol == "icmp":
+ hop = self.current
+ self.current = None
+ return hop
+
+ elif (dst != None) and (src != None):
+ hop = self.current[src][dst]
+ self.current[src][dst] = None
+ return hop
+
+ else:
+ raise Exception("Did not specify dst and src ports")
+
+ @classmethod
+ def hops(self, target, ttl):
+ """
+ Generates a set of ooni probes for traceroute based network tampering
+ detection.
+
+ We send in one round a set of packets with same TTL but on all protocols
+ and with all possible source and destination ports.
+ """
+ hops = []
+ for src, dst in itertools.product(self.src_ports, self.dst_ports):
+ hops.append(Hop(target, ttl,
+ "tcp", src, dst))
+ hops.append(Hop(target, ttl,
+ "udp", src, dst))
+ hops.append(Hop(target, ttl, "icmp", 0, 0))
+ return hops
+
+class TracerouteProtocol(object):
+ def __init__(self, target, **settings):
+
+ self.target = target
+ self.settings = settings
+ self.verbose = settings.get("verbose")
+ self.proto = settings.get("proto")
+ self.rfd = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ self.sfd = {}
+
+ # Create the data structures to contain the test results
+ self.traceroute = {}
+ self.traceroute["tcp"] = TracerouteResult("tcp")
+ self.traceroute["udp"] = TracerouteResult("udp")
+ self.traceroute["icmp"] = TracerouteResult("icmp")
+
+ if self.settings.get("ooni"):
+ self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_TCP)
+ self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_UDP)
+ elif self.proto == "icmp":
+ self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_ICMP)
+ elif self.proto == "udp":
+ self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_UDP)
+ elif self.proto == "tcp":
+ self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+ socket.IPPROTO_TCP)
+
+ # Let me add IP Headers myself, just give me a socket!
+ self.rfd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
+ for fd in self.sfd:
+ self.sfd[fd].setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
+
+ self.out_queue = []
+ self.waiting = True
+ self.deferred = defer.Deferred()
+
+ reactor.addReader(self)
+ reactor.addWriter(self)
+
+ # send 1st probe packet(s)
+ if self.settings.get("ooni"):
+ hops = list(TracerouteResult.hops(self.target, 1))
+ else:
+ hops = [Hop(self.target, 1,
+ settings.get("proto"),
+ self.settings.get("sport"),
+ self.settings.get("dport"))]
+ for hop in hops:
+ # Store the to be completed items inside of a dictionary
+ self.traceroute[hop.proto].add_to_current_probes(hop)
+ self.out_queue.append(hop)
+
+ def logPrefix(self):
+ return "TracerouteProtocol(%s)" % self.target
+
+ def fileno(self):
+ return self.rfd.fileno()
+
+ @defer.inlineCallbacks
+ def hopFound(self, hop, ip, icmp, ref, subref):
+ hop.remote_ip = ip
+ hop.remote_icmp = icmp
+
+ if (ip and icmp):
+ hop.found = time.time()
+ if self.settings.get("geoip_lookup") is True:
+ hop.location = yield geoip_lookup(ip.src)
+
+ if self.settings.get("reverse_lookup") is True:
+ hop.remote_host = yield reverse_lookup(ip.src)
+
+ ttl = hop.ttl + 1
+
+ if (ttl > (self.settings.get("max_hops", 30) + 1)):
+ done = True
+ else:
+ done = False
+
+ if not done:
+ cb = self.settings.get("hop_callback")
+ if callable(cb):
+ yield defer.maybeDeferred(cb, hop)
+
+ if not self.waiting:
+ if self.deferred:
+ self.deferred.callback(self.hops)
+ self.deferred = None
+ else:
+ hops = []
+ if self.settings.get("ooni"):
+ if not (self.traceroute["icmp"].is_in_progress() or
+ self.traceroute["tcp"].is_in_progress() or
+ self.traceroute["udp"].is_in_progress()):
+ # Add hops only if we are not in progress
+ hops = list(TracerouteResult.hops(self.target, ttl))
+ else:
+ hops = [Hop(self.target, ttl,
+ settings.get("proto"),
+ self.settings.get("sport"),
+ self.settings.get("dport"))]
+
+ for hop in hops:
+ # Store the to be completed items inside of a dictionary
+ self.traceroute[hop.proto].add_to_current_probes(hop)
+ self.out_queue.append(hop)
+
+ def doRead(self):
+ if not self.waiting:
+ return
+
+ pkt = self.rfd.recv(4096)
+ # disassemble ip header
+ ip = iphdr.disassemble(pkt[:20])
+
+ if self.verbose:
+ print "Got this packet:"
+ print "src %s" % ip.src
+ pprintp(pkt)
+
+ # Not interested in non ICMP packets.
+ if ip.proto != socket.IPPROTO_ICMP:
+ return
+
+ found = False
+ foundHop = None
+
+ # disassemble icmp header
+ icmp = icmphdr.disassemble(pkt[20:28])
+
+ if self.verbose:
+ print icmp
+
+ # If it's an ICMP Echo reply then our ICMP probe has hit destination
+ if icmp.type == 0 and icmp.id == self.current_hop["icmp"][1].icmp.id:
+ foundHop = self.traceroute["icmp"].pop()
+ found = True
+
+ elif icmp.type == 11:
+ # disassemble referenced ip header
+ ref = iphdr.disassemble(pkt[28:48])
+ subref = None
+
+ if self.verbose:
+ print ref
+
+ if ref.dst == self.target:
+ found = True
+
+ if ref.proto == socket.IPPROTO_UDP:
+ subref = udphdr.disassemble(pkt[48:])
+ proto = "udp"
+
+ elif ref.proto == socket.IPPROTO_TCP:
+ subref = tcphdr.disassemble(pkt[48:])
+ proto = "tcp"
+
+ else:
+ proto = "icmp"
+
+ if subref:
+ sport = subref.sport
+ dport = subref.dport
+ else:
+ sport = None
+ dport = None
+ # Remove completed hops
+ foundHop = self.traceroute[proto].pop(sport,
+ dport)
+
+ if ip.src == self.target:
+ self.waiting = False
+
+ if found:
+ self.hopFound(foundHop, ip, icmp, ref, subref)
+ elif foundHop:
+ self.hopFound(foundHop, ip, icmp, ref, subref)
+
+ def hopTimeout(self, hop):
+ if not hop.found:
+ if hop.tries < self.settings.get("max_tries", 3):
+ # retry
+ hop.tries += 1
+ self.out_queue.append(hop)
+ else:
+ # give up and move forward
+ self.traceroute[hop.proto].pop(hop.dport,
+ hop.sport)
+ self.hopFound(hop, None, None, None, None)
+
+ def doWrite(self):
+ if self.waiting and self.out_queue:
+ hop = self.out_queue.pop(0)
+ pkt = hop.pkt
+ if self.verbose:
+ print "Sending this packet:"
+ pprintp(pkt)
+ print hop
+
+ self.sfd[hop.proto].sendto(pkt, (hop.ip.dst, 0))
+
+ self.traceroute[hop.proto].add_to_current_probes(hop)
+
+ timeout = self.settings.get("timeout", 1)
+ reactor.callLater(timeout, self.hopTimeout, hop)
+
+ def connectionLost(self, why):
+ pass
+
+
+def traceroute(target, **settings):
+ tr = TracerouteProtocol(target, **settings)
+ return tr.deferred
+
+
+ at defer.inlineCallbacks
+def start_trace(target, **settings):
+ hops = yield traceroute(target, **settings)
+ if settings["hop_callback"] is None:
+ for hop in hops:
+ print hop
+ reactor.stop()
+
+class Options(usage.Options):
+ optFlags = [
+ ["quiet", "q", "Only print results at the end."],
+ ["no-dns", "n", "Show numeric IPs only, not their host names."],
+ ["no-geoip", "g", "Do not collect and show GeoIP information"],
+ ["verbose", "v", "Be more verbose"],
+ ["ooni", "o", "Run the ooni common port multiprotocol traceroute"],
+ ["help", "h", "Show this help"],
+ ]
+ optParameters = [
+ ["timeout", "t", 2, "Timeout for probe packets"],
+ ["tries", "r", 3, "How many tries before give up probing a hop"],
+ ["proto", "p", "icmp", "What protocol to use (tcp, udp, icmp)"],
+ ["dport", "d", random.randint(2**10, 2**16), "Destination port (TCP and UDP only)"],
+ ["sport", "s", random.randint(2**10, 2**16), "Source port (TCP and UDP only)"],
+ ["max_hops", "m", 30, "Max number of hops to probe"]
+ ]
+
+def main():
+ def show(hop):
+ print hop
+
+ defaults = dict(hop_callback=show,
+ reverse_lookup=True,
+ geoip_lookup=True,
+ timeout=2,
+ proto="icmp",
+ dport=None,
+ sport=None,
+ verbose=False,
+ ooni=False,
+ max_tries=3,
+ max_hops=30)
+
+ if len(sys.argv) < 2:
+ print("Usage: %s [options] host" % (sys.argv[0]))
+ print("%s: Try --help for usage details." % (sys.argv[0]))
+ sys.exit(1)
+
+ target = sys.argv.pop(-1) if sys.argv[-1][0] != "-" else ""
+ config = Options()
+ try:
+ config.parseOptions()
+ if not target:
+ raise
+ except usage.UsageError, e:
+ print("%s: %s" % (sys.argv[0], e))
+ print("%s: Try --help for usage details." % (sys.argv[0]))
+ sys.exit(1)
+
+ settings = defaults.copy()
+ if config.get("silent"):
+ settings["hop_callback"] = None
+ if config.get("no-dns"):
+ settings["reverse_lookup"] = False
+ if config.get("no-geoip"):
+ settings["geoip_lookup"] = False
+ if config.get("verbose"):
+ settings["verbose"] = True
+ if config.get("ooni"):
+ settings["ooni"] = True
+ if "timeout" in config:
+ settings["timeout"] = config["timeout"]
+ if "tries" in config:
+ settings["max_tries"] = config["tries"]
+ if "proto" in config:
+ settings["proto"] = config["proto"]
+ if "max_hops" in config:
+ settings["max_hops"] = config["max_hops"]
+ if "dport" in config:
+ settings["dport"] = int(config["dport"])
+ if "sport" in config:
+ settings["sport"] = int(config["sport"])
+
+ if os.getuid() != 0:
+ print("traceroute needs root privileges for the raw socket")
+ sys.exit(1)
+ try:
+ target = socket.gethostbyname(target)
+ except Exception, e:
+ print("could not resolve '%s': %s" % (target, str(e)))
+ sys.exit(1)
+
+ reactor.callWhenRunning(start_trace, target, **settings)
+ reactor.run()
+
+if __name__ == "__main__":
+ main()
+
diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py
index 95b5a18..539c2ac 100755
--- a/ooni/ooniprobe.py
+++ b/ooni/ooniprobe.py
@@ -27,9 +27,11 @@ from zope.interface.verify import verifyObject
from zope.interface.exceptions import BrokenImplementation
from zope.interface.exceptions import BrokenMethodImplementation
-from ooni.plugoo import tests, work, assets, reports
-from ooni.logo import getlogo
-from ooni import plugins, log
+from plugoo import tests, work, assets, reports
+from plugoo.interface import ITest
+from utils.logo import getlogo
+from utils import log
+import plugins
__version__ = "0.0.1-prealpha"
@@ -38,7 +40,7 @@ def retrieve_plugoo():
Get all the plugins that implement the ITest interface and get the data
associated to them into a dict.
"""
- interface = tests.ITest
+ interface = ITest
d = {}
error = False
for p in getPlugins(interface, plugins):
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
index 72fd49f..f3c20e1 100644
--- a/ooni/plugins/blocking.py
+++ b/ooni/plugins/blocking.py
@@ -3,7 +3,7 @@ from twisted.python import usage
from twisted.plugin import IPlugin
from plugoo.assets import Asset
-from plugoo.tests import ITest, TwistedTest
+from plugoo.tests import ITest, OONITest
class BlockingArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
index 54cfbdf..3d373ce 100644
--- a/ooni/plugins/dnstamper.py
+++ b/ooni/plugins/dnstamper.py
@@ -38,9 +38,9 @@ from twisted.python import usage
from twisted.plugin import IPlugin
from zope.interface import implements
-from ooni.plugoo.assets import Asset
-from ooni.plugoo.tests import ITest, OONITest
-from ooni import log
+from plugoo.assets import Asset
+from plugoo.tests import ITest, OONITest
+from utils import log
class AlexaAsset(Asset):
"""
diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py
index e8808bd..7c783a1 100644
--- a/ooni/plugins/httphost.py
+++ b/ooni/plugins/httphost.py
@@ -21,7 +21,7 @@ from twisted.python import usage
from twisted.plugin import IPlugin
from plugoo.assets import Asset
-from plugoo.tests import ITest, TwistedTest
+from plugoo.tests import ITest, OONITest
class HTTPHostArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py
index db3d969..bbf62a5 100644
--- a/ooni/plugins/tcpconnect.py
+++ b/ooni/plugins/tcpconnect.py
@@ -9,8 +9,9 @@ from twisted.plugin import IPlugin
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset
+from plugoo.interface import ITest
+from plugoo.tests import OONITest
+from plugoo.assets import Asset
from ooni.utils import log
class tcpconnectArgs(usage.Options):
diff --git a/ooni/plugoo/nodes.py b/ooni/plugoo/nodes.py
index 0d01348..155f183 100644
--- a/ooni/plugoo/nodes.py
+++ b/ooni/plugoo/nodes.py
@@ -7,7 +7,7 @@
This contains all the code related to Nodes
both network and code execution.
- :copyright: (c) 2012 by Arturo Filastò.
+ :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft
:license: see LICENSE for more details.
"""
@@ -20,10 +20,6 @@ try:
except:
print "Error: module paramiko is not installed."
from pprint import pprint
-try:
- import pyXMLRPCssh
-except:
- print "Error: module pyXMLRPCssh is not installed."
import sys
import socks
import xmlrpclib
diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py
index ef15a04..d0d9af3 100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@ -4,7 +4,7 @@ import os
import yaml
import itertools
-import log
+from utils import log, date, net
class Report:
"""This is the ooni-probe reporting mechanism. It allows
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 19d42b2..42f9542 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -8,9 +8,10 @@ from twisted.internet import reactor, defer, threads
## XXX why is this imported and not used?
from twisted.python import failure
-from ooni import log
+from utils import log, date
from plugoo import assets, work
from plugoo.reports import Report
+from plugoo.interface import ITest
class OONITest(object):
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index cf57186..dd5cf13 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -93,7 +93,6 @@ def debug(message, level="debug", **kw):
def msg(message, level="info", **kw):
log.msg(message, logLevel=level, **kw)
-## XXX fixme log.err messages get printed to stdout twice
def err(message, level="err", **kw):
log.err(message, logLevel=level, **kw)
More information about the tor-commits
mailing list