commit cf5e13c241389f8192a587c0601a8cf561329ae7 Author: Isis Lovecruft isis@torproject.org Date: Fri Sep 14 02:20:52 2012 +0000
Fixed imports. --- ooni/lib/Makefile | 30 ++ ooni/lib/__init__.py | 43 +--- ooni/lib/txscapy | 1 - ooni/lib/txscapy.py | 348 ++++++++++++++++++++ ooni/lib/txtraceroute | 1 - ooni/lib/txtraceroute.py | 752 ++++++++++++++++++++++++++++++++++++++++++++ ooni/ooniprobe.py | 10 +- ooni/plugins/blocking.py | 2 +- ooni/plugins/dnstamper.py | 6 +- ooni/plugins/httphost.py | 2 +- ooni/plugins/tcpconnect.py | 5 +- ooni/plugoo/nodes.py | 6 +- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 3 +- ooni/utils/log.py | 1 - 15 files changed, 1152 insertions(+), 60 deletions(-)
diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile new file mode 100644 index 0000000..3b0c922 --- /dev/null +++ b/ooni/lib/Makefile @@ -0,0 +1,30 @@ +all: txtorcon txtraceroute + +txtraceroute: + echo "Processing dependency txtraceroute..." + git clone https://github.com/hellais/txtraceroute.git txtraceroute.git + mv txtraceroute.git/txtraceroute.py txtraceroute.py + rm -rf txtraceroute.git + +txtorcon: + echo "Processing dependency txtorcon..." + git clone https://github.com/meejah/txtorcon.git txtorcon.git + mv txtorcon.git/txtorcon txtorcon + rm -rf txtorcon.git + +clean: + rm -rf txtorcon + rm -rf txtraceroute.py + +#txscapy: +# echo "Processing dependency txscapy" +# git clone https://github.com/hellais/txscapy.git txscapy.git +# mv txscapy.git/txscapy.py txscapy.py +# rm -rf txscapy.git + +#rfc3339: +# echo "Processing RFC3339 dependency" +# hg clone https://bitbucket.org/henry/rfc3339 rfc3339 +# mv rfc3339/rfc3339.py rfc3339.py +# rm -rf rfc3339 + diff --git a/ooni/lib/__init__.py b/ooni/lib/__init__.py index 0fd36c5..611d50c 100644 --- a/ooni/lib/__init__.py +++ b/ooni/lib/__init__.py @@ -1,40 +1,5 @@ -import pkgutil -import sys -from os import listdir, path +from sys import path as syspath +from os import path as ospath
-__all__ = ['txtorcon', 'txscapy', 'txtraceroute'] - -__sub_modules__ = [ ] - -def callback(arg, directory, files): - for file in listdir(directory): - fullpath = path.abspath(file) - if path.isdir(fullpath) and not path.islink(fullpath): - __sub_modules__.append(fullpath) - sys.path.append(fullpath) - -path.walk(".", callback, None) - -def load_submodules(init, list): - for subdir in list: - contents=[x for x in pkgutil.iter_modules(path=subdir, - prefix='ooni.lib.')] - for loader, module_name, ispkg in contents: - init_dot_module = init + "." + module_name - if init_dot_module in sys.modules: - module = sys.modules[module_name] - else: - if module_name in __all__: - grep = loader.find_module(module_name) - module = grep.load_module(module_name) - else: - module = None - - if module is not None: - globals()[module_name] = module - -load_submodules(__name__, __sub_modules__) - -print "system paths are: %s" % sys.path -print "globals are: %s" % globals() -print "system modules are: %s" % sys.modules +pwd = ospath.dirname(__file__) +syspath.append(pwd) diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy deleted file mode 160000 index 19fb281..0000000 --- a/ooni/lib/txscapy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py new file mode 100644 index 0000000..4d83dce --- /dev/null +++ b/ooni/lib/txscapy.py @@ -0,0 +1,348 @@ +# -*- coding:utf8 -*- +""" + txscapy + ****** + (c) 2012 Arturo Filastò + a twisted wrapper for scapys send and receive functions. + + This software has been written to be part of OONI, the Open Observatory of + Network Interference. More information on that here: http://ooni.nu/ + +""" + +import struct +import socket +import os +import sys +import time + +from twisted.internet import protocol, base, fdesc, error, defer +from twisted.internet import reactor, threads +from twisted.python import log +from zope.interface import implements + +from scapy.all import Gen +from scapy.all import SetGen + +LINUX=sys.platform.startswith("linux") +OPENBSD=sys.platform.startswith("openbsd") +FREEBSD=sys.platform.startswith("freebsd") +NETBSD=sys.platform.startswith("netbsd") +DARWIN=sys.platform.startswith("darwin") +SOLARIS=sys.platform.startswith("sunos") +WINDOWS=sys.platform.startswith("win32") + +from scapy.all import RawPcapWriter, MTU, BasePacketList, conf +class PcapWriter(RawPcapWriter): + def __init__(self, filename, linktype=None, gz=False, endianness="", + append=False, sync=False): + RawPcapWriter.__init__(self, filename, linktype=None, gz=False, + endianness="", append=False, sync=False) + fdesc.setNonBlocking(self.f) + + def _write_header(self, pkt): + if self.linktype == None: + if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList): + pkt = pkt[0] + try: + self.linktype = conf.l2types[pkt.__class__] + except KeyError: + self.linktype = 1 + RawPcapWriter._write_header(self, pkt) + + def _write_packet(self, packet): + sec = int(packet.time) + usec = int(round((packet.time-sec)*1000000)) + s = str(packet) + caplen = len(s) + RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen) + +class ScapySocket(object): + MTU = 1500 + def __init__(self, filter=None, iface=None, nofilter=None): + from scapy.all import conf + self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) + + def fileno(self): + return self.ssocket.ins.fileno() + + def send(self, data): + return self.ssocket.send(data) + + def recv(self): + if FREEBSD or DARWIN: + return self.ssocket.nonblock_recv() + else: + return self.ssocket.recv(self.MTU) + +class Scapy(object): + """ + A twisted based wrapper for scapy send and receive functionality. + + It sends packets inside of a threadpool and receives packets using the + libdnet receive non blocking file descriptor. + """ + min = 2 + max = 6 + debug = True + write_only_answers = False + pcapwriter = None + recv = False + + def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, + iface=None, nofilter=None, pcapfile=None): + + if self.debug: + log.startLogging(sys.stdout) + + self.maxPacketSize = maxPacketSize + if not reactor: + from twisted.internet import reactor + + self._reactor = reactor + + if pkts: + self._buildPacketQueues(pkts) + self._buildSocket() + + self.cthreads = 0 + self.mthreads = 80 + + self.running = False + self.done = False + self.finished = False + + import thread + from twisted.python import threadpool + self.threadID = thread.get_ident + self.threadpool = threadpool.ThreadPool(self.min, self.max) + self.startID = self._reactor.callWhenRunning(self._start) + + self.deferred = defer.Deferred() + + if pcapfile: + self.pcapwriter = PcapWriter(pcapfile) + + def _buildSocket(self, filter=None, iface=None, nofilter=None): + self.socket = ScapySocket(filter, iface, nofilter) + if self.recv: + self._reactor.addReader(self) + + def _buildPacketQueues(self, pkts): + """ + Converts the list of packets to a Scapy generator and sets up all the + necessary attributes for understanding if all the needed responses have + been received. + """ + if not isinstance(pkts, Gen): + self.pkts = SetGen(pkts) + + self.outqueue = [p for p in pkts] + + self.total_count = len(self.outqueue) + self.answer_count = 0 + self.out_count = 0 + + self.hsent = {} + for p in self.outqueue: + h = p.hashret() + if h in self.hsent: + self.hsent[h].append(p) + else: + self.hsent[h] = [p] + + + def gotAnswer(self, answer, question): + """ + Got a packet that has been identified as an answer to one of the sent + out packets. + + If the answer count matches the sent count the finish callback is + fired. + + @param answer: the packet received on the wire. + + @param question: the sent packet that matches that response. + + """ + + if self.pcapwriter and self.write_only_answers: + self.pcapwriter.write(question) + self.pcapwriter.write(answer) + self.answer_count += 1 + if self.answer_count >= self.total_count: + print "Got all the answers I need" + self.deferred.callback(None) + + def processAnswer(self, pkt, hlst): + """ + Checks if the potential answer is in fact an answer to one of the + matched sent packets. Uses the scapy .answers() function to verify + this. + + @param pkt: The packet to be tested if is the answer to a sent packet. + + @param hlst: a list of packets that match the hash for an answer to + pkt. + """ + for i in range(len(hlst)): + if pkt.answers(hlst[i]): + self.gotAnswer(pkt, hlst[i]) + + def fileno(self): + """ + Returns a fileno for use by twisteds Reader. + """ + return self.socket.fileno() + + def processPacket(self, pkt): + """ + Override this method to process your packets. + + @param pkt: the packet that has been received. + """ + #pkt.show() + + + def doRead(self): + """ + There is something to be read on the wire. Do all the processing on the + received packet. + """ + pkt = self.socket.recv() + if self.pcapwriter and not self.write_only_answers: + self.pcapwriter.write(pkt) + self.processPacket(pkt) + + h = pkt.hashret() + if h in self.hsent: + hlst = self.hsent[h] + self.processAnswer(pkt, hlst) + + def logPrefix(self): + """ + The prefix to be prepended in logging. + """ + return "txScapy" + + def _start(self): + """ + Start the twisted thread pool. + """ + self.startID = None + return self.start() + + def start(self): + """ + Actually start the thread pool. + """ + if not self.running: + self.threadpool.start() + self.shutdownID = self._reactor.addSystemEventTrigger( + 'during', 'shutdown', self.finalClose) + self.running = True + + def sendPkt(self, pkt): + """ + Send a packet to the wire. + + @param pkt: The packet to be sent. + """ + self.socket.send(pkt) + + def sr(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): + """ + Wraps the scapy sr function. + + @param nofilter: put 1 to avoid use of bpf filters + + @param retry: if positive, how many times to resend unanswered packets + if negative, how many times to retry when no more packets are + answered (XXX to be implemented) + + @param timeout: how much time to wait after the last packet has + been sent (XXX to be implemented) + + @param multi: whether to accept multiple answers for the same + stimulus (XXX to be implemented) + + @param filter: provide a BPF filter + @param iface: listen answers only on the given interface + """ + self.recv = True + self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) + + def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): + """ + Wraps the scapy send function. Its the same as send and receive, except + it does not receive. Who would have ever guessed? ;) + + @param nofilter: put 1 to avoid use of bpf filters + + @param retry: if positive, how many times to resend unanswered packets + if negative, how many times to retry when no more packets are + answered (XXX to be implemented) + + @param timeout: how much time to wait after the last packet has + been sent (XXX to be implemented) + + @param multi: whether to accept multiple answers for the same + stimulus (XXX to be implemented) + + @param filter: provide a BPF filter + @param iface: listen answers only on the given interface + """ + self.recv = False + self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) + + def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0): + self._buildSocket(filter, iface, nofilter) + self._buildPacketQueues(pkts) + def sent(cb): + if self.cthreads < self.mthreads and not self.done: + pkt = None + try: + pkt = self.outqueue.pop() + except: + self.done = True + if not self.recv: + self.deferred.callback(None) + return + d = threads.deferToThreadPool(reactor, self.threadpool, + self.sendPkt, pkt) + d.addCallback(sent) + return d + + for x in range(self.mthreads): + try: + pkt = self.outqueue.pop() + except: + self.done = True + return + if self.cthreads >= self.mthreads and self.done: + return + d = threads.deferToThreadPool(reactor, self.threadpool, + self.sendPkt, pkt) + d.addCallback(sent) + return d + + def connectionLost(self, why): + pass + + def finalClose(self): + """ + Clean all the thread related stuff up. + """ + self.shutdownID = None + self.threadpool.stop() + self.running = False + +def txsr(*args, **kw): + tr = Scapy(*args, **kw) + tr.sr(*args, **kw) + return tr.deferred + +def txsend(*arg, **kw): + tr = Scapy(*arg, **kw) + tr.send(*arg, **kw) + return tr.deferred diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute deleted file mode 160000 index 067a260..0000000 --- a/ooni/lib/txtraceroute +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 067a2609390e77bf9187275638cf8786efef7e13 diff --git a/ooni/lib/txtraceroute.py b/ooni/lib/txtraceroute.py new file mode 100644 index 0000000..8182b34 --- /dev/null +++ b/ooni/lib/txtraceroute.py @@ -0,0 +1,752 @@ +#!/usr/bin/env python +# coding: utf-8 +# +# Copyright (c) 2012 Alexandre Fiori +# Arturo Filastò +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import json +import operator +import os +import socket +import struct +import sys +import time +import random +import itertools +from pprint import pprint + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import threads +from twisted.python import usage +from twisted.web.client import getPage + +class iphdr(object): + """ + This represents an IP packet header. + + XXX enable IP_TIMESTAMP in setsockopt + to get the timestamp of when the router says it has gotten an ICMP + timeout. + + @assemble packages the packet + @disassemble disassembles the packet + """ + def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None): + self.version = 4 + self.hlen = 5 + self.tos = 0 + self.length = 20 + self.id = os.getpid() + self.frag = 0 + self.ttl = 255 + self.proto = proto + self.cksum = 0 + self.src = src + self.saddr = socket.inet_aton(src) + self.dst = dst or "0.0.0.0" + self.daddr = socket.inet_aton(self.dst) + self.data = "" + + def assemble(self): + header = struct.pack('BBHHHBB', + (self.version & 0x0f) << 4 | (self.hlen & 0x0f), + self.tos, self.length + len(self.data), + socket.htons(self.id), self.frag, + self.ttl, self.proto) + self._raw = header + "\x00\x00" + self.saddr + self.daddr + self.data + return self._raw + + @classmethod + def disassemble(self, data): + self._raw = data + ip = iphdr() + pkt = struct.unpack('!BBHHHBBH', data[:12]) + ip.version = (pkt[0] >> 4 & 0x0f) + ip.hlen = (pkt[0] & 0x0f) + ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:] + ip.saddr = data[12:16] + ip.daddr = data[16:20] + ip.src = socket.inet_ntoa(ip.saddr) + ip.dst = socket.inet_ntoa(ip.daddr) + return ip + + def __repr__(self): + return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \ + "%s -> %s" % \ + (self.tos, self.ttl, self.id, self.frag, self.proto, + self.length, self.src, self.dst) + +class tcphdr(object): + def __init__(self, data="", sport=4242, dport=4242): + self.seq = 0 + self.hlen = 44 + self.flags = 2 + self.wsize = 200 + self.cksum = 123 + self.options = 0 + self.mss = 1460 + self.dport = dport + self.sport = sport + + def assemble(self): + header = struct.pack("!HHL", self.sport, self.dport, self.seq) + header += '\x00\x00\x00\x00' + header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags & + 0xff), self.wsize, self.cksum) + header += "\x00\x00" + options = '\x02\x04\x05\xb4\x01\x03\x03\x01\x01\x01\x08\x0a' + options += '\x4d\xcf\x52\x33\x00\x00\x00\x00\x04\x02\x00\x00' + # XXX There is something wrong here fixme + # options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1) + # options += struct.pack("!BBL", 8, 10, 1209452188) + # options += '\00'*4 + # options += struct.pack("!BB", 4, 2) + # options += '\00' + self._raw = header+options + return self._raw + + @classmethod + def checksum(self, data): + pass + + def __repr__(self): + return "<TCPPacket (sport: %s dport: %s seq: %s) " %\ + (self.sport, self.dport, self.seq) + + @classmethod + def disassemble(self, data): + self._raw = data + tcp = tcphdr() + pkt = struct.unpack("!HHL", data[:8]) + tcp.sport, tcp.dport, tcp.seq = pkt + if len(data) > 10: + pkt = struct.unpack("!H", data[8:10]) + tcp.hlen = (pkt[0] >> 10 ) & 0xff + tcp.flags = pkt[0] & 0xff + tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:24]) + return tcp + +class udphdr(object): + def __init__(self, data="", sport=4242, dport=4242): + self.dport = dport + self.sport = sport + self.cksum = 0 + self.length = 0 + self.data = data + + def assemble(self): + self.length = len(self.data) + 8 + part1 = struct.pack("!HHH", self.sport, self.dport, self.length) + cksum = self.checksum(self.data) + cksum = struct.pack("!H", cksum) + + self._raw = part1 + cksum + self.data + return self._raw + + @classmethod + def checksum(self, data): + # XXX implement proper checksum + cksum = 0 + return cksum + + def __repr__(self): + return "<UDPPacket (sport %s, dport %s, length %s, data %s)>" % \ + (self.sport, self.dport, self.length, self.data) + + @classmethod + def disassemble(self, data): + self._raw = data + udp = udphdr() + pkt = struct.unpack("!HHHH", data[:8]) + udp.sport, udp.dport, udp.length, udp.cksum = pkt + udp.data = data[8:] + return udp + +class icmphdr(object): + def __init__(self, data=""): + self.type = 8 + self.code = 0 + self.cksum = 0 + self.id = os.getpid() + self.sequence = 0 + self.data = data + + def assemble(self): + part1 = struct.pack("BB", self.type, self.code) + part2 = struct.pack("!HH", self.id, self.sequence) + cksum = self.checksum(part1 + "\x00\x00" + part2 + self.data) + cksum = struct.pack("!H", cksum) + self._raw = part1 + cksum + part2 + self.data + return self._raw + + @classmethod + def checksum(self, data): + if len(data) & 1: + data += "\x00" + cksum = reduce(operator.add, + struct.unpack('!%dH' % (len(data) >> 1), data)) + cksum = (cksum >> 16) + (cksum & 0xffff) + cksum += (cksum >> 16) + cksum = (cksum & 0xffff) ^ 0xffff + return cksum + + @classmethod + def disassemble(self, data): + self._raw = data + icmp = icmphdr() + pkt = struct.unpack("!BBHHH", data) + icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt + return icmp + + def __repr__(self): + return "ICMP (type %s, code %s, id %s, sequence %s)" % \ + (self.type, self.code, self.id, self.sequence) + + +def pprintp(packet): + """ + Used to pretty print packets. + """ + lines = [] + line = [] + for i, byte in enumerate(packet): + line.append(("%.2x" % ord(byte), byte)) + if (i + 1) % 8 == 0: + lines.append(line) + line = [] + + lines.append(line) + + for row in lines: + left = "" + right = " " * (8 - len(row)) + for y in row: + left += "%s " % y[0] + right += "%s" % y[1] + + print left + " " + right + +@defer.inlineCallbacks +def geoip_lookup(ip): + try: + r = yield getPage("http://freegeoip.net/json/%s" % ip) + d = json.loads(r) + items = [d["country_name"], d["region_name"], d["city"]] + text = ", ".join([s for s in items if s]) + defer.returnValue(text.encode("utf-8")) + except Exception: + defer.returnValue("Unknown location") + + +@defer.inlineCallbacks +def reverse_lookup(ip): + try: + r = yield threads.deferToThread(socket.gethostbyaddr, ip) + defer.returnValue(r[0]) + except Exception: + defer.returnValue(None) + + +class Hop(object): + def __init__(self, target, ttl, proto, sport=None, dport=None): + self.proto = proto + self.dport = dport + self.sport = sport + + self.found = False + self.tries = 0 + self.last_try = 0 + self.remote_ip = None + self.remote_icmp = None + self.remote_host = None + self.location = "" + + self.ttl = ttl + self.ip = iphdr(dst=target) + self.ip.ttl = ttl + self.ip.id += ttl + if self.proto == "icmp": + self.icmp = icmphdr('\x00'*20) + self.icmp.id = self.ip.id + self.ip.data = self.icmp.assemble() + elif self.proto == "udp": + self.udp = udphdr('\x00'*20, self.sport, self.dport) + self.ip.data = self.udp.assemble() + self.ip.proto = socket.IPPROTO_UDP + else: + self.tcp = tcphdr('\x42'*20, self.sport, self.dport) + self.ip.data = self.tcp.assemble() + self.ip.proto = socket.IPPROTO_TCP + + self._pkt = self.ip.assemble() + + @property + def pkt(self): + self.tries += 1 + self.last_try = time.time() + return self._pkt + + def get(self): + if self.found: + if self.remote_host: + ip = self.remote_host + else: + ip = self.remote_ip.src + ping = self.found - self.last_try + else: + ip = None + ping = None + + location = self.location if self.location else None + return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location, + 'proto': self.proto, 'dport': self.dport, 'sport': self.sport} + + def __repr__(self): + if self.found: + if self.remote_host: + ip = ":: %s" % self.remote_host + else: + ip = ":: %s" % self.remote_ip.src + ping = "%0.3fs" % (self.found - self.last_try) + else: + ip = "??" + ping = "-" + + location = ":: %s" % self.location if self.location else "" + return "%02d. %s %s %s (%s, sport: %s dport: %s)" % (self.ttl, ping, ip, location, self.proto, self.sport, self.dport) + +class TracerouteResult(object): + """ + Used to store the results of a Traceroute. + """ + #src_ports = [0, 9090] + #dst_ports = [0, 21, 123, 80, 443] + src_ports = [0, 80] + dst_ports = [0, 80] + hops = [] + done = False + + def __init__(self, protocol): + self.protocol = protocol + self.probes = {} + + if protocol == "icmp": + self.current = None + else: + self.current = {} + for src, dst in itertools.product(self.src_ports, + self.dst_ports): + if src not in self.probes: + self.probes[src] = {} + self.probes[src][dst] = [] + + if src not in self.current: + self.current[src] = {} + self.current[src][dst] = None + + def get_current_probes(self): + if self.protocol == "icmp": + return self.current + + def add_to_current_probes(self, probe): + if self.protocol == "icmp": + self.current = probe + else: + self.current[probe.sport][probe.dport] = probe + + def is_in_progress(self): + if self.protocol == "icmp": + progress = self.current + else: + progress = None + for x in self.current: + for y in self.current[x]: + if self.current[x][y] != None: + progress = True + if progress is None: + return False + else: + return True + + def get(self, src=None, dst=None): + if self.protocol == "icmp": + return self.probes + else: + return self.probes[src][dst] + + def append(self, probe, src=None, dst=None): + if self.protocol == "icmp": + self.probes.append(hop) + else: + self.probes[src][dst].append(probe) + + def pop(self, src=None, dst=None): + if self.protocol == "icmp": + hop = self.current + self.current = None + return hop + + elif (dst != None) and (src != None): + hop = self.current[src][dst] + self.current[src][dst] = None + return hop + + else: + raise Exception("Did not specify dst and src ports") + + @classmethod + def hops(self, target, ttl): + """ + Generates a set of ooni probes for traceroute based network tampering + detection. + + We send in one round a set of packets with same TTL but on all protocols + and with all possible source and destination ports. + """ + hops = [] + for src, dst in itertools.product(self.src_ports, self.dst_ports): + hops.append(Hop(target, ttl, + "tcp", src, dst)) + hops.append(Hop(target, ttl, + "udp", src, dst)) + hops.append(Hop(target, ttl, "icmp", 0, 0)) + return hops + +class TracerouteProtocol(object): + def __init__(self, target, **settings): + + self.target = target + self.settings = settings + self.verbose = settings.get("verbose") + self.proto = settings.get("proto") + self.rfd = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + self.sfd = {} + + # Create the data structures to contain the test results + self.traceroute = {} + self.traceroute["tcp"] = TracerouteResult("tcp") + self.traceroute["udp"] = TracerouteResult("udp") + self.traceroute["icmp"] = TracerouteResult("icmp") + + if self.settings.get("ooni"): + self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_TCP) + self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_UDP) + elif self.proto == "icmp": + self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_ICMP) + elif self.proto == "udp": + self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_UDP) + elif self.proto == "tcp": + self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW, + socket.IPPROTO_TCP) + + # Let me add IP Headers myself, just give me a socket! + self.rfd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) + for fd in self.sfd: + self.sfd[fd].setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) + + self.out_queue = [] + self.waiting = True + self.deferred = defer.Deferred() + + reactor.addReader(self) + reactor.addWriter(self) + + # send 1st probe packet(s) + if self.settings.get("ooni"): + hops = list(TracerouteResult.hops(self.target, 1)) + else: + hops = [Hop(self.target, 1, + settings.get("proto"), + self.settings.get("sport"), + self.settings.get("dport"))] + for hop in hops: + # Store the to be completed items inside of a dictionary + self.traceroute[hop.proto].add_to_current_probes(hop) + self.out_queue.append(hop) + + def logPrefix(self): + return "TracerouteProtocol(%s)" % self.target + + def fileno(self): + return self.rfd.fileno() + + @defer.inlineCallbacks + def hopFound(self, hop, ip, icmp, ref, subref): + hop.remote_ip = ip + hop.remote_icmp = icmp + + if (ip and icmp): + hop.found = time.time() + if self.settings.get("geoip_lookup") is True: + hop.location = yield geoip_lookup(ip.src) + + if self.settings.get("reverse_lookup") is True: + hop.remote_host = yield reverse_lookup(ip.src) + + ttl = hop.ttl + 1 + + if (ttl > (self.settings.get("max_hops", 30) + 1)): + done = True + else: + done = False + + if not done: + cb = self.settings.get("hop_callback") + if callable(cb): + yield defer.maybeDeferred(cb, hop) + + if not self.waiting: + if self.deferred: + self.deferred.callback(self.hops) + self.deferred = None + else: + hops = [] + if self.settings.get("ooni"): + if not (self.traceroute["icmp"].is_in_progress() or + self.traceroute["tcp"].is_in_progress() or + self.traceroute["udp"].is_in_progress()): + # Add hops only if we are not in progress + hops = list(TracerouteResult.hops(self.target, ttl)) + else: + hops = [Hop(self.target, ttl, + settings.get("proto"), + self.settings.get("sport"), + self.settings.get("dport"))] + + for hop in hops: + # Store the to be completed items inside of a dictionary + self.traceroute[hop.proto].add_to_current_probes(hop) + self.out_queue.append(hop) + + def doRead(self): + if not self.waiting: + return + + pkt = self.rfd.recv(4096) + # disassemble ip header + ip = iphdr.disassemble(pkt[:20]) + + if self.verbose: + print "Got this packet:" + print "src %s" % ip.src + pprintp(pkt) + + # Not interested in non ICMP packets. + if ip.proto != socket.IPPROTO_ICMP: + return + + found = False + foundHop = None + + # disassemble icmp header + icmp = icmphdr.disassemble(pkt[20:28]) + + if self.verbose: + print icmp + + # If it's an ICMP Echo reply then our ICMP probe has hit destination + if icmp.type == 0 and icmp.id == self.current_hop["icmp"][1].icmp.id: + foundHop = self.traceroute["icmp"].pop() + found = True + + elif icmp.type == 11: + # disassemble referenced ip header + ref = iphdr.disassemble(pkt[28:48]) + subref = None + + if self.verbose: + print ref + + if ref.dst == self.target: + found = True + + if ref.proto == socket.IPPROTO_UDP: + subref = udphdr.disassemble(pkt[48:]) + proto = "udp" + + elif ref.proto == socket.IPPROTO_TCP: + subref = tcphdr.disassemble(pkt[48:]) + proto = "tcp" + + else: + proto = "icmp" + + if subref: + sport = subref.sport + dport = subref.dport + else: + sport = None + dport = None + # Remove completed hops + foundHop = self.traceroute[proto].pop(sport, + dport) + + if ip.src == self.target: + self.waiting = False + + if found: + self.hopFound(foundHop, ip, icmp, ref, subref) + elif foundHop: + self.hopFound(foundHop, ip, icmp, ref, subref) + + def hopTimeout(self, hop): + if not hop.found: + if hop.tries < self.settings.get("max_tries", 3): + # retry + hop.tries += 1 + self.out_queue.append(hop) + else: + # give up and move forward + self.traceroute[hop.proto].pop(hop.dport, + hop.sport) + self.hopFound(hop, None, None, None, None) + + def doWrite(self): + if self.waiting and self.out_queue: + hop = self.out_queue.pop(0) + pkt = hop.pkt + if self.verbose: + print "Sending this packet:" + pprintp(pkt) + print hop + + self.sfd[hop.proto].sendto(pkt, (hop.ip.dst, 0)) + + self.traceroute[hop.proto].add_to_current_probes(hop) + + timeout = self.settings.get("timeout", 1) + reactor.callLater(timeout, self.hopTimeout, hop) + + def connectionLost(self, why): + pass + + +def traceroute(target, **settings): + tr = TracerouteProtocol(target, **settings) + return tr.deferred + + +@defer.inlineCallbacks +def start_trace(target, **settings): + hops = yield traceroute(target, **settings) + if settings["hop_callback"] is None: + for hop in hops: + print hop + reactor.stop() + +class Options(usage.Options): + optFlags = [ + ["quiet", "q", "Only print results at the end."], + ["no-dns", "n", "Show numeric IPs only, not their host names."], + ["no-geoip", "g", "Do not collect and show GeoIP information"], + ["verbose", "v", "Be more verbose"], + ["ooni", "o", "Run the ooni common port multiprotocol traceroute"], + ["help", "h", "Show this help"], + ] + optParameters = [ + ["timeout", "t", 2, "Timeout for probe packets"], + ["tries", "r", 3, "How many tries before give up probing a hop"], + ["proto", "p", "icmp", "What protocol to use (tcp, udp, icmp)"], + ["dport", "d", random.randint(2**10, 2**16), "Destination port (TCP and UDP only)"], + ["sport", "s", random.randint(2**10, 2**16), "Source port (TCP and UDP only)"], + ["max_hops", "m", 30, "Max number of hops to probe"] + ] + +def main(): + def show(hop): + print hop + + defaults = dict(hop_callback=show, + reverse_lookup=True, + geoip_lookup=True, + timeout=2, + proto="icmp", + dport=None, + sport=None, + verbose=False, + ooni=False, + max_tries=3, + max_hops=30) + + if len(sys.argv) < 2: + print("Usage: %s [options] host" % (sys.argv[0])) + print("%s: Try --help for usage details." % (sys.argv[0])) + sys.exit(1) + + target = sys.argv.pop(-1) if sys.argv[-1][0] != "-" else "" + config = Options() + try: + config.parseOptions() + if not target: + raise + except usage.UsageError, e: + print("%s: %s" % (sys.argv[0], e)) + print("%s: Try --help for usage details." % (sys.argv[0])) + sys.exit(1) + + settings = defaults.copy() + if config.get("silent"): + settings["hop_callback"] = None + if config.get("no-dns"): + settings["reverse_lookup"] = False + if config.get("no-geoip"): + settings["geoip_lookup"] = False + if config.get("verbose"): + settings["verbose"] = True + if config.get("ooni"): + settings["ooni"] = True + if "timeout" in config: + settings["timeout"] = config["timeout"] + if "tries" in config: + settings["max_tries"] = config["tries"] + if "proto" in config: + settings["proto"] = config["proto"] + if "max_hops" in config: + settings["max_hops"] = config["max_hops"] + if "dport" in config: + settings["dport"] = int(config["dport"]) + if "sport" in config: + settings["sport"] = int(config["sport"]) + + if os.getuid() != 0: + print("traceroute needs root privileges for the raw socket") + sys.exit(1) + try: + target = socket.gethostbyname(target) + except Exception, e: + print("could not resolve '%s': %s" % (target, str(e))) + sys.exit(1) + + reactor.callWhenRunning(start_trace, target, **settings) + reactor.run() + +if __name__ == "__main__": + main() + diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 95b5a18..539c2ac 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -27,9 +27,11 @@ from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation
-from ooni.plugoo import tests, work, assets, reports -from ooni.logo import getlogo -from ooni import plugins, log +from plugoo import tests, work, assets, reports +from plugoo.interface import ITest +from utils.logo import getlogo +from utils import log +import plugins
__version__ = "0.0.1-prealpha"
@@ -38,7 +40,7 @@ def retrieve_plugoo(): Get all the plugins that implement the ITest interface and get the data associated to them into a dict. """ - interface = tests.ITest + interface = ITest d = {} error = False for p in getPlugins(interface, plugins): diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index 72fd49f..f3c20e1 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -3,7 +3,7 @@ from twisted.python import usage from twisted.plugin import IPlugin
from plugoo.assets import Asset -from plugoo.tests import ITest, TwistedTest +from plugoo.tests import ITest, OONITest
class BlockingArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index 54cfbdf..3d373ce 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -38,9 +38,9 @@ from twisted.python import usage from twisted.plugin import IPlugin from zope.interface import implements
-from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest -from ooni import log +from plugoo.assets import Asset +from plugoo.tests import ITest, OONITest +from utils import log
class AlexaAsset(Asset): """ diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index e8808bd..7c783a1 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -21,7 +21,7 @@ from twisted.python import usage from twisted.plugin import IPlugin
from plugoo.assets import Asset -from plugoo.tests import ITest, TwistedTest +from plugoo.tests import ITest, OONITest
class HTTPHostArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py index db3d969..bbf62a5 100644 --- a/ooni/plugins/tcpconnect.py +++ b/ooni/plugins/tcpconnect.py @@ -9,8 +9,9 @@ from twisted.plugin import IPlugin from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint
-from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset +from plugoo.interface import ITest +from plugoo.tests import OONITest +from plugoo.assets import Asset from ooni.utils import log
class tcpconnectArgs(usage.Options): diff --git a/ooni/plugoo/nodes.py b/ooni/plugoo/nodes.py index 0d01348..155f183 100644 --- a/ooni/plugoo/nodes.py +++ b/ooni/plugoo/nodes.py @@ -7,7 +7,7 @@ This contains all the code related to Nodes both network and code execution.
- :copyright: (c) 2012 by Arturo Filastò. + :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft :license: see LICENSE for more details.
""" @@ -20,10 +20,6 @@ try: except: print "Error: module paramiko is not installed." from pprint import pprint -try: - import pyXMLRPCssh -except: - print "Error: module pyXMLRPCssh is not installed." import sys import socks import xmlrpclib diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index ef15a04..d0d9af3 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml
import itertools -import log +from utils import log, date, net
class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 19d42b2..42f9542 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -8,9 +8,10 @@ from twisted.internet import reactor, defer, threads ## XXX why is this imported and not used? from twisted.python import failure
-from ooni import log +from utils import log, date from plugoo import assets, work from plugoo.reports import Report +from plugoo.interface import ITest
class OONITest(object): diff --git a/ooni/utils/log.py b/ooni/utils/log.py index cf57186..dd5cf13 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -93,7 +93,6 @@ def debug(message, level="debug", **kw): def msg(message, level="info", **kw): log.msg(message, logLevel=level, **kw)
-## XXX fixme log.err messages get printed to stdout twice def err(message, level="err", **kw): log.err(message, logLevel=level, **kw)