commit cfa9a2355869746ea0a32a51be45dd45de436281
Author: aagbsn <aagbsn(a)extc.org>
Date: Sat Feb 1 20:37:56 2014 +0000
Refactor Multi Protocol Traceroute, add ParasiticTraceroute
Make the report output closer to the original, refactor packet
matching logic.
Adds a ParasiticTraceroute ScapyProtocol. Experimental.
---
ooni/nettests/manipulation/traceroute.py | 63 +++++----
ooni/utils/txscapy.py | 208 ++++++++++++++++++++++++------
2 files changed, 201 insertions(+), 70 deletions(-)
diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py
index b970127..0fb715c 100644
--- a/ooni/nettests/manipulation/traceroute.py
+++ b/ooni/nettests/manipulation/traceroute.py
@@ -12,7 +12,7 @@ from itertools import chain
from scapy.all import *
from ooni.utils import log
-from ooni.utils.txscapy import ScapyTraceroute
+from ooni.utils.txscapy import MPTraceroute
from ooni.settings import config
class UsageOptions(usage.Options):
@@ -20,7 +20,9 @@ class UsageOptions(usage.Options):
['backend', 'b', None, 'Test backend to use'],
['timeout', 't', 5, 'The timeout for the traceroute test'],
['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'],
- ['srcport', 'p', None, 'Set the source port to a specific value (only applies to TCP and UDP)']
+ ['dstport', 'd', None, 'Specify a single destination port. May be repeated.'],
+ ['interval', 'i', None, 'Specify the inter-packet delay in seconds'],
+ ['numPackets', 'n', None, 'Specify the number of packets to send per hop'],
]
class TracerouteTest(scapyt.BaseScapyTest):
@@ -29,33 +31,31 @@ class TracerouteTest(scapyt.BaseScapyTest):
requiredTestHelpers = {'backend': 'traceroute'}
usageOptions = UsageOptions
dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535]
- timeout = 5
+ version = "0.3"
def setUp(self):
- self.st = ScapyTraceroute()
+ self.st = MPTraceroute()
if self.localOptions['maxttl']:
self.st.ttl_max = int(self.localOptions['maxttl'])
+ if self.localOptions['dstport']:
+ self.st.dst_ports = int(self.localOptions['dstport'])
+ if self.localOptions['interval']:
+ self.st.interval = float(self.localOptions['interval'])
+
config.scapyFactory.registerProtocol(self.st)
- self.done = defer.Deferred()
- self.tcp = self.udp = self.icmp = None
+
+ self.report['test_tcp_traceroute'] = dict([('hops_%d' % d,[]) for d in self.dst_ports])
+ self.report['test_udp_traceroute'] = dict([('hops_%d' % d,[]) for d in self.dst_ports])
+ self.report['test_icmp_traceroute'] = {'hops': []}
def test_icmp_traceroute(self):
- self.st.ICMPTraceroute(self.localOptions['backend'])
- d = defer.Deferred()
- reactor.callLater(self.timeout, d.callback, self.st)
- return d
+ return self.st.ICMPTraceroute(self.localOptions['backend'])
def test_tcp_traceroute(self):
- self.st.TCPTraceroute(self.localOptions['backend'])
- d = defer.Deferred()
- reactor.callLater(self.timeout, d.callback, self.st)
- return d
+ return self.st.TCPTraceroute(self.localOptions['backend'])
def test_udp_traceroute(self):
- self.st.UDPTraceroute(self.localOptions['backend'])
- d = defer.Deferred()
- reactor.callLater(self.timeout, d.callback, self.st)
- return d
+ return self.st.UDPTraceroute(self.localOptions['backend'])
def postProcessor(self, measurements):
# should be called after all deferreds have calledback
@@ -68,19 +68,18 @@ class TracerouteTest(scapyt.BaseScapyTest):
self.report['answered_packets'] = self.st.matched_packets.items()
self.report['received_packets'] = self.st.received_packets.values()
- # display responses by hop:
- self.report['hops'] = {}
- for i in xrange(self.st.ttl_min, self.st.ttl_max):
- self.report['hops'][i] = []
- matchedPackets = filter(lambda x: x.ttl == i, self.st.matched_packets.keys())
- routers = {}
+ for ttl in xrange(self.st.ttl_min, self.st.ttl_max):
+ matchedPackets = filter(lambda x: x.ttl == ttl, self.st.matched_packets.keys())
for packet in matchedPackets:
- for pkt in self.st.matched_packets[packet]:
- router = pkt.src
- if router in routers:
- routers[router].append(pkt)
- else:
- routers[router] = [pkt]
- for router in routers.keys():
- self.report['hops'][i].append(router)
+ for response in self.st.matched_packets[packet]:
+ self.addToReport(packet, response)
return self.report
+
+ def addToReport(self, packet, response):
+ p = {6: 'tcp', 17: 'udp', 1: 'icmp'}
+ if packet.proto == 1:
+ self.report['test_icmp_traceroute']['hops'].append((packet.ttl, response.src))
+ elif packet.proto == 6:
+ self.report['test_tcp_traceroute']['hops_%s' % packet.dport].append((packet.ttl, response.src))
+ else:
+ self.report['test_udp_traceroute']['hops_%s' % packet.dport].append((packet.ttl, response.src))
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index 9e899c8..3e4e1b4 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -1,3 +1,4 @@
+import ipaddr
import struct
import socket
import os
@@ -104,6 +105,13 @@ def getNetworksFromRoutes():
class IfaceError(Exception):
pass
+def getAddresses():
+ from scapy.all import get_if_addr, get_if_list
+ from ipaddr import IPAddress
+ addresses = set([get_if_addr(i) for i in get_if_list()])
+ addresses.remove('0.0.0.0')
+ return [IPAddress(addr) for addr in addresses]
+
def getDefaultIface():
""" Return the default interface or raise IfaceError """
#XXX: currently broken on OpenVZ environments, because
@@ -288,40 +296,167 @@ class ScapySniffer(ScapyProtocol):
def packetReceived(self, packet):
self.pcapwriter.write(packet)
-class ScapyTraceroute(ScapyProtocol):
+class ParasiticTraceroute(ScapyProtocol):
+ def __init__(self):
+ self.numHosts = 7
+ self.rate = 15
+ self.hosts = {}
+ self.ttl_max = 15
+ self.ttl_min = 1
+ self.sent_packets = []
+ self.received_packets = []
+ self.matched_packets = {}
+ self.addresses = [str(x) for x in getAddresses()]
+
+ def sendPacket(self, packet):
+ self.factory.send(packet)
+
+ def packetReceived(self, packet):
+ try:
+ packet[IP]
+ except IndexError:
+ return
+
+ if isinstance(packet.getlayer(3), TCPerror):
+ self.received_packets.append(packet)
+ return
+
+ elif packet.dst in self.hosts:
+ if random.randint(1, 100) > self.rate:
+ return
+ try:
+ packet[IP].ttl = self.hosts[packet.dst]['ttl'].pop()
+ del packet.chksum #XXX Why is this incorrect?
+ log.debug("Sent packet to %s with ttl %d" % (packet.dst, packet.ttl))
+ self.sendPacket(packet)
+ k = (packet.id, packet[TCP].sport, packet[TCP].dport, packet[TCP].seq)
+ self.matched_packets[k] = {'ttl': packet.ttl}
+ return
+ except IndexError:
+ pass
+ return
+
+ def maxttl(packet=None):
+ if packet:
+ return min(self.ttl_max,
+ min(
+ abs( 64 - packet.ttl ),
+ abs( 128 - packet.ttl ),
+ abs( 256 - packet.ttl ))) - 1
+ else:
+ return self.ttl_max
+
+ def genttl(packet=None):
+ ttl = range(self.ttl_min, maxttl(packet))
+ random.shuffle(ttl)
+ return ttl
+
+ if len(self.hosts) < self.numHosts:
+ if packet.dst not in self.hosts \
+ and packet.dst not in self.addresses \
+ and isinstance(packet.getlayer(1), TCP):
+
+ self.hosts[packet.dst] = {'ttl' : genttl()}
+ log.debug("Tracing to %s" % packet.dst)
+
+ elif packet.src not in self.hosts \
+ and packet.src not in self.addresses \
+ and isinstance(packet.getlayer(1), TCP):
+
+ self.hosts[packet.src] = {'ttl' : genttl(packet),
+ 'ttl_max': maxttl(packet)}
+ log.debug("Tracing to %s" % packet.src)
+ return
+
+ elif packet.src in self.hosts and not 'ttl_max' in self.hosts[packet.src]:
+ self.hosts[packet.src]['ttl_max'] = ttl_max = maxttl(packet)
+ log.debug("set ttl_max to %d for host %s" % (ttl_max, packet.src))
+ ttl = []
+ for t in self.hosts[packet.src]['ttl']:
+ if t < ttl_max:
+ ttl.append(t)
+ self.hosts[packet.src]['ttl'] = ttl
+ return
+
+ def stopListening(self):
+ self.factory.unRegisterProtocol(self)
+
+class MPTraceroute(ScapyProtocol):
dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535]
ttl_min = 1
ttl_max = 30
def __init__(self):
self.sent_packets = []
+ self._recvbuf = []
self.received_packets = {}
self.matched_packets = {}
self.hosts = []
+ self.interval = 0.2
+ self.timeout = ((self.ttl_max - self.ttl_min) * len(self.dst_ports) * self.interval) + 5
+ self.numPackets = 1
def ICMPTraceroute(self, host):
if host not in self.hosts: self.hosts.append(host)
+
+ d = defer.Deferred()
+ reactor.callLater(self.timeout, d.callback, self)
+
self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/ICMP(id=RandShort()))
+ return d
def UDPTraceroute(self, host):
if host not in self.hosts: self.hosts.append(host)
+
+ d = defer.Deferred()
+ reactor.callLater(self.timeout, d.callback, self)
+
for dst_port in self.dst_ports:
self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/UDP(dport=dst_port, sport=RandShort()))
+ return d
def TCPTraceroute(self, host):
if host not in self.hosts: self.hosts.append(host)
+
+ d = defer.Deferred()
+ reactor.callLater(self.timeout, d.callback, self)
+
for dst_port in self.dst_ports:
self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/TCP(flags=2L, dport=dst_port, sport=RandShort(), seq=RandShort()))
+ return d
+ @defer.inlineCallbacks
def sendPackets(self, packets):
- #if random.randint(0,1):
- # random.shuffle(packets)
+ def sleep(seconds):
+ d = defer.Deferred()
+ reactor.callLater(seconds, d.callback, seconds)
+ return d
+
+ if not isinstance(packets, Gen):
+ packets = SetGen(packets)
+
for packet in packets:
- self.sent_packets.append(packet)
- self.factory.super_socket.send(packet)
+ for i in xrange(self.numPackets):
+ self.sent_packets.append(packet)
+ self.factory.super_socket.send(packet)
+ yield sleep(self.interval)
def matchResponses(self):
- def _pe(k, p):
+ def addToReceivedPackets(key, packet):
+ """
+ Add a packet into the received packets dictionary,
+ typically the key is a tuple of packet fields used
+ to correlate sent packets with recieved packets.
+ """
+
+ # Initialize or append to the lists of packets
+ # with the same key
+ if k in self.received_packets:
+ self.received_packets[key].append(packet)
+ else:
+ self.received_packets[key] = [pcket]
+
+ def matchResponse(k, p):
if k in self.received_packets:
if p in self.matched_packets:
log.debug("Matched sent packet to more than one response!")
@@ -332,50 +467,47 @@ class ScapyTraceroute(ScapyProtocol):
return 1
return 0
+ for p in self._recvbuf:
+ l = p.getlayer(2)
+ if isinstance(l, IPerror):
+ pid = l.id
+ l = p.getlayer(3)
+ if isinstance(l, ICMPerror):
+ addToReceivedPackets(('icmp', pid), p)
+ elif isinstance(l, TCPerror):
+ addToReceivedPackets(('tcp', pid), p)
+ elif isinstance(l, UDPerror):
+ addToReceivedPackets(('udp', pid), p)
+ elif p.src in self.hosts:
+ l = p.getlayer(1)
+ if isinstance(l, ICMP):
+ addToReceivedPackets(('icmp', l.id), p)
+ elif isinstance(l, TCP):
+ addToReceivedPackets(('tcp', l.ack - 1, l.dport, l.sport), p)
+ elif isinstance(l, UDP):
+ addToReceivedPackets(('udp', l.dport, l.sport), p)
+
for p in self.sent_packets:
# for each sent packet, find corresponding
# received packets
l = p.getlayer(1)
i = 0
if isinstance(l, ICMP):
- i += _pe((ICMP, p.id), p) # match by ipid
- i += _pe((ICMP, l.id), p) # match by icmpid
+ i += matchResponse(('icmp', p.id), p) # match by ipid
+ i += matchResponse(('icmp', l.id), p) # match by icmpid
if isinstance(l, TCP):
- i += _pe((TCP, p.id), p) # match by ipid
- i += _pe((TCP, p.id, l.seq, l.ack, l.sport, l.dport), p)
+ i += matchResponse(('tcp', p.id), p) # match by ipid
+ i += matchResponse(('tcp', l.seq, l.sport, l.dport), p)
if isinstance(l, UDP):
- i += _pe((UDP, p.id), p)
+ i += matchResponse(('udp', p.id), p)
+ i += matchResponse(('udp', l.sport, l.dport), p)
if i == 0:
log.debug("No response for packet %s" % [p])
- def packetReceived(self, packet):
- def _ae(k, p):
- if k in self.received_packets:
- self.received_packets[k].append(p)
- else:
- self.received_packets[k] = [p]
+ del self._recvbuf
- l = packet.getlayer(2)
- try:
- if isinstance(l, IPerror):
- pid = l.id
- l = packet.getlayer(3)
- if isinstance(l, ICMPerror):
- _ae((ICMP, pid), packet)
- elif isinstance(l, TCPerror):
- _ae((TCP, pid, l.seq, l.ack, l.sport, l.dport), packet)
- elif isinstance(l, UDPerror):
- _ae((UDP, pid), packet)
- elif packet.src in self.hosts:
- l = packet.getlayer(1)
- if isinstance(l, ICMP):
- _ae((ICMP, l.id), packet)
- elif isinstance(l, TCP):
- _ae((TCP, l.seq, l.ack, l.sport, l.dport), packet)
- elif isinstance(l, UDP):
- _ae((UDP, l.sport, l.dport), packet)
- except Exception, e:
- import pdb;pdb.set_trace()
+ def packetReceived(self, packet):
+ self._recvbuf.append(packet)
def stopListening(self):
self.factory.unRegisterProtocol(self)
