commit deb507f97825bba597478053d9fc130a4f8f2df3 Author: aagbsn <aagbsn@extc.org> Date: Mon Jan 20 22:46:23 2014 +0000 Add a ScapyTraceroute ScapyProtocol This protocol handles creating and matching packets for conducting ICMP, TCP, and UDP multi-port traceroute. --- ooni/nettests/manipulation/traceroute.py | 149 ++++++++++-------------------- ooni/reporter.py | 2 +- ooni/utils/txscapy.py | 94 +++++++++++++++++++ 3 files changed, 145 insertions(+), 100 deletions(-) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index 7639060..b970127 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -4,13 +4,16 @@ # :licence: see LICENSE from twisted.python import usage -from twisted.internet import defer +from twisted.internet import defer, reactor from ooni.templates import scapyt +from itertools import chain from scapy.all import * from ooni.utils import log +from ooni.utils.txscapy import ScapyTraceroute +from ooni.settings import config class UsageOptions(usage.Options): optParameters = [ @@ -23,113 +26,61 @@ class UsageOptions(usage.Options): class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" description = "Performs a UDP, TCP, ICMP traceroute with destination port number set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535" - author = "Arturo Filastò" - version = "0.2" - requiredTestHelpers = {'backend': 'traceroute'} usageOptions = UsageOptions dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535] + timeout = 5 def setUp(self): - def get_sport(protocol): - if self.localOptions['srcport']: - return int(self.localOptions['srcport']) - else: - return random.randint(1024, 65535) - - self.get_sport = get_sport - self.report['test_tcp_traceroute'] = {} - self.report['test_udp_traceroute'] = {} - self.report['test_icmp_traceroute'] = {} + self.st = ScapyTraceroute() + if self.localOptions['maxttl']: + self.st.ttl_max = int(self.localOptions['maxttl']) + config.scapyFactory.registerProtocol(self.st) + self.done = defer.Deferred() + self.tcp = self.udp = self.icmp = None - def max_ttl_and_timeout(self): - max_ttl = int(self.localOptions['maxttl']) - timeout = int(self.localOptions['timeout']) - self.report['max_ttl'] = max_ttl - self.report['timeout'] = timeout - return max_ttl, timeout + def test_icmp_traceroute(self): + self.st.ICMPTraceroute(self.localOptions['backend']) + d = defer.Deferred() + reactor.callLater(self.timeout, d.callback, self.st) + return d def test_tcp_traceroute(self): - """ - Does a traceroute to the destination by sending TCP SYN packets - with TTLs from 1 until max_ttl. - """ - def finished(packets, port): - log.msg("Finished running TCP traceroute test on port %s" % port) - answered, unanswered = packets - self.report['test_tcp_traceroute']['hops_'+str(port)] = [] - for snd, rcv in answered: - report = {'ttl': snd.ttl, - 'address': rcv.src, - 'rtt': rcv.time - snd.time, - 'sport': snd[TCP].sport - } - log.msg("%s: %s" % (port, report)) - self.report['test_tcp_traceroute']['hops_'+str(port)].append(report) - - dl = [] - max_ttl, timeout = self.max_ttl_and_timeout() - for port in self.dst_ports: - packets = IP(dst=self.localOptions['backend'], - ttl=(1,max_ttl),id=RandShort())/TCP(flags=0x2, dport=port, - sport=self.get_sport('tcp')) - - d = self.sr(packets, timeout=timeout) - d.addCallback(finished, port) - dl.append(d) - return defer.DeferredList(dl) + self.st.TCPTraceroute(self.localOptions['backend']) + d = defer.Deferred() + reactor.callLater(self.timeout, d.callback, self.st) + return d def test_udp_traceroute(self): - """ - Does a traceroute to the destination by sending UDP packets with empty - payloads with TTLs from 1 until max_ttl. - """ - def finished(packets, port): - log.msg("Finished running UDP traceroute test on port %s" % port) - answered, unanswered = packets - self.report['test_udp_traceroute']['hops_'+str(port)] = [] - for snd, rcv in answered: - report = {'ttl': snd.ttl, - 'address': rcv.src, - 'rtt': rcv.time - snd.time, - 'sport': snd[UDP].sport - } - log.msg("%s: %s" % (port, report)) - self.report['test_udp_traceroute']['hops_'+str(port)].append(report) - dl = [] - max_ttl, timeout = self.max_ttl_and_timeout() - for port in self.dst_ports: - packets = IP(dst=self.localOptions['backend'], - ttl=(1,max_ttl),id=RandShort())/UDP(dport=port, - sport=self.get_sport('udp')) - - d = self.sr(packets, timeout=timeout) - d.addCallback(finished, port) - dl.append(d) - return defer.DeferredList(dl) - - def test_icmp_traceroute(self): - """ - Does a traceroute to the destination by sending ICMP echo request - packets with TTLs from 1 until max_ttl. - """ - def finished(packets): - log.msg("Finished running ICMP traceroute test") - answered, unanswered = packets - self.report['test_icmp_traceroute']['hops'] = [] - for snd, rcv in answered: - report = {'ttl': snd.ttl, - 'address': rcv.src, - 'rtt': rcv.time - snd.time - } - log.msg("%s" % (report)) - self.report['test_icmp_traceroute']['hops'].append(report) - dl = [] - max_ttl, timeout = self.max_ttl_and_timeout() - packets = IP(dst=self.localOptions['backend'], - ttl=(1,max_ttl), id=RandShort())/ICMP() - - d = self.sr(packets, timeout=timeout) - d.addCallback(finished) + self.st.UDPTraceroute(self.localOptions['backend']) + d = defer.Deferred() + reactor.callLater(self.timeout, d.callback, self.st) return d + def postProcessor(self, measurements): + # should be called after all deferreds have calledback + self.st.stopListening() + self.st.matchResponses() + + if measurements[0][1].result == self.st: + for packet in self.st.sent_packets: + self.report['sent_packets'].append(packet) + self.report['answered_packets'] = self.st.matched_packets.items() + self.report['received_packets'] = self.st.received_packets.values() + + # display responses by hop: + self.report['hops'] = {} + for i in xrange(self.st.ttl_min, self.st.ttl_max): + self.report['hops'][i] = [] + matchedPackets = filter(lambda x: x.ttl == i, self.st.matched_packets.keys()) + routers = {} + for packet in matchedPackets: + for pkt in self.st.matched_packets[packet]: + router = pkt.src + if router in routers: + routers[router].append(pkt) + else: + routers[router] = [pkt] + for router in routers.keys(): + self.report['hops'][i].append(router) + return self.report diff --git a/ooni/reporter.py b/ooni/reporter.py index 545997d..e976e5c 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -50,7 +50,7 @@ def createPacketReport(packet_list): report = [] for packet in packet_list: report.append({'raw_packet': str(packet), - 'summary': str(packet.summary())}) + 'summary': str([packet])}) return report class OSafeRepresenter(SafeRepresenter): diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index f9b7dfd..9e899c8 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -3,6 +3,7 @@ import socket import os import sys import time +import random from twisted.internet import protocol, base, fdesc from twisted.internet import reactor, threads, error @@ -11,6 +12,8 @@ from zope.interface import implements from scapy.config import conf from scapy.supersocket import L3RawSocket +from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror +from scapy.all import TCP, TCPerror, UDP, UDPerror from ooni.utils import log from ooni.settings import config @@ -285,3 +288,94 @@ class ScapySniffer(ScapyProtocol): def packetReceived(self, packet): self.pcapwriter.write(packet) +class ScapyTraceroute(ScapyProtocol): + dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535] + ttl_min = 1 + ttl_max = 30 + + def __init__(self): + self.sent_packets = [] + self.received_packets = {} + self.matched_packets = {} + self.hosts = [] + + def ICMPTraceroute(self, host): + if host not in self.hosts: self.hosts.append(host) + self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/ICMP(id=RandShort())) + + def UDPTraceroute(self, host): + if host not in self.hosts: self.hosts.append(host) + for dst_port in self.dst_ports: + self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/UDP(dport=dst_port, sport=RandShort())) + + def TCPTraceroute(self, host): + if host not in self.hosts: self.hosts.append(host) + for dst_port in self.dst_ports: + self.sendPackets(IP(dst=host,ttl=(self.ttl_min,self.ttl_max), id=RandShort())/TCP(flags=2L, dport=dst_port, sport=RandShort(), seq=RandShort())) + + def sendPackets(self, packets): + #if random.randint(0,1): + # random.shuffle(packets) + for packet in packets: + self.sent_packets.append(packet) + self.factory.super_socket.send(packet) + + def matchResponses(self): + def _pe(k, p): + if k in self.received_packets: + if p in self.matched_packets: + log.debug("Matched sent packet to more than one response!") + self.matched_packets[p].extend(self.received_packets[k]) + else: + self.matched_packets[p] = self.received_packets[k] + log.debug("Packet %s matched %s" % ([p], self.received_packets[k])) + return 1 + return 0 + + for p in self.sent_packets: + # for each sent packet, find corresponding + # received packets + l = p.getlayer(1) + i = 0 + if isinstance(l, ICMP): + i += _pe((ICMP, p.id), p) # match by ipid + i += _pe((ICMP, l.id), p) # match by icmpid + if isinstance(l, TCP): + i += _pe((TCP, p.id), p) # match by ipid + i += _pe((TCP, p.id, l.seq, l.ack, l.sport, l.dport), p) + if isinstance(l, UDP): + i += _pe((UDP, p.id), p) + if i == 0: + log.debug("No response for packet %s" % [p]) + + def packetReceived(self, packet): + def _ae(k, p): + if k in self.received_packets: + self.received_packets[k].append(p) + else: + self.received_packets[k] = [p] + + l = packet.getlayer(2) + try: + if isinstance(l, IPerror): + pid = l.id + l = packet.getlayer(3) + if isinstance(l, ICMPerror): + _ae((ICMP, pid), packet) + elif isinstance(l, TCPerror): + _ae((TCP, pid, l.seq, l.ack, l.sport, l.dport), packet) + elif isinstance(l, UDPerror): + _ae((UDP, pid), packet) + elif packet.src in self.hosts: + l = packet.getlayer(1) + if isinstance(l, ICMP): + _ae((ICMP, l.id), packet) + elif isinstance(l, TCP): + _ae((TCP, l.seq, l.ack, l.sport, l.dport), packet) + elif isinstance(l, UDP): + _ae((UDP, l.sport, l.dport), packet) + except Exception, e: + import pdb;pdb.set_trace() + + def stopListening(self): + self.factory.unRegisterProtocol(self)