commit c44cc83cbefa9fcd6ba81bb18a6e52a1762f935f Author: Arturo Filastò <art@fuffa.org> Date: Sat Nov 24 21:32:44 2012 +0100 Do some important refactoring of scapy related functionality * Move functions for detecting the default network interface into txscapy * Make sr1 follow the syntax of scapy * Add notes as to why the parasitic traceroute test does not work --- nettests/bridge_reachability/echo.py | 4 +- nettests/core/dnsspoof.py | 4 +- nettests/core/dnstamper.py | 12 ++-- nettests/core/parasitictraceroute.py | 117 ++++++++++++++++++---------------- nettests/core/tcpconnect.py | 2 +- ooni/templates/scapyt.py | 65 +++++++++++++------ ooni/utils/net.py | 42 +------------ ooni/utils/txscapy.py | 75 +++++++++++++--------- ooniprobe.conf | 4 + 9 files changed, 166 insertions(+), 159 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 40436e7..d4033dd 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -18,7 +18,7 @@ import sys from twisted.python import usage from twisted.internet import reactor, defer from ooni import nettest -from ooni.utils import log, net, Storage +from ooni.utils import log, net, Storage, txscapy try: from scapy.all import IP, ICMP @@ -66,7 +66,7 @@ class EchoTest(nettest.NetTestCase): if not self.interface: try: - iface = net.getDefaultIface() + iface = txscapy.getDefaultIface() except Exception, e: log.msg("No network interface specified!") log.err(e) diff --git a/nettests/core/dnsspoof.py b/nettests/core/dnsspoof.py index 8f3e6a2..48991ea 100644 --- a/nettests/core/dnsspoof.py +++ b/nettests/core/dnsspoof.py @@ -51,7 +51,7 @@ class DNSSpoof(scapyt.ScapyTest): question = IP(dst=self.resolverAddr)/UDP()/DNS(rd=1, qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) log.msg("Performing query to %s with %s:%s" % (self.hostname, self.resolverAddr, self.resolverPort)) - answered, unanswered = yield self.sr1(question) + yield self.sr1(question) @defer.inlineCallbacks def test_control_a_lookup(self): @@ -59,6 +59,6 @@ class DNSSpoof(scapyt.ScapyTest): qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) log.msg("Performing query to %s with %s:%s" % (self.hostname, self.controlResolverAddr, self.controlResolverPort)) - answered, unanswered = yield self.sr1(question) + yield self.sr1(question) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index d007d77..967ae26 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -28,8 +28,10 @@ from ooni.utils import log class UsageOptions(usage.Options): optParameters = [['backend', 'b', '8.8.8.8:53', 'The OONI backend that runs the DNS resolver'], - ['testresolvers', 't', None, - 'file containing list of DNS resolvers to test against'] + ['testresolvers', 'T', None, + 'File containing list of DNS resolvers to test against'], + ['testresolver', 't', None, + 'Specify a single test resolver to use for testing'] ] class DNSTamperTest(dnst.DNSTest): @@ -44,12 +46,10 @@ class DNSTamperTest(dnst.DNSTest): 'Input file of list of hostnames to attempt to resolve'] usageOptions = UsageOptions - requiredOptions = ['backend', 'file', 'testresolvers'] + requiredOptions = ['backend', 'file'] def setUp(self): - if not self.localOptions['testresolvers']: - self.test_resolvers = ['8.8.8.8'] raise usage.UsageError("You did not specify a file of DNS servers to test!" "See the '--testresolvers' option.") @@ -68,7 +68,7 @@ class DNSTamperTest(dnst.DNSTest): self.report['control_resolver'] = self.control_dns_server @defer.inlineCallbacks - def test_a_queries(self): + def test_a_lookup(self): """ We perform an A lookup on the DNS test servers for the domains to be tested and an A lookup on the known good DNS server. diff --git a/nettests/core/parasitictraceroute.py b/nettests/core/parasitictraceroute.py index 8ea27bc..631c24b 100644 --- a/nettests/core/parasitictraceroute.py +++ b/nettests/core/parasitictraceroute.py @@ -13,14 +13,12 @@ from scapy.all import * from ooni.utils import log class UsageOptions(usage.Options): - optParameters = [['backend', 'b', '8.8.8.8', 'Test backend to use'], + optParameters = [['backend', 'b', 'google.com', 'Test backend to use'], ['timeout', 't', 5, 'The timeout for the traceroute test'], - ['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'], + ['maxttl', 'm', 64, 'The maximum value of ttl to set on packets'], ['dstport', 'd', 80, 'Set the destination port of the traceroute test'], ['srcport', 'p', None, 'Set the source port to a specific value']] - optFlags = [['randomize','r', 'Randomize the source port']] - class ParasiticalTracerouteTest(scapyt.BaseScapyTest): name = "Parasitic TCP Traceroute Test" author = "Arturo Filastò" @@ -32,58 +30,57 @@ class ParasiticalTracerouteTest(scapyt.BaseScapyTest): def get_sport(): if self.localOptions['srcport']: return int(self.localOptions['srcport']) - elif self.localOptions['randomize']: - return random.randint(1024, 65535) else: - return 80 - + return random.randint(1024, 65535) self.get_sport = get_sport - self.dport = int(self.localOptions['dstport']) - def max_ttl_and_timeout(self): - max_ttl = int(self.localOptions['maxttl']) - timeout = int(self.localOptions['timeout']) - self.report['max_ttl'] = max_ttl - self.report['timeout'] = timeout - return max_ttl, timeout + self.dst_ip = socket.gethostbyaddr(self.localOptions['backend'])[2][0] + + self.dport = int(self.localOptions['dstport']) + self.max_ttl = int(self.localOptions['maxttl']) @defer.inlineCallbacks def test_parasitic_tcp_traceroute(self): """ - Establishes a TCP stream and send the packets inside of such stream. - Requires the backend to respond with an ACK to our SYN packet. - """ - max_ttl, timeout = self.max_ttl_and_timeout() + Establishes a TCP stream, then sequentially sends TCP packets with + increasing TTL until we reach the ttl of the destination. + + Requires the backend to respond with an ACK to our SYN packet (i.e. + the port must be open) + XXX this currently does not work properly. The problem lies in the fact + that we are currently using the scapy layer 3 socket. This socket makes + packets received be trapped by the kernel TCP stack, therefore when we + send out a SYN and get back a SYN-ACK the kernel stack will reply with + a RST because it did not send a SYN. + + The quick fix to this would be to establish a TCP stream using socket + calls and then "cannibalizing" the TCP session with scapy. + + The real fix is to make scapy use libpcap instead of raw sockets + obviously as we previously did... arg. + """ sport = self.get_sport() dport = self.dport ipid = int(RandShort()) - packet = IP(dst=self.localOptions['backend'], ttl=max_ttl, - id=ipid)/TCP(sport=sport, dport=dport, - flags="S", seq=0) + ip_layer = IP(dst=self.dst_ip, + id=ipid, ttl=self.max_ttl) - log.msg("Sending SYN towards %s" % dport) + syn = ip_layer/TCP(sport=sport, dport=dport, flags="S", seq=0) - try: - answered, unanswered = yield self.sr(packet, timeout=timeout) - except Exception, e: - log.exception(e) - except: - log.exception() + log.msg("Sending...") + syn.show2() - try: - snd, rcv = answered[0] - synack = rcv[0] + synack = yield self.sr1(syn) - except IndexError: - print answered, unanswered + log.msg("Got response...") + synack.show2() + + if not synack: log.err("Got no response. Try increasing max_ttl") return - except Exception, e: - log.exception(e) - if synack[TCP].flags == 11: log.msg("Got back a FIN ACK. The destination port is closed") return @@ -92,33 +89,41 @@ class ParasiticalTracerouteTest(scapyt.BaseScapyTest): log.msg("Got a SYN ACK. All is well.") else: log.err("Got an unexpected result") + return - self.report['hops'] = [] - for ttl in range(1, max_ttl): - log.msg("Sending ACK with ttl %s" % ttl) - # We generate an ack for the syn-ack we got with increasing ttl - packet = IP(dst=self.localOptions['backend'], - ttl=ttl, id=ipid)/TCP(sport=synack.dport, + ack = ip_layer/TCP(sport=synack.dport, dport=dport, flags="A", seq=synack.ack, ack=synack.seq + 1) - answered, unanswered = yield self.sr(packet, timeout=timeout) - try: - snd, rcv = answered[0] - except IndexError: - log.err("Got no response.") + yield self.send(ack) - try: - icmp = rcv[ICMP] + self.report['hops'] = [] + # For the time being we make the assumption that we are NATted and + # that the NAT will forward the packet to the destination even if the TTL has + for ttl in range(1, self.max_ttl): + log.msg("Sending packet with ttl of %s" % ttl) + ip_layer.ttl = ttl + empty_tcp_packet = ip_layer/TCP(sport=synack.dport, + dport=dport, flags="A", + seq=synack.ack, ack=synack.seq + 1) + + answer = yield self.sr1(empty_tcp_packet) + if not answer: + log.err("Got no response for ttl %s" % ttl) + continue - except IndexError: - report = {'ttl': snd.ttl, - 'address': rcv.src, - 'rtt': rcv.time - snd.time + try: + icmp = answer[ICMP] + report = {'ttl': empty_tcp_packet.ttl, + 'address': answer.src, + 'rtt': answer.time - empty_tcp_packet.time } - log.debug("%s: %s" % (dport, report)) + log.msg("%s: %s" % (dport, report)) self.report['hops'].append(report) - if rcv.src == self.localOptions['backend']: + + except IndexError: + if answer.src == self.dst_ip: + answer.show() log.msg("Reached the destination. We have finished the traceroute") return diff --git a/nettests/core/tcpconnect.py b/nettests/core/tcpconnect.py index b763bf8..d0a53f8 100644 --- a/nettests/core/tcpconnect.py +++ b/nettests/core/tcpconnect.py @@ -16,10 +16,10 @@ class TCPConnectTest(nettest.NetTestCase): name = "TCP Connect" author = "Arturo Filastò" version = "0.1" - inputFile = ['file', 'f', None, 'File containing the IP:PORT combinations to be tested, one per line'] + requiredOptions = ['file'] def test_connect(self): """ This test performs a TCP connection to the remote host on the specified port. diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index d1ffb36..11b4381 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -12,13 +12,11 @@ from twisted.internet import protocol, defer, threads from scapy.all import send, sr, IP, TCP, config from ooni.reporter import createPacketReport - from ooni.nettest import NetTestCase from ooni.utils import log - from ooni import config -from ooni.utils.txscapy import ScapyProtocol +from ooni.utils.txscapy import ScapyProtocol, getDefaultIface class BaseScapyTest(NetTestCase): """ @@ -35,9 +33,12 @@ class BaseScapyTest(NetTestCase): requiresRoot = True baseFlags = [ - ['ipsrc', 's', 'Does *not* check if IP src and ICMP IP citation matches when processing answers'], - ['seqack', 'k', 'Check if TCP sequence number and ACK match in the ICMP citation when processing answers'], - ['ipid', 'i', 'Check if the IPID matches when processing answers'] + ['ipsrc', 's', + 'Does *not* check if IP src and ICMP IP citation matches when processing answers'], + ['seqack', 'k', + 'Check if TCP sequence number and ACK match in the ICMP citation when processing answers'], + ['ipid', 'i', + 'Check if the IPID matches when processing answers'] ] def _setUp(self): @@ -65,15 +66,26 @@ class BaseScapyTest(NetTestCase): else: config.check_TCPerror_seqack = 0 + if config.advanced.interface == 'auto': + self.interface = getDefaultIface() + else: + self.interface = config.advanced.interface + + def reportSentPacket(self, packet): + if 'sent_packets' not in self.report: + self.report['sent_packets'] = [] + self.report['sent_packets'].append(packet) + + def reportReceivedPacket(self, packet): + if 'answered_packets' not in self.report: + self.report['answered_packets'] = [] + self.report['answered_packets'].append(packet) + def finishedSendReceive(self, packets): """ This gets called when all packets have been sent and received. """ answered, unanswered = packets - if 'answered_packets' not in self.report: - self.report['answered_packets'] = [] - if 'sent_packets' not in self.report: - self.report['sent_packets'] = [] for snd, rcv in answered: log.debug("Writing report for scapy test") @@ -86,11 +98,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - #pkt_report_r = createPacketReport(received_packet) - #pkt_report_s = createPacketReport(sent_packet) - self.report['answered_packets'].append(received_packet) - self.report['sent_packets'].append(sent_packet) - log.debug("Done") + self.reportSentPacket(sent_packet) + self.reportReceivedPacket(received_packet) return packets def sr(self, packets, *arg, **kw): @@ -98,25 +107,41 @@ class BaseScapyTest(NetTestCase): Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - scapyProtocol = ScapyProtocol(*arg, **kw) + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) d = scapyProtocol.startSending(packets) d.addCallback(self.finishedSendReceive) return d def sr1(self, packets, *arg, **kw): - scapyProtocol = ScapyProtocol(*arg, **kw) + def done(packets): + """ + We do this so that the returned value is only the one packet that + we expected a response for, identical to the scapy implementation + of sr1. + """ + try: + return packets[0][0][1] + except IndexError: + log.err("Got no response...") + return None + + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) scapyProtocol.expected_answers = 1 log.debug("Running sr1") d = scapyProtocol.startSending(packets) log.debug("Started to send") d.addCallback(self.finishedSendReceive) + d.addCallback(done) return d - def send(self, pkts, *arg, **kw): + def send(self, packets, *arg, **kw): """ Wrapper around scapy.sendrecv.send for sending of packets at layer 3 """ - raise Exception("Not implemented") + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) + scapyProtocol.sendPackets(packets) + scapyProtocol.stopSending() + for packet in packets: + self.reportSentPacket(packet) ScapyTest = BaseScapyTest - diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 155abd2..e828977 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -22,13 +22,6 @@ from ooni.utils import log, txscapy #if sys.platform.system() == 'Windows': # import _winreg as winreg -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} userAgents = [ ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), @@ -44,7 +37,6 @@ userAgents = [ ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP") ] - class UnsupportedPlatform(Exception): """Support for this platform is not currently available.""" @@ -54,7 +46,6 @@ class IfaceError(Exception): class PermissionsError(SystemExit): """This test requires admin or root privileges to run. Exiting...""" - PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'OPENBSD': sys.platform.startswith("openbsd"), 'FREEBSD': sys.platform.startswith("freebsd"), @@ -63,14 +54,6 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} -class UnsupportedPlatform(Exception): - """Support for this platform is not currently available.""" - -class IfaceError(Exception): - """Could not find default network interface.""" - -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" class StringProducer(object): implements(IBodyProducer) @@ -128,7 +111,6 @@ def getSystemResolver(): XXX implement a function that returns the resolver that is currently default on the system. """ - pass def getClientPlatform(platform_name=None): for name, test in PLATFORMS.items(): @@ -225,30 +207,8 @@ def getNonLoopbackIfaces(platform_name=None): else: return interfaces -def getNetworksFromRoutes(): - from scapy.all import conf, ltoa, read_routes - from ipaddr import IPNetwork, IPAddress - - ## Hide the 'no routes' warnings - conf.verb = 0 - - networks = [] - for nw, nm, gw, iface, addr in read_routes(): - n = IPNetwork( ltoa(nw) ) - (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] - n.iface = iface - if not n.compressed in networks: - networks.append(n) - - return networks - -def getDefaultIface(): - networks = getNetworksFromRoutes() - for net in networks: - if net.is_private: - return net.iface - raise IfaceError def getLocalAddress(): default_iface = getDefaultIface() return default_iface.ipaddr + diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 4341f68..e41d649 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -17,28 +17,59 @@ from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract from zope.interface import implements - from scapy.all import PcapWriter, MTU from scapy.all import BasePacketList, conf, PcapReader from scapy.all import conf, Gen, SetGen +from scapy.arch import pcapdnet from ooni.utils import log +conf.use_pcap = True +conf.use_dnet = True + +def getNetworksFromRoutes(): + from scapy.all import conf, ltoa, read_routes + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] + for nw, nm, gw, iface, addr in read_routes(): + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: + return net.iface + raise IfaceError + class TXPcapWriter(PcapWriter): def __init__(self, *arg, **kw): PcapWriter.__init__(self, *arg, **kw) fdesc.setNonBlocking(self.f) class ScapyProtocol(abstract.FileDescriptor): - def __init__(self, super_socket=None, - reactor=None, timeout=4, receive=True): + def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) # By default we use the conf.L3socket if not super_socket: - super_socket = conf.L3socket() + super_socket = pcapdnet.L3dnetSocket(iface=interface) + print super_socket + log.msg("Creating layer 3 socket with interface %s" % interface) + + #fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket + self.interface = interface self.timeout = timeout # This dict is used to store the unique hashes that allow scapy to @@ -53,13 +84,8 @@ class ScapyProtocol(abstract.FileDescriptor): # This deferred will fire when we have finished sending a receiving packets. self.d = defer.Deferred() - self.debug = False + # Should we look for multiple answers for the same sent packet? self.multi = False - # XXX this needs to be implemented. It would involve keeping track of - # the state of the sending via the super socket file descriptor and - # firing the callback when we have concluded sending. Check out - # twisted.internet.udp to see how this is done. - self.receive = receive # When 0 we stop when all the packets we have sent have received an # answer @@ -85,7 +111,6 @@ class ScapyProtocol(abstract.FileDescriptor): if len(self.answered_packets) == len(self.sent_packets): log.debug("All of our questions have been answered.") - log.debug("%s" % self.__hash__) self.stopSending() return @@ -94,12 +119,11 @@ class ScapyProtocol(abstract.FileDescriptor): log.debug("Got the number of expected answers") self.stopSending() - def doRead(self): timeout = time.time() - self._start_time if self.timeout and time.time() - self._start_time > self.timeout: self.stopSending() - packet = self.super_socket.recv() + packet = self.super_socket.recv(MTU) if packet: self.processPacket(packet) # A string that has the same value for the request than for the @@ -110,7 +134,6 @@ class ScapyProtocol(abstract.FileDescriptor): self.processAnswer(packet, answer_hr) def stopSending(self): - log.debug("Stopping sending") self.stopReading() self.super_socket.close() if hasattr(self, "d"): @@ -120,20 +143,20 @@ class ScapyProtocol(abstract.FileDescriptor): def write(self, packet): """ - Write a scapy packet to the wire + Write a scapy packet to the wire. """ - hashret = packet.hashret() - if hashret in self.hr_sent_packets: - self.hr_sent_packets[hashret].append(packet) - else: - self.hr_sent_packets[hashret] = [packet] - self.sent_packets.append(packet) return self.super_socket.send(packet) def sendPackets(self, packets): if not isinstance(packets, Gen): packets = SetGen(packets) for packet in packets: + hashret = packet.hashret() + if hashret in self.hr_sent_packets: + self.hr_sent_packets[hashret].append(packet) + else: + self.hr_sent_packets[hashret] = [packet] + self.sent_packets.append(packet) self.write(packet) def startSending(self, packets): @@ -142,14 +165,4 @@ class ScapyProtocol(abstract.FileDescriptor): self.sendPackets(packets) return self.d -def sr(x, filter=None, iface=None, nofilter=0, timeout=None): - super_socket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - sp = ScapyProtocol(super_socket=super_socket, timeout=timeout) - return sp.startSending(x) - -def send(x, filter=None, iface=None, nofilter=0, timeout=None): - super_socket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - sp = ScapyProtocol(super_socket=super_socket, timeout=timeout) - return sp.startSending(x) - diff --git a/ooniprobe.conf b/ooniprobe.conf index 1998c93..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -25,4 +25,8 @@ advanced: debug: true threadpool_size: 10 tor_socksport: 9050 + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0