commit 1008bd5c04e9a9fa1f1726fa5474725d2714f37c Merge: 870d882 2bf3da5 Author: Isis Lovecruft isis@torproject.org Date: Thu Nov 15 00:26:43 2012 +0000

Merge branch 'master' of git-rw.torproject.org:ooni-probe

Conflicts: ooni/utils/net.py

AUTHORS | 2 +- CHANGES.yaml | 5 - HACKING | 141 ++++++------ INSTALL | 1 - README.md | 123 +++++------ TODO | 105 +-------- before_i_commit.sh | 2 +- bin/oonib | 41 +++- bin/ooniprobe | 5 +- docs/source/index.rst | 51 +---- docs/source/install.rst | 119 ---------- nettests/bridge_reachability/echo.py | 2 +- nettests/core/captiveportal.py | 23 +- nettests/core/chinatrigger.py | 9 +- nettests/core/daphn3.py | 117 +++++++++ nettests/core/http_host.py | 74 +++++- nettests/core/http_requests.py | 118 +++++++++ nettests/core/http_uk_mobile_networks.py | 85 +++++++ nettests/core/keyword_filtering.py | 52 ++++ nettests/core/squid.py | 2 +- nettests/core/tcpconnect.py | 46 ++++ nettests/core/traceroute.py | 113 +++++++++ nettests/core/url_list.py | 59 ++++- nettests/examples/example_scapyt.py | 22 ++- nettests/simpletest.py | 27 -- ooni/__init__.py | 4 + ooni/config.py | 20 ++- ooni/inputunit.py | 19 +- ooni/kit/daphn3.py | 208 ++++++++++++++++ ooni/lib/rfc3339.py | 283 ---------------------- ooni/lib/txscapy.py | 381 ------------------------------ ooni/nettest.py | 38 ++-- ooni/nodes.py | 24 +- ooni/ooni-probe.conf | 100 -------- ooni/oonicli.py | 67 ++++-- ooni/protocols/daphn3.py | 311 ------------------------ ooni/reporter.py | 340 +++++++++++++++------------ ooni/runner.py | 117 ++++++---- ooni/templates/httpt.py | 120 ++++------ ooni/templates/scapyt.py | 124 +++-------- ooni/utils/__init__.py | 102 +-------- ooni/utils/date.py | 30 --- ooni/utils/geodata.py | 31 ++- ooni/utils/hacks.py | 75 ++++--- ooni/utils/log.py | 26 ++- ooni/utils/net.py | 146 ++++++++---- ooni/utils/otime.py | 8 + ooni/utils/txscapy.py | 140 +++++++++++ oonib/__init__.py | 22 ++ oonib/config.py | 43 ++++ oonib/db/__init__.py | 30 +++ oonib/db/tables.py | 123 ++++++++++ oonib/lib/__init__.py | 8 - oonib/lib/ssl.py | 8 - oonib/models.py | 122 ++++++++++ oonib/oonibackend.conf.sample | 12 - oonib/oonibackend.py | 117 +++++----- oonib/report/api.py | 164 ++++++++++++- oonib/report/db/__init__.py | 33 --- oonib/requirements.txt | 24 ++ oonib/runner.py | 25 ++ oonib/testhelpers/__init__.py | 5 + oonib/testhelpers/daphn3.py | 47 ---- oonib/testhelpers/dns.py | 16 -- oonib/testhelpers/dns_helpers.py | 16 ++ oonib/testhelpers/http_helpers.py | 80 +++++++ oonib/testhelpers/httph.py | 92 ------- oonib/testhelpers/ssl_helpers.py | 9 + oonib/testhelpers/tcp_helpers.py | 72 ++++++ ooniprobe.conf | 6 +- requirements.txt | 45 ++-- tests/assets/ipports.txt | 2 - tests/assets/urllist.txt | 3 - tests/legacy/test_plugins.py | 40 --- tests/legacy/test_tests.py | 45 ---- tests/test_mutate.py | 15 ++ tests/test_worker.py | 29 --- to-be-ported/old-api/daphn3.py | 152 ------------ to-be-ported/old-api/httpt.py | 94 -------- to-be-ported/old-api/tcpconnect.py | 65 ----- 80 files changed, 2696 insertions(+), 2926 deletions(-)

diff --cc ooni/utils/net.py index 4d8907d,3ddba61..2583c74 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@@ -2,169 -2,91 +2,221 @@@ # # net.py # -------- - # OONI utilities for network infrastructure and hardware. - # - # :authors: Isis Lovecruft, Arturo Filasto - # :version: 0.0.1-pre-alpha - # :license: (c) 2012 Isis Lovecruft, Arturo Filasto - # see attached LICENCE file + # OONI utilities for networking related operations

- from pprint import pprint - from sys import platform + import sys -from zope.interface import implements

- #if platformm.system() == 'Windows': - # import _winreg as winreg ++from zope.interface import implements + from twisted.internet import protocol, defer + from twisted.internet import threads, reactor + from twisted.web.iweb import IBodyProducer

- from ooni.utils import log + from scapy.all import utils

- PLATFORMS = {'LINUX': platform.startswith("linux"), - 'OPENBSD': platform.startswith("openbsd"), - 'FREEBSD': platform.startswith("freebsd"), - 'NETBSD': platform.startswith("netbsd"), - 'DARWIN': platform.startswith("darwin"), - 'SOLARIS': platform.startswith("sunos"), - 'WINDOWS': platform.startswith("win32")} + from ooni.utils import log, txscapy

++#if sys.platformm.system() == 'Windows': ++# import _winreg as winreg + - class PlatformNameException(Exception): - """Specified platform does not match client platform.""" + userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + ++PLATFORMS = {'LINUX': sys.platform.startswith("linux"), ++ 'OPENBSD': sys.platform.startswith("openbsd"), ++ 'FREEBSD': sys.platform.startswith("freebsd"), ++ 'NETBSD': sys.platform.startswith("netbsd"), ++ 'DARWIN': sys.platform.startswith("darwin"), ++ 'SOLARIS': sys.platform.startswith("sunos"), ++ 'WINDOWS': sys.platform.startswith("win32")} + +class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" + +class IfaceError(Exception): + """Could not find default network interface.""" + +class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" + - def getClientAddress(): - address = {'asn': 'REPLACE_ME', - 'ip': 'REPLACE_ME'} - return address + class StringProducer(object): + implements(IBodyProducer) + + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + + def capturePackets(pcap_filename): + from scapy.all import sniff + global stop_packet_capture + stop_packet_capture = False + + def stopCapture(): + # XXX this is a bit of a hack to stop capturing packets when we close + # the reactor. Ideally we would want to be able to do this + # programmatically, but this requires some work on implementing + # properly the sniff function with deferreds. + global stop_packet_capture + stop_packet_capture = True + + def writePacketToPcap(pkt): + from scapy.all import utils + pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True) + pcapwriter.write(pkt) + if stop_packet_capture: + sys.exit(1) - + d = threads.deferToThread(sniff, lfilter=writePacketToPcap) + reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) + return d + -class PermissionsError(SystemExit): - def __init__(self, *args, **kwargs): - if not args and not kwargs: - pe = "This test requires admin or root privileges to run. Exiting..." - super(PermissionsError, self).__init__(pe, *args, **kwargs) - else: - super(PermissionsError, self).__init__(*args, **kwargs)

-class IfaceError(SystemExit): - def __init__(self, *args, **kwargs): - super(IfaceError, self).__init__(*args, **kwargs) +def getClientPlatform(platform_name=None): + for name, test in PLATFORMS.items(): + if not platform_name or platform_name.upper() == name: + if test: + return name, test + - def getPosixIface(): - from twisted.internet.test import _posixifaces ++def getPlatformAndIfaces(platform_name=None): + - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - ifup = tryInterfaces(ifaces) - return ifup ++ client, test = getClientPlatform(platform_name) + - def getWindowsIface(): - from twisted.internet.test import _win32ifaces ++ def getPosixIfaces(): ++ from twisted.internet.test import _posixifaces ++ all_ifaces = _posixifaces._interfaces() ++ ifup = checkInterfaces(all_ifaces) ++ return all_ifaces + - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - ifup = tryInterfaces(ifaces) - return ifup ++ def getWindowsIfacse(): ++ from twisted.internet.test import _win32ifaces ++ all_ifaces = _win32ifaces._interfaces() ++ return all_ifaces + - def getPlatformAndIfaces(platform_name=None): - client, test = getClientPlatform(platform_name) + if client: + if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': - return getPosixIface() ++ all_ifaces = getPosixIfaces() + elif client == 'WINDOWS': - return getWindowsIface() ++ all_ifaces = getWindowsIfaces() + ## XXX fixme figure out how to get iface for Solaris + else: + return None ++ ifup = checkInterfaces(all_ifaces) ++ return ifup + else: + raise UnsupportedPlatform + +def checkInterfaces(ifaces=None, timeout=1): + """ + @param ifaces: + A dictionary in the form of ifaces['if_name'] = 'if_addr'. + """ + try: + from scapy.all import IP, ICMP + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg(("Scapy required: www.secdev.org/projects/scapy")) + + ifup = {} + if not ifaces: + log.debug("net.checkInterfaces(): no interfaces specified!") + return None + + for iface in ifaces: + for ifname, ifaddr in iface: + log.debug("net.checkInterfaces(): testing iface {} by pinging" + + " local address {}".format(ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + if ans.summary(): + log.debug("net.checkInterfaces(): got answer on interface %s" + + ":\n%s".format(ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.debug("Interface test packet was unanswered:\n%s" + % unans.summary()) + if len(ifup) > 0: + log.msg("Discovered working network interfaces: %s" % ifup) + return ifup + else: + raise IfaceError + +def getNonLoopbackIfaces(platform_name=None): + try: + ifaces = getPlatformAndIfaces(platform_name) + except UnsupportedPlatform, up: + log.err(up) + + if not ifaces: + log.msg("Unable to discover network interfaces...") + return None + else: + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s" - % pprint(found)) ++ % found) + try: + interfaces = checkInterfaces(found) + except IfaceError, ie: + log.err(ie) + return None + else: + return interfaces + +def getNetworksFromRoutes(): - from scapy.all import conf, ltoa ++ from scapy.all import conf, ltoa, read_routes + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] - client = conf.route - log.debug("Local Routing Table:\n{}".format(client)) - - for nw, nm, gw, iface, addr in client.routes: ++ for nw, nm, gw, iface, addr in read_routes(): + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: - return net ++ return net.iface + raise IfaceError + +def getLocalAddress(): + default_iface = getDefaultIface() + return default_iface.ipaddr