[tor-commits] [ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe

isis at torproject.org isis at torproject.org
Fri Nov 16 00:50:07 UTC 2012


commit 1008bd5c04e9a9fa1f1726fa5474725d2714f37c
Merge: 870d882 2bf3da5
Author: Isis Lovecruft <isis at torproject.org>
Date:   Thu Nov 15 00:26:43 2012 +0000

    Merge branch 'master' of git-rw.torproject.org:ooni-probe
    
    Conflicts:
    	ooni/utils/net.py

 AUTHORS                                  |    2 +-
 CHANGES.yaml                             |    5 -
 HACKING                                  |  141 ++++++------
 INSTALL                                  |    1 -
 README.md                                |  123 +++++------
 TODO                                     |  105 +--------
 before_i_commit.sh                       |    2 +-
 bin/oonib                                |   41 +++-
 bin/ooniprobe                            |    5 +-
 docs/source/index.rst                    |   51 +----
 docs/source/install.rst                  |  119 ----------
 nettests/bridge_reachability/echo.py     |    2 +-
 nettests/core/captiveportal.py           |   23 +-
 nettests/core/chinatrigger.py            |    9 +-
 nettests/core/daphn3.py                  |  117 +++++++++
 nettests/core/http_host.py               |   74 +++++-
 nettests/core/http_requests.py           |  118 +++++++++
 nettests/core/http_uk_mobile_networks.py |   85 +++++++
 nettests/core/keyword_filtering.py       |   52 ++++
 nettests/core/squid.py                   |    2 +-
 nettests/core/tcpconnect.py              |   46 ++++
 nettests/core/traceroute.py              |  113 +++++++++
 nettests/core/url_list.py                |   59 ++++-
 nettests/examples/example_scapyt.py      |   22 ++-
 nettests/simpletest.py                   |   27 --
 ooni/__init__.py                         |    4 +
 ooni/config.py                           |   20 ++-
 ooni/inputunit.py                        |   19 +-
 ooni/kit/daphn3.py                       |  208 ++++++++++++++++
 ooni/lib/rfc3339.py                      |  283 ----------------------
 ooni/lib/txscapy.py                      |  381 ------------------------------
 ooni/nettest.py                          |   38 ++--
 ooni/nodes.py                            |   24 +-
 ooni/ooni-probe.conf                     |  100 --------
 ooni/oonicli.py                          |   67 ++++--
 ooni/protocols/daphn3.py                 |  311 ------------------------
 ooni/reporter.py                         |  340 +++++++++++++++------------
 ooni/runner.py                           |  117 ++++++----
 ooni/templates/httpt.py                  |  120 ++++------
 ooni/templates/scapyt.py                 |  124 +++--------
 ooni/utils/__init__.py                   |  102 +--------
 ooni/utils/date.py                       |   30 ---
 ooni/utils/geodata.py                    |   31 ++-
 ooni/utils/hacks.py                      |   75 ++++---
 ooni/utils/log.py                        |   26 ++-
 ooni/utils/net.py                        |  146 ++++++++----
 ooni/utils/otime.py                      |    8 +
 ooni/utils/txscapy.py                    |  140 +++++++++++
 oonib/__init__.py                        |   22 ++
 oonib/config.py                          |   43 ++++
 oonib/db/__init__.py                     |   30 +++
 oonib/db/tables.py                       |  123 ++++++++++
 oonib/lib/__init__.py                    |    8 -
 oonib/lib/ssl.py                         |    8 -
 oonib/models.py                          |  122 ++++++++++
 oonib/oonibackend.conf.sample            |   12 -
 oonib/oonibackend.py                     |  117 +++++-----
 oonib/report/api.py                      |  164 ++++++++++++-
 oonib/report/db/__init__.py              |   33 ---
 oonib/requirements.txt                   |   24 ++
 oonib/runner.py                          |   25 ++
 oonib/testhelpers/__init__.py            |    5 +
 oonib/testhelpers/daphn3.py              |   47 ----
 oonib/testhelpers/dns.py                 |   16 --
 oonib/testhelpers/dns_helpers.py         |   16 ++
 oonib/testhelpers/http_helpers.py        |   80 +++++++
 oonib/testhelpers/httph.py               |   92 -------
 oonib/testhelpers/ssl_helpers.py         |    9 +
 oonib/testhelpers/tcp_helpers.py         |   72 ++++++
 ooniprobe.conf                           |    6 +-
 requirements.txt                         |   45 ++--
 tests/assets/ipports.txt                 |    2 -
 tests/assets/urllist.txt                 |    3 -
 tests/legacy/test_plugins.py             |   40 ---
 tests/legacy/test_tests.py               |   45 ----
 tests/test_mutate.py                     |   15 ++
 tests/test_worker.py                     |   29 ---
 to-be-ported/old-api/daphn3.py           |  152 ------------
 to-be-ported/old-api/httpt.py            |   94 --------
 to-be-ported/old-api/tcpconnect.py       |   65 -----
 80 files changed, 2696 insertions(+), 2926 deletions(-)

diff --cc ooni/utils/net.py
index 4d8907d,3ddba61..2583c74
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@@ -2,169 -2,91 +2,221 @@@
  #
  # net.py
  # --------
- # OONI utilities for network infrastructure and hardware.
- #
- # :authors: Isis Lovecruft, Arturo Filasto
- # :version: 0.0.1-pre-alpha
- # :license: (c) 2012 Isis Lovecruft, Arturo Filasto
- #           see attached LICENCE file
+ # OONI utilities for networking related operations
  
- from pprint import pprint
- from sys import platform
+ import sys
 -from zope.interface import implements
  
- #if platformm.system() == 'Windows':
- #    import _winreg as winreg
++from zope.interface import implements
+ from twisted.internet import protocol, defer
+ from twisted.internet import threads, reactor
+ from twisted.web.iweb import IBodyProducer
  
- from ooni.utils import log
+ from scapy.all import utils
  
- PLATFORMS = {'LINUX': platform.startswith("linux"),
-              'OPENBSD': platform.startswith("openbsd"),
-              'FREEBSD': platform.startswith("freebsd"),
-              'NETBSD': platform.startswith("netbsd"),
-              'DARWIN': platform.startswith("darwin"),
-              'SOLARIS': platform.startswith("sunos"),
-              'WINDOWS': platform.startswith("win32")}
+ from ooni.utils import log, txscapy
  
++#if sys.platformm.system() == 'Windows':
++#    import _winreg as winreg
 +
- class PlatformNameException(Exception):
-     """Specified platform does not match client platform."""
+ userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"),
+               ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"),
+               ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"),
+               ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"),
+               ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"),
+               ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
+ 
++PLATFORMS = {'LINUX': sys.platform.startswith("linux"),
++             'OPENBSD': sys.platform.startswith("openbsd"),
++             'FREEBSD': sys.platform.startswith("freebsd"),
++             'NETBSD': sys.platform.startswith("netbsd"),
++             'DARWIN': sys.platform.startswith("darwin"),
++             'SOLARIS': sys.platform.startswith("sunos"),
++             'WINDOWS': sys.platform.startswith("win32")}
 +
 +class UnsupportedPlatform(Exception):
 +    """Support for this platform is not currently available."""
 +
 +class IfaceError(Exception):
 +    """Could not find default network interface."""
 +
 +class PermissionsError(SystemExit):
 +    """This test requires admin or root privileges to run. Exiting..."""
 +
- def getClientAddress():
-     address = {'asn': 'REPLACE_ME',
-                'ip': 'REPLACE_ME'}
-     return address
+ class StringProducer(object):
+     implements(IBodyProducer)
+ 
+     def __init__(self, body):
+         self.body = body
+         self.length = len(body)
+ 
+     def startProducing(self, consumer):
+         consumer.write(self.body)
+         return defer.succeed(None)
+ 
+     def pauseProducing(self):
+         pass
+ 
+     def stopProducing(self):
+         pass
+ 
+ class BodyReceiver(protocol.Protocol):
+     def __init__(self, finished):
+         self.finished = finished
+         self.data = ""
+ 
+     def dataReceived(self, bytes):
+         self.data += bytes
+ 
+     def connectionLost(self, reason):
+         self.finished.callback(self.data)
+ 
+ def capturePackets(pcap_filename):
+     from scapy.all import sniff
+     global stop_packet_capture
+     stop_packet_capture = False
+ 
+     def stopCapture():
+         # XXX this is a bit of a hack to stop capturing packets when we close
+         # the reactor. Ideally we would want to be able to do this
+         # programmatically, but this requires some work on implementing
+         # properly the sniff function with deferreds.
+         global stop_packet_capture
+         stop_packet_capture = True
+ 
+     def writePacketToPcap(pkt):
+         from scapy.all import utils
+         pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True)
+         pcapwriter.write(pkt)
+         if stop_packet_capture:
+             sys.exit(1)
 -
+     d = threads.deferToThread(sniff, lfilter=writePacketToPcap)
+     reactor.addSystemEventTrigger('before', 'shutdown', stopCapture)
+     return d
+ 
 -class PermissionsError(SystemExit):
 -    def __init__(self, *args, **kwargs):
 -        if not args and not kwargs:
 -            pe = "This test requires admin or root privileges to run. Exiting..."
 -            super(PermissionsError, self).__init__(pe, *args, **kwargs)
 -        else:
 -            super(PermissionsError, self).__init__(*args, **kwargs)
  
 -class IfaceError(SystemExit):
 -    def __init__(self, *args, **kwargs):
 -        super(IfaceError, self).__init__(*args, **kwargs)
 +def getClientPlatform(platform_name=None):
 +    for name, test in PLATFORMS.items():
 +        if not platform_name or platform_name.upper() == name:
 +            if test:
 +                return name, test
 +
- def getPosixIface():
-     from twisted.internet.test import _posixifaces
++def getPlatformAndIfaces(platform_name=None):
 +
-     log.msg("Attempting to discover network interfaces...")
-     ifaces = _posixifaces._interfaces()
-     ifup = tryInterfaces(ifaces)
-     return ifup
++    client, test = getClientPlatform(platform_name)
 +
- def getWindowsIface():
-     from twisted.internet.test import _win32ifaces
++    def getPosixIfaces():
++        from twisted.internet.test import _posixifaces
++        all_ifaces = _posixifaces._interfaces()
++        ifup = checkInterfaces(all_ifaces)
++        return all_ifaces
 +
-     log.msg("Attempting to discover network interfaces...")
-     ifaces = _win32ifaces._interfaces()
-     ifup = tryInterfaces(ifaces)
-     return ifup
++    def getWindowsIfacse():
++        from twisted.internet.test import _win32ifaces
++        all_ifaces = _win32ifaces._interfaces()
++        return all_ifaces
 +
- def getPlatformAndIfaces(platform_name=None):
-     client, test = getClientPlatform(platform_name)
 +    if client:
 +        if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD':
-             return getPosixIface()
++            all_ifaces = getPosixIfaces()
 +        elif client == 'WINDOWS':
-             return getWindowsIface()
++            all_ifaces = getWindowsIfaces()
 +        ## XXX fixme figure out how to get iface for Solaris
 +        else:
 +            return None
++        ifup = checkInterfaces(all_ifaces)
++        return ifup
 +    else:
 +        raise UnsupportedPlatform
 +
 +def checkInterfaces(ifaces=None, timeout=1):
 +    """
 +    @param ifaces:
 +        A dictionary in the form of ifaces['if_name'] = 'if_addr'.
 +    """
 +    try:
 +        from scapy.all import IP, ICMP
 +        from scapy.all import sr1   ## we want this check to be blocking
 +    except:
 +        log.msg(("Scapy required: www.secdev.org/projects/scapy"))
 +
 +    ifup = {}
 +    if not ifaces:
 +        log.debug("net.checkInterfaces(): no interfaces specified!")
 +        return None
 +
 +    for iface in ifaces:
 +        for ifname, ifaddr in iface:
 +            log.debug("net.checkInterfaces(): testing iface {} by pinging"
 +                      + " local address {}".format(ifname, ifaddr))
 +            try:
 +                pkt = IP(dst=ifaddr)/ICMP()
 +                ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3)
 +            except Exception, e:
 +                raise PermissionsError if e.find("Errno 1") else log.err(e)
 +            else:
 +                if ans.summary():
 +                    log.debug("net.checkInterfaces(): got answer on interface %s"
 +                             + ":\n%s".format(ifname, ans.summary()))
 +                    ifup.update(ifname, ifaddr)
 +                else:
 +                    log.debug("Interface test packet was unanswered:\n%s"
 +                             % unans.summary())
 +    if len(ifup) > 0:
 +        log.msg("Discovered working network interfaces: %s" % ifup)
 +        return ifup
 +    else:
 +        raise IfaceError
 +
 +def getNonLoopbackIfaces(platform_name=None):
 +    try:
 +        ifaces = getPlatformAndIfaces(platform_name)
 +    except UnsupportedPlatform, up:
 +        log.err(up)
 +
 +    if not ifaces:
 +        log.msg("Unable to discover network interfaces...")
 +        return None
 +    else:
 +        found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo']
 +        log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s"
-                   % pprint(found))
++                  % found)
 +        try:
 +            interfaces = checkInterfaces(found)
 +        except IfaceError, ie:
 +            log.err(ie)
 +            return None
 +        else:
 +            return interfaces
 +
 +def getNetworksFromRoutes():
-     from scapy.all import conf, ltoa
++    from scapy.all import conf, ltoa, read_routes
 +    from ipaddr    import IPNetwork, IPAddress
 +
 +    ## Hide the 'no routes' warnings
 +    conf.verb = 0
 +
 +    networks = []
-     client   = conf.route
-     log.debug("Local Routing Table:\n{}".format(client))
- 
-     for nw, nm, gw, iface, addr in client.routes:
++    for nw, nm, gw, iface, addr in read_routes():
 +        n = IPNetwork( ltoa(nw) )
 +        (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]]
 +        n.iface = iface
 +        if not n.compressed in networks:
 +            networks.append(n)
 +
 +    return networks
 +
 +def getDefaultIface():
 +    networks = getNetworksFromRoutes()
 +    for net in networks:
 +        if net.is_private:
-             return net
++            return net.iface
 +    raise IfaceError
 +
 +def getLocalAddress():
 +    default_iface = getDefaultIface()
 +    return default_iface.ipaddr





More information about the tor-commits mailing list