[tor-commits] [ooni-probe/master] * Removing old tests which have already been ported: dnstamper, echo,

isis at torproject.org isis at torproject.org
Sat Nov 3 01:24:44 UTC 2012


commit 77c07070e7e8575abc7e6b9fdeed4d7664736ec3
Author: Isis Lovecruft <isis at torproject.org>
Date:   Fri Nov 2 16:47:40 2012 +0000

    * Removing old tests which have already been ported: dnstamper, echo,
      blocking.
---
 nettests/core/dnstamper.py |   29 +++--
 ooni/plugins/blocking.py   |   46 ------
 ooni/plugins/dnstamper.py  |  338 --------------------------------------------
 ooni/plugins/echo.py       |  127 -----------------
 4 files changed, 19 insertions(+), 521 deletions(-)

diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py
index b5fcea3..aad2ef3 100644
--- a/nettests/core/dnstamper.py
+++ b/nettests/core/dnstamper.py
@@ -1,6 +1,5 @@
 # -*- encoding: utf-8 -*-
 #
-#
 #  dnstamper
 #  *********
 #
@@ -25,16 +24,13 @@ from twisted.names.error import DNSQueryRefusedError
 class DNSTamperTest(nettest.TestCase):
 
     name = "DNS tamper"
-
     description = "DNS censorship detection test"
     version = "0.2"
-
     lookupTimeout = [1]
-
     requirements = None
+
     inputFile = ['file', 'f', None,
                  'Input file of list of hostnames to attempt to resolve']
-
     optParameters = [['controlresolver', 'c', '8.8.8.8',
                       'Known good DNS server'],
                      ['testresolvers', 't', None,
@@ -43,20 +39,18 @@ class DNSTamperTest(nettest.TestCase):
     def setUp(self):
         self.report['test_lookups'] = {}
         self.report['test_reverse'] = {}
-
         self.report['control_lookup'] = []
-
         self.report['a_lookups'] = {}
-
         self.report['tampering'] = {}
 
         self.test_a_lookups = {}
         self.control_a_lookups = []
-
         self.control_reverse = None
         self.test_reverse = {}
 
         if not self.localOptions['testresolvers']:
+            log.msg("You did not specify a file of DNS servers to test!",
+                    "See the '--testresolvers' option.")
             self.test_resolvers = ['8.8.8.8']
             return
 
@@ -181,6 +175,14 @@ class DNSTamperTest(nettest.TestCase):
         return r
 
     def do_reverse_lookups(self, result):
+        """
+        Take a resolved address in the form "176.139.79.178.in-addr.arpa." and
+        attempt to reverse the domain with both the control and test DNS
+        servers to see if they match.
+
+        :param result:
+            A resolved domain name.
+        """
         log.msg("Doing the reverse lookups %s" % self.input)
         list_of_ds = []
 
@@ -209,6 +211,12 @@ class DNSTamperTest(nettest.TestCase):
         return dl
 
     def compare_results(self, *arg, **kw):
+        """
+        Take the set intersection of two test result sets. If the intersection
+        is greater than zero (there are matching addresses in both sets) then
+        the no censorship is reported. Else, if no IP addresses match other
+        addresses, then we mark it as a censorship event.
+        """
         log.msg("Comparing results for %s" % self.input)
         log.msg(self.test_a_lookups)
 
@@ -222,7 +230,8 @@ class DNSTamperTest(nettest.TestCase):
                 # Address has not tampered with on DNS server
                 self.report['tampering'][test] = False
 
-            elif self.control_reverse and set([self.control_reverse]) & set([self.report['test_reverse'][test]]):
+            elif self.control_reverse and set([self.control_reverse]) \
+                    & set([self.report['test_reverse'][test]]):
                 # Further testing has eliminated false positives
                 self.report['tampering'][test] = 'reverse-match'
 
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
deleted file mode 100644
index 4dd2db1..0000000
--- a/ooni/plugins/blocking.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-
-from plugoo.assets import Asset
-from plugoo.tests import ITest, OONITest
-
-class BlockingArgs(usage.Options):
-    optParameters = [['asset', 'a', None, 'Asset file'],
-                     ['resume', 'r', 0, 'Resume at this index'],
-                     ['shit', 'o', None, 'Other arguments']]
-
-class BlockingTest(OONITest):
-    implements(IPlugin, ITest)
-
-    shortName = "blocking"
-    description = "Blocking plugin"
-    requirements = None
-    options = BlockingArgs
-    # Tells this to be blocking.
-    blocking = True
-
-    def control(self, experiment_result, args):
-        print "Experiment Result:", experiment_result
-        print "Args", args
-        return experiment_result
-
-    def experiment(self, args):
-        import urllib
-        url = 'http://torproject.org/' if not 'asset' in args else args['asset']
-        try:
-            req = urllib.urlopen(url)
-        except:
-            return {'error': 'Connection failed!'}
-
-        return {'page': req.readlines()}
-
-    def load_assets(self):
-        if self.local_options and self.local_options['asset']:
-            return {'asset': Asset(self.local_options['asset'])}
-        else:
-            return {}
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-#blocking = BlockingTest(None, None, None)
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
deleted file mode 100644
index 40df505..0000000
--- a/ooni/plugins/dnstamper.py
+++ /dev/null
@@ -1,338 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    dnstamper
-    *********
-
-    This test resolves DNS for a list of domain names, one per line, in the
-    file specified in the ooni-config under the setting "dns_experiment". If
-    the file is top-1m.txt, the test will be run using Amazon's list of top
-    one million domains. The experimental dns servers to query should
-    be specified one per line in assets/dns_servers.txt.
-
-    The test reports censorship if the cardinality of the intersection of
-    the query result set from the control server and the query result set
-    from the experimental server is zero, which is to say, if the two sets
-    have no matching results whatsoever.
-
-    NOTE: This test frequently results in false positives due to GeoIP-based
-    load balancing on major global sites such as google, facebook, and
-    youtube, etc.
-
-    :author: Isis Lovecruft, Arturo Filastò
-    :license: see LICENSE for more details
-
-    TODO:
-         * Finish porting to twisted
-         * Finish the client.Resolver() subclass and test it
-         * Use the DNS tests from captiveportal
-         * Use plugoo/reports.py for final data
-"""
-
-import os
-
-from twisted.names import client, dns
-from twisted.internet import reactor, defer
-from twisted.internet.error import CannotListenError
-from twisted.internet.protocol import Factory, Protocol
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from zope.interface import implements
-
-from ooni.plugoo.assets import Asset
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.utils import log
-
-class AlexaAsset(Asset):
-    """
-    Class for parsing the Alexa top-1m.txt as an asset.
-    """
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-    def parse_line(self, line):
-        self = Asset.parse_line(self, line)
-        return line.split(',')[1].replace('\n','')
-
-class DNSTamperArgs(usage.Options):
-    optParameters = [['hostnames', 'h', None, 
-                      'Asset file of hostnames to resolve'],
-                     ['controlresolver', 'c', '8.8.8.8', 
-                      'Known good DNS server'],
-                     ['testresolvers', 't', None, 
-                      'Asset file of DNS servers to test'],
-                     ['localresolvers', 'l', False, 
-                      'Also test local servers'],
-                     ['port', 'p', None, 
-                      'Local UDP port to send queries over'],
-                     ['usereverse', 'r', False, 
-                      'Also try reverse DNS resolves'],
-                     ['resume', 's', 0, 
-                      'Resume at this index in the asset file']]
-
-class DNSTamperResolver(client.Resolver):
-    """
-    Twisted by default issues DNS queries over cryptographically random
-    UDP ports to mitigate the Berstein/Kaminsky attack on limited DNS
-    Transaction ID numbers.[1][2][3]
-
-    This is fine, unless the client has external restrictions which require
-    DNS queries to be conducted over UDP port 53. Twisted does not provide
-    an easy way to change this, ergo subclassing client.Resolver.[4] It 
-    would perhaps be wise to patch twisted.names.client and request a merge
-    into upstream.
-
-    [1] https://twistedmatrix.com/trac/ticket/3342
-    [2] http://blog.netherlabs.nl/articles/2008/07/09/ \
-        some-thoughts-on-the-recent-dns-vulnerability
-    [3] http://www.blackhat.com/presentations/bh-dc-09/Kaminsky/ \
-        BlackHat-DC-09-Kaminsky-DNS-Critical-Infrastructure.pdf
-    [4] http://comments.gmane.org/gmane.comp.python.twisted/22794
-    """
-    def __init__(self):
-        super(DNSTamperResolver, self).__init__(self, resolv, servers,
-                                                timeout, reactor)
-        #client.Resolver.__init__(self)
-
-        if self.local_options['port']:
-            self.port = self.local_options['port']
-        else:
-            self.port = '53'
-
-    def _connectedProtocol(self):
-        """
-        Return a new DNSDatagramProtocol bound to a specific port
-        rather than the default cryptographically-random port.
-        """
-        if 'protocol' in self.__dict__:
-            return self.protocol
-        proto = dns.DNSDatagramProtocol(self)
-        
-        ## XXX We may need to remove the while loop, which was 
-        ## originally implemented to safeguard against attempts to
-        ## bind to the same random port twice...but then the code
-        ## would be blocking...
-        while True:
-            try:
-                self._reactor.listenUDP(self.port, proto)
-            except error.CannotListenError:
-                pass
-            else:
-                return proto
-
-class DNSTamperTest(OONITest):
-    """
-    XXX fill me in
-    """
-    implements(IPlugin, ITest)
-
-    shortName = "dnstamper"
-    description = "DNS censorship detection test"
-    requirements = None
-    options = DNSTamperArgs
-    blocking = False
-    
-    def __init__(self, local_options, global_options, 
-                 report, ooninet=None, reactor=None):
-        super(DNSTamperTest, self).__init__(local_options, global_options,
-                                            report, ooninet, reactor)
-
-    def __repr__(self):
-        represent = "DNSTamperTest(OONITest): local_options=%r, " \
-            "global_options=%r, assets=%r" % (self.local_options, 
-                                              self.global_options, 
-                                              self.assets)
-        return represent
-
-    def initialize(self):
-        if self.local_options:
-            ## client.createResolver() turns 'None' into '/etc/resolv.conf' on
-            ## posix systems, ignored on Windows.
-            if self.local_options['localresolvers']:
-                self.resolvconf = None
-            else:
-                self.resolvconf = ''
-
-    def load_assets(self):
-        assets = {}
-        
-        #default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', 
-        #                     'ooni.nu', 'google.com', 'torproject.org']
-        #default_resolvers = ['209.244.0.3', '208.67.222.222']
-
-        def asset_file(asset_option):
-            return self.local_options[asset_option]
-
-        def list_to_asset(list_):
-            def next(list_):
-                host = list_.pop()
-                if host is not None:
-                    yield str(host)
-            while len(list_) > 0:
-                next(list_)
-
-        if self.local_options:
-            if asset_file('hostnames'):
-                ## The default filename for the Alexa Top 1 Million:
-                if asset_file('hostnames') == 'top-1m.txt':
-                    assets.update({'hostnames': 
-                                   AlexaAsset(asset_file('hostnames'))})
-                else:
-                    assets.update({'hostnames': 
-                                   Asset(asset_file('hostnames'))})
-            else:
-                log.msg("Error! We need an asset file containing the " + 
-                        "hostnames that we should test DNS with! Please use " + 
-                        "the '-h' option. Using pre-defined hostnames...")
-
-            if asset_file('testresolvers'):
-                assets.update({'testresolvers': 
-                               Asset(asset_file('testresolvers'))})
-
-        return assets
-
-    def lookup(self, hostname, resolver):
-        """
-        Resolves a hostname through a DNS nameserver to the corresponding IP
-        addresses.
-        """
-        def got_result(result, hostname, resolver):
-            log.msg('Resolved %s through %s to %s' 
-                    % (hostname, resolver, result))
-            report = {'resolved': True,
-                       'domain': hostname,
-                       'nameserver': resolver,
-                       'address': result }
-            log.msg(report)
-            return result
-
-        def got_error(err, hostname, resolver):
-            log.msg(err.printTraceback())
-            report = {'resolved': False,
-                       'domain': hostname,
-                       'nameserver': resolver,
-                       'address': err }
-            log.msg(report)
-            return err
-
-        res = client.createResolver(resolvconf=self.resolvconf, 
-                                    servers=[(resolver, 53)])
-
-        ## XXX should we do self.d.addCallback(resHostByName, hostname)?
-        #d = res.getHostByName(hostname)
-        #d.addCallbacks(got_result, got_error)
-
-        #d = defer.Deferred()
-        #d.addCallback(res.getHostByName, hostname)
-
-        #d = res.getHostByName(hostname)
-        #d.addCallback(got_result, result, hostname, resolver) 
-        #d.addErrback(got_error, err, hostname, resolver)
-
-        res.addCallback(getHostByName, hostname)
-        res.addCallback(got_result, result, hostname, resolver)
-        res.addErrback(got_error, err, hostname, resolver)
-
-        if self.local_options['usereverse']:
-            #d.addCallback(self.reverse_lookup, result, resolver)
-            #d.addErrback(log.msg(err.printTraceback()))
-
-            #d.addCallback(self.reverse_lookup, result, resolver)
-            #d.addErrback(log.msg(err.printTraceback()))
-
-            res.addCallback(self.reverse_lookup, result, resolver)
-            res.addErraback(log.msg(err.printTraceback()))
-        
-        return res
-
-    def reverse_lookup(self, address, resolver):
-        """
-        Attempt to do a reverse DNS lookup to determine if the control and exp
-        sets from a positive result resolve to the same domain, in order to
-        remove false positives due to GeoIP load balancing.
-        """
-        res = client.createResolver(resolvconf=self.resolvconf, 
-                                    servers=[(resolver, 53)])
-        ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa'
-        reverse = res.lookupPointer(ptr)
-        reverse.addCallback(lambda (address, auth, add): 
-                            util.println(address[0].payload.name))
-        reverse.addErrback(log.err)
-
-        ## XXX do we need to stop the reactor?
-        #d.addBoth(lambda r: reactor.stop())
-
-        return reverse
-        
-    def experiment(self, args):
-        """
-        Compares the lookup() sets of the control and experiment groups.
-        """
-        for hostname in args:
-            for testresolver in self.assets['testresolvers']:
-                #addressd = defer.Deferred()
-                #addressd.addCallback(self.lookup, hostname, testresolver)
-                #addressd.addErrback(log.err)
-
-                self.d.addCallback(self.lookup, hostname, testresolver)
-                self.d.addErrback(log.err)
-
-                #addressd = self.lookup(hostname, testresolver)
-
-                #self.d.addCallback(self.lookup, hostname, testserver)
-
-                print "%s" % type(addressd)
-
-                return self.d
-
-    def control(self, experiment_result, args):
-        print "EXPERIMENT RESULT IS %s" % experiment_result
-        (exp_address, hostname, testserver, exp_reversed) = experiment_result
-        control_server = self.local_options['controlserver']
-        ctrl_address = self.lookup(hostname, control_server)
-        
-        ## XXX getHostByName() appears to be returning only one IP...
-
-        if len(set(exp_address) & set(ctrl_address)) > 0:
-            log.msg("Address %s has not tampered with on DNS server %s" 
-                    % (hostname, test_server))
-            return {'hostname': hostname,
-                    'test-nameserver': test_server,
-                    'test-address': exp_address,
-                    'control-nameserver': control_server,
-                    'control-address': ctrl_address,
-                    'tampering-detected': False}
-        else:
-            log.msg("Address %s has possibly been tampered on %s:"
-                    % (hostname, test_server))
-            log.msg("DNS resolution through testserver %s yeilds: %s"
-                    % (test_server, exp_address))
-            log.msg("However, DNS resolution through controlserver %s yeilds: %s"
-                    % (control_server, ctrl_address))
-
-            if self.local_options['usereverse']:
-                ctrl_reversed = self.reverse_lookup(experiment_result, control_server)
-                if len(set(ctrl_reversed) & set(exp_reversed)) > 0:
-                    log.msg("Further testing has eliminated false positives")
-                else:
-                    log.msg("Reverse DNS on the results returned by %s returned:"
-                            % (test_server))
-                    log.msg("%s" % exp_reversed)
-                    log.msg("which does not match the expected domainname: %s"
-                            % ctrl_reversed)
-                return {'hostname': hostname,
-                        'test-nameserver': test_server,
-                        'test-address': exp_address,
-                        'test-reversed': exp_reversed,
-                        'control-nameserver': control_server,
-                        'control-address': ctrl_address,
-                        'control-reversed': ctrl_reversed,
-                        'tampering-detected': True}
-            else:
-                return {'hostname': hostname,
-                        'test-nameserver': test_server,
-                        'test-address': exp_address,
-                        'control-nameserver': control_server,
-                        'control-address': ctrl_address,
-                        'tampering-detected': False}
-
-#dnstamper = DNSTamperTest(None, None, None)
diff --git a/ooni/plugins/echo.py b/ooni/plugins/echo.py
deleted file mode 100644
index bc1b2a8..0000000
--- a/ooni/plugins/echo.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-#
-#  +---------+
-#  | echo.py |
-#  +---------+
-#     A simply ICMP-8 ping test.
-#
-# :author: Isis Lovecruft
-# :version: 0.1.0-pre-alpha
-# :license: (c) 2012 Isis Lovecruft
-#           see attached LICENCE file
-#
-
-import os
-import sys
-
-from twisted.plugin import IPlugin
-from twisted.python import usage
-from zope.interface import implements
-
-from lib                  import txscapy
-from utils                import log
-from plugoo.assets        import Asset
-from plugoo.interface     import ITest
-from protocols.scapyproto import ScapyTest
-
-class EchoOptions(usage.Options):
-    optParameters = [
-        ['interface', 'i', None, 'Network interface to use'],
-        ['destination', 'd', None, 'File of hosts to ping'],
-        ['count', 'c', 5, 'Number of packets to send', int],
-        ['size', 's', 56, 'Number of bytes to send in ICMP data field', int],
-        ['ttl', 't', 25, 'Set the IP Time to Live', int],
-        ]
-    optFlags = []
-
-class EchoAsset(Asset):
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-    def parse_line(self, line):
-        if line.startswith('#'):
-            return
-        else:
-            return line.replace('\n', '')
-
-class EchoTest(ScapyTest):
-    implements(IPlugin, ITest)
-    
-    shortName    = 'echo'
-    description  = 'A simple ICMP-8 test to check if a host is reachable'
-    options      = EchoOptions
-    requirements = None
-    blocking     = False
-
-    pcap_file = 'echo.pcap'
-    receive   = True
-
-    def initialize(self):
-        self.request = {}
-        self.response = {}
-
-        if self.local_options:
-
-            options = self.local_options
-
-            if options['interface']:
-                self.interface = options['interface']
-
-            if options['count']:
-                ## there's a Counter() somewhere, use it
-                self.count = options['count']
-
-            if options['size']:
-                self.size = options['size']
-
-            if options['ttl']:
-                self.ttl = options['ttl']
-
-    def load_assets(self):
-        assets = {}
-        option = self.local_options
-
-        if option and option['destination']:
-
-            try:
-                from scapy.all import IP
-            except:
-                log.err()
-
-            if os.path.isfile(option['destination']):
-                with open(option['destination']) as hosts:
-                    for line in hosts.readlines():
-                        assets.update({'host': EchoAsset(line)})
-            else:
-                while type(options['destination']) is str:
-                    try:
-                        IP(options['destination'])
-                    except:
-                        log.err()
-                        break
-                    assets.update({'host': options['destination']})
-                else:
-                    log.msg("Couldn't understand destination option...")
-                    log.msg("Give one IPv4 address, or a file with one address per line.")                     
-        return assets
-
-    def experiment(self, args):
-        if len(args) == 0:
-            log.err("Error: We're Echo, not Narcissus!")
-            log.err("       Provide a list of hosts to ping...")
-            d = sys.exit(1)
-            return d
-
-        ## XXX v4 / v6 
-        from scapy.all import ICMP, IP, sr
-        ping = sr(IP(dst=args)/ICMP())
-        if ping:
-            self.response.update(ping.show())
-        else:
-            log.msg('No response received from %s' % args)
-
-    def control(self, *args):
-        pass
-
-echo = EchoTest(None, None, None)





More information about the tor-commits mailing list