[ooni-probe/master] Merge branch 'twisted' of github.com:hellais/ooni-probe

2 Jun 2012

--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,0 -1,72 +1,77 @@@
+ # ooni-probe
+ #
+ # These are the global configuration parameters necessary to
+ # make ooni-probe work
+ [main]
+ reportdir = reports/
+ logfile = ooniprobe.log
+ assetdir = assets/
+ testdir = oonitests/
+ 
+ loglevel = DEBUG
+ consoleloglevel = DEBUG
+ proxyaddress = 127.0.0.1:9050
+ 
+ # The following configurations are for searching for PlanetLab
+ # nodes, adding them to a slice, and PlanetLab general API
+ # authentication:
+ pl_username = yourusername
+ pl_password = yourpassword
+ 
+ # These are configurations specific to the tests that should be
+ # run by ooni-probe
+ [tests]
+ run = dnstamper
+ ### DNS testing related config parameters
+ 
+ # This is the list of hostnames that must be looked up
+ dns_experiment = top-1m.txt
+ 
+ # This is the dns servers to be tested
+ dns_experiment_dns = dns_servers.txt
+ 
+ # This is the control known good DNS server
 -dns_control_server = 8.8.8.8
++dns_control_server = 91.191.136.152
++
++# Specify whether the dnstamper test should attempt to remove 
++# GeoIP-based false positives by doing a reverse DNS resolve
++# on positive results.
++dns_reverse_lookup = true
+ 
+ ### traceroute testing related config parameters
+ 
+ # This is the list of ips to traceroute to
+ traceroute = example_exp_list.txt
+ 
+ # This is the list of ports that should be used
+ # src_x,src_y,src_z|dst_x,dst_y,dst_z
+ traceroute_ports = 0,53,80,123,443|0,53,80,123,443
+ 
+ # The protocol to be used in the scan
+ traceroute_proto = UDP, TCP, ICMP
+ 
+ ### keyword injection related tests
+ 
+ # List of keywords
+ keywords = keywordlist.txt
+ 
+ # hosts
+ keywords_hosts = hostslist.txt
+ 
+ # Methods to be used for testing
+ keyword_method = http,telnet
+ 
+ ### Tor bridge testing
+ 
+ tor_bridges = bridgetests.txt
+ tor_bridges_timeout = 40
+ 
+ [report]
+ file = report.log
+ timestamp = true
+ #ssh = 127.0.0.1:22
+ #ssh_user = theusername
+ #ssh_password = thepassword
+ #ssh_keyfile = ~/.ssh/mykey_rsa
+ #ssh_rpath = ~/ooni-probe/
+ #tcp = "127.0.0.1:9088"
diff --cc ooni/oonitests/dnstamper.py
index 0000000,68be12d..498ba04
mode 000000,100644..100644
--- a/ooni/oonitests/dnstamper.py
+++ b/ooni/oonitests/dnstamper.py
@@@ -1,0 -1,70 +1,156 @@@
++# -*- coding: utf-8 -*-
++"""
++    dnstamper
++    *********
++
++    This test resolves DNS for a list of domain names, one per line, in the
++    file specified in the ooni-config under the setting "dns_experiment". If
++    the file is top-1m.txt, the test will be run using Amazon's list of top
++    one million domains. The experimental dns servers to query should
++    be specified one per line in assets/dns_servers.txt.
++
++    The test reports censorship if the cardinality of the intersection of
++    the query result set from the control server and the query result set
++    from the experimental server is zero, which is to say, if the two sets
++    have no matching results whatsoever.
++
++    NOTE: This test frequently results in false positives due to GeoIP-based
++    load balancing on major global sites such as google, facebook, and
++    youtube, etc.
++
++    :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
++    :license: see LICENSE for more details
++"""
++
++try:
++    from dns import resolver, reversename
++except:
++    print "Error: dnspython is not installed! (http://www.dnspython.org/)"
+ try:
 -    from dns import resolver
++    import gevent
+ except:
 -    print "Error dnspython is not installed! (http://www.dnspython.org/)"
 -import gevent
++    print "Error: gevent is not installed! (http://www.gevent.org/)"
++
+ import os
++
+ import plugoo
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+ 
 -
+ __plugoo__ = "DNST"
+ __desc__ = "DNS censorship detection test"
+ 
++class Top1MAsset(Asset):
++    """
++    Class for parsing top-1m.txt as an asset.
++    """
++    def __init__(self, file=None):
++        self = Asset.__init__(self, file)
++
++    def parse_line(self, line):
++        self = Asset.parse_line(self, line)
++        return line.split(',')[1].replace('\n','')
++
+ class DNSTAsset(Asset):
++    """
++    Creates DNS testing specific Assets.
++    """
+     def __init__(self, file=None):
 -        self = asset.__init__(self, file)
++        self = Asset.__init__(self, file)
+ 
+ class DNST(Test):
+     def lookup(self, hostname, ns):
++        """
++        Resolves a hostname through a DNS nameserver, ns, to the corresponding
++        IP address(es).
++        """
+         res = resolver.Resolver(configure=False)
+         res.nameservers = [ns]
+         answer = res.query(hostname)
+ 
+         ret = []
+ 
+         for data in answer:
+             ret.append(data.address)
+ 
+         return ret
+ 
++    def reverse_lookup(self, ip, ns):
++        """
++        Attempt to do a reverse DNS lookup to determine if the control and exp
++        sets from a positive result resolve to the same domain, in order to
++        remove false positives due to GeoIP load balancing.
++        """
++        res = resolver.Resolver(configure=False)
++        res.nameservers = [ns]
++        n = reversename.from_address(ip)
++        revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
++
++        return revn
++
+     def experiment(self, *a, **kw):
++        """
++        Compares the lookup() sets of the control and experiment groups.
++        """
+         # this is just a dirty hack
+         address = kw['data'][0]
+         ns = kw['data'][1]
+ 
+         config = self.config
++        ctrl_ns = config.tests.dns_control_server
+ 
+         print "ADDRESS: %s" % address
+         print "NAMESERVER: %s" % ns
+ 
+         exp = self.lookup(address, ns)
 -        control = self.lookup(address, config.tests.dns_control_server)
++        control = self.lookup(address, ctrl_ns)
++
++        result = []
+ 
+         if len(set(exp) & set(control)) > 0:
 -            print "%s : no tampering on %s" % (address, ns)
 -            return (address, ns, False)
++            print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
++            result = (address, ns, exp, control, False)
++            return result
+         else:
 -            print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control)
 -            return (address, ns, exp, control, True)
++            print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
++            result = (address, ns, exp, control, True)
++
++            if config.tests.dns_reverse_lookup:
++
++                exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
++                ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
++                            for ip in control]
++
++                if len(set(exprevn) & set(ctrlrevn)) > 0:
++                    print "Further testing has eliminated this as a false positive."
++                else:
++                    print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
++                return result
++
++            else:
++                print "\n"
++                return result
+ 
+ def run(ooni):
 -    """Run the test
++    """
++    Run the test.
+     """
+     config = ooni.config
+     urls = []
+ 
 -    dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \
 -                                            config.tests.dns_experiment))
 -    dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \
++    if (config.tests.dns_experiment == "top-1m.txt"):
++        dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
++                                                 config.tests.dns_experiment))
++    else:
++        dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
++                                                config.tests.dns_experiment))
++    dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
+                                                 config.tests.dns_experiment_dns))
+ 
+     assets = [dns_experiment, dns_experiment_dns]
+ 
+     dnstest = DNST(ooni)
 -    ooni.logger.info("starting test")
 -    dnstest.run(assets)
 -    ooni.logger.info("finished")
 -
++    ooni.logger.info("Beginning dnstamper test...")
++    dnstest.run(assets, {'index': 1})
++    ooni.logger.info("Dnstamper test completed!")
+ 
diff --cc ooni/oonitests/httphost.py
index 0000000,6446e1f..25adff2
mode 000000,100644..100644
--- a/ooni/oonitests/httphost.py
+++ b/ooni/oonitests/httphost.py
@@@ -1,0 -1,132 +1,135 @@@
+ """
+     HTTP Host based filtering
+     *************************
+ 
+     This test detect HTTP Host field
+     based filtering.
+     It is used to detect censorship on
+     performed with Web Guard (used by
+     T-Mobile US).
+ """
+ import os
+ from datetime import datetime
+ from gevent import monkey
+ 
+ import urllib2
+ import httplib
+ # WARNING! Using gevent's socket
+ # introduces the 0x20 DNS "feature".
+ # This will result is weird DNS requests
+ # appearing on the wire.
+ monkey.patch_socket()
+ 
 -from BeautifulSoup import BeautifulSoup
++try:
++    from BeautifulSoup import BeautifulSoup
++except:
++    print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/"
+ 
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+ 
+ __plugoo__ = "HTTP Host"
+ __desc__ = "This detects HTTP Host field based filtering"
+ 
+ class HTTPHostAsset(Asset):
+     """
+     This is the asset that should be used by the Test. It will
+     contain all the code responsible for parsing the asset file
+     and should be passed on instantiation to the test.
+     """
+     def __init__(self, file=None):
+         self = Asset.__init__(self, file)
+ 
+     def parse_line(self, line):
+         return line.split(',')[1].replace('\n','')
+ 
+ class HTTPHost(Test):
+     """
+     The main Test class
+     """
+ 
+     def check_response(self, response):
+         soup = BeautifulSoup(response)
+         if soup.head.title.string == "Content Filtered":
+             # Response indicates censorship
+             return True
+         else:
+             # Response does not indicate censorship
+             return False
+ 
+ 
+     def is_censored(self, response):
+         if response:
+             soup = BeautifulSoup(response)
+             censored = self.check_response(response)
+         else:
+             censored = "unreachable"
+         return censored
+ 
+     def urllib2_test(self, control_server, host):
+         req = urllib2.Request(control_server)
+         req.add_header('Host', host)
+         try:
+             r = urllib2.urlopen(req)
+             response = r.read()
+             censored = self.is_censored(response)
+         except Exception, e:
+             censored = "Error! %s" % e
+ 
+         return censored
+ 
+     def httplib_test(self, control_server, host):
+         try:
+             conn = httplib.HTTPConnection(control_server)
+             conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
+             conn.putheader("Host", host)
+             conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
+             conn.endheaders()
+             r = conn.getresponse()
+             response = r.read()
+             censored = self.is_censored(response)
+         except Exception, e:
+             censored = "Error! %s" % e
+ 
+         return censored
+ 
+ 
+     def experiment(self, *a, **kw):
+         """
+         Try to connect to the control server with
+         the specified host field.
+         """
+         host = kw['data']
+         control_server = kw['control_server']
+         self.logger.info("Testing %s (%s)" % (host, control_server))
+ 
+         #censored = self.urllib2_test(control_server, host)
+         censored = self.httplib_test(control_server, host)
+ 
+         self.logger.info("%s: %s" % (host, censored))
+         return {'Time': datetime.now(),
+                 'Host': host,
+                 'Censored': censored}
+ 
+ 
+ def run(ooni):
+     """
+     This is the function that will be called by OONI
+     and it is responsible for instantiating and passing
+     the arguments to the Test class.
+     """
+     config = ooni.config
+ 
+     # This the assets array to be passed to the run function of
+     # the test
+     assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
+                                             "top-1m.csv"))]
+ 
+     # Instantiate the Test
+     thetest = HTTPHost(ooni)
+     ooni.logger.info("starting HTTP Host Test...")
+     # Run the test with argument assets
+     thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
+     ooni.logger.info("finished.")
+ 
+ 
diff --cc ooni/plugoo/reports.py
index 0000000,c099456..c3e7c23
mode 000000,100644..100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@@ -1,0 -1,175 +1,175 @@@
+ import os
+ from datetime import datetime
+ import yaml
+ 
+ import logging
+ import itertools
+ import gevent
+ 
+ class Report:
+     """This is the ooni-probe reporting mechanism. It allows
+     reporting to multiple destinations and file formats.
+ 
+     :scp the string of <host>:<port> of an ssh server
+ 
+     :yaml the filename of a the yaml file to write
+ 
+     :file the filename of a simple txt file to write
+ 
+     :tcp the <host>:<port> of a TCP server that will just listen for
+          inbound connection and accept a stream of data (think of it
+          as a `nc -l -p <port> > filename.txt`)
+     """
+     def __init__(self, ooni,
+                  scp="127.0.0.1:22",
+                  file="test.report",
+                  tcp="127.0.0.1:9000"):
+ 
+         self.file = file
+         self.tcp = tcp
+         self.scp = scp
+         self.config = ooni.config.report
+         self.logger = ooni.logger
+ 
+         if self.config.timestamp:
+             tmp = self.file.split('.')
+             self.file = '.'.join(tmp[:-1]) + "-" + \
+                         datetime.now().isoformat('-') + '.' + \
+                         tmp[-1]
+             print self.file
+ 
+         try:
+             import paramiko
+         except:
+             self.scp = None
 -            self.logger.warn("Could not import paramiko. SCP will not be disabled")
++            self.logger.warn("Could not import paramiko. SCP will be disabled")
+ 
+     def __call__(self, data):
+         """
+         This should be invoked every time you wish to write some
+         data to the reporting system
+         """
+         #print "Writing report(s)"
+         #dump = '--- \n'
+         dump = yaml.dump([data])
+         #dump += yaml.dump(data)
+         reports = []
+ 
+         if self.file:
+             reports.append("file")
+ 
+         if self.tcp:
+             reports.append("tcp")
+ 
+         if self.scp:
+             reports.append("scp")
+ 
+         jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
+         gevent.joinall(jobs)
+         ret = []
+         for job in jobs:
+             #print job.value
+             ret.append(job.value)
+         return ret
+ 
+     def file_report(self, data, file=None, mode='a+'):
+         """
+         This reports to a file in YAML format
+         """
+         if not file:
+             file = self.file
+         try:
+             f = open(file, mode)
+             f.write(data)
+         except Exception, e:
+             raise e
+         finally:
+             f.close()
+ 
+ 
+     def tcp_report(self, data):
+         """
+         This connect to the specified tcp server
+         and writes the data passed as argument.
+         """
+         host, port = self.tcp.split(":")
+         tcp = socket.getprotobyname('tcp')
+         send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
+         try:
+             send_socket.connect((host, int(port)))
+             send_socket.send(data)
+ 
+         except Exception, e:
+             raise e
+ 
+         finally:
+             send_socket.close()
+ 
+ 
+     def scp_report(self, data, rfile=None, mode='a+'):
+         """
+         Push data to the remote ssh server.
+ 
+         :rfile the remote filename to write
+         :data the raw data content that should be written
+         :mode in what mode the file should be created
+         """
+         if not rfile:
+             rfile = self.file
+         host, port = self.scp.split(":")
+         transport = paramiko.Transport((host, port))
+ 
+         # The remote path of the remote file to write
+         rfpath = os.path.join(self.config.ssh_rpath, rfile)
+ 
+         try:
+             username = self.config.ssh_username
+         except:
+             raise "No username provided"
+ 
+         # Load the local known host key file
+         transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+ 
+         # We prefer to use an ssh keyfile fo authentication
+         if self.config.ssh_keyfile:
+             keyfile = os.path.expanduser(self.config.ssh_keyfile)
+             key = paramiko.RSAKey.from_private_key_file(keylocfile)
+             try:
+                 transport.connect(username=username, pkey=key)
+             except Exception, e:
+                 raise e
+ 
+         # If not even a password is fine
+         elif self.config.ssh_password:
+             try:
+                 transport.connect(username=username, password=self.config.ssh_password)
+             except Exception, e:
+                 raise e
+ 
+         # ... but no authentication, that is madness!
+         else:
+             raise "No key or password provided for ssh"
+ 
+         sftp = paramiko.SFTPClient.from_transport(transport)
+         try:
+             sftp = ssh.open_sftp()
+             remote_file = sftp.file(rfile, mode)
+             remote_file.set_pipelined(True)
+             remote_file.write(data)
+ 
+         except Exception, e:
+             raise e
+         sftp.close()
+         transport.close()
+ 
+ 
+     def send_report(self, data, type):
+         """
+         This sends the report using the
+         specified type.
+         """
+         #print "Reporting %s to %s" % (data, type)
+         self.logger.info("Reporting to %s" % type)
+         getattr(self, type+"_report").__call__(data)
+ 
+

    

art＠torproject.org

tags

participants (1)