commit c7145bb3177eefcfcebb013db379f34b4c8baf35 Merge: a319db3 abfcf7c Author: Arturo Filastò hellais@torproject.org Date: Thu May 31 04:44:39 2012 +0200

Merge branch 'master' into mastermerge

Conflicts: ooni/plugoo/reports.py

HACKING | 2 +- LICENSE | 26 +++++++++ TODO | 5 ++ ooni/ooni-probe.conf | 7 ++- ooni/oonitests/dnstamper.py | 122 ++++++++++++++++++++++++++++++++++++------ ooni/oonitests/httphost.py | 5 ++- 6 files changed, 146 insertions(+), 21 deletions(-)

diff --cc ooni/ooni-probe.conf index 336af99,0000000..d95e410 mode 100644,000000..100644 --- a/ooni/ooni-probe.conf +++ b/ooni/ooni-probe.conf @@@ -1,72 -1,0 +1,77 @@@ +# ooni-probe +# +# These are the global configuration parameters necessary to +# make ooni-probe work +[main] +reportdir = reports/ +logfile = ooniprobe.log +assetdir = assets/ +testdir = oonitests/ + +loglevel = DEBUG +consoleloglevel = DEBUG +proxyaddress = 127.0.0.1:9050 + +# The following configurations are for searching for PlanetLab +# nodes, adding them to a slice, and PlanetLab general API +# authentication: +pl_username = yourusername +pl_password = yourpassword + +# These are configurations specific to the tests that should be +# run by ooni-probe +[tests] +run = dnstamper +### DNS testing related config parameters + +# This is the list of hostnames that must be looked up +dns_experiment = top-1m.txt + +# This is the dns servers to be tested +dns_experiment_dns = dns_servers.txt + +# This is the control known good DNS server - dns_control_server = 8.8.8.8 ++dns_control_server = 91.191.136.152 ++ ++# Specify whether the dnstamper test should attempt to remove ++# GeoIP-based false positives by doing a reverse DNS resolve ++# on positive results. ++dns_reverse_lookup = true + +### traceroute testing related config parameters + +# This is the list of ips to traceroute to +traceroute = example_exp_list.txt + +# This is the list of ports that should be used +# src_x,src_y,src_z|dst_x,dst_y,dst_z +traceroute_ports = 0,53,80,123,443|0,53,80,123,443 + +# The protocol to be used in the scan +traceroute_proto = UDP, TCP, ICMP + +### keyword injection related tests + +# List of keywords +keywords = keywordlist.txt + +# hosts +keywords_hosts = hostslist.txt + +# Methods to be used for testing +keyword_method = http,telnet + +### Tor bridge testing + +tor_bridges = bridgetests.txt +tor_bridges_timeout = 40 + +[report] +file = report.log +timestamp = true +#ssh = 127.0.0.1:22 +#ssh_user = theusername +#ssh_password = thepassword +#ssh_keyfile = ~/.ssh/mykey_rsa +#ssh_rpath = ~/ooni-probe/ +#tcp = "127.0.0.1:9088" diff --cc ooni/oonitests/dnstamper.py index 68be12d,0000000..498ba04 mode 100644,000000..100644 --- a/ooni/oonitests/dnstamper.py +++ b/ooni/oonitests/dnstamper.py @@@ -1,70 -1,0 +1,156 @@@ ++# -*- coding: utf-8 -*- ++""" ++ dnstamper ++ ********* ++ ++ This test resolves DNS for a list of domain names, one per line, in the ++ file specified in the ooni-config under the setting "dns_experiment". If ++ the file is top-1m.txt, the test will be run using Amazon's list of top ++ one million domains. The experimental dns servers to query should ++ be specified one per line in assets/dns_servers.txt. ++ ++ The test reports censorship if the cardinality of the intersection of ++ the query result set from the control server and the query result set ++ from the experimental server is zero, which is to say, if the two sets ++ have no matching results whatsoever. ++ ++ NOTE: This test frequently results in false positives due to GeoIP-based ++ load balancing on major global sites such as google, facebook, and ++ youtube, etc. ++ ++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft ++ :license: see LICENSE for more details ++""" ++ ++try: ++ from dns import resolver, reversename ++except: ++ print "Error: dnspython is not installed! (http://www.dnspython.org/)" +try: - from dns import resolver ++ import gevent +except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" - import gevent ++ print "Error: gevent is not installed! (http://www.gevent.org/)" ++ +import os ++ +import plugoo +from plugoo.assets import Asset +from plugoo.tests import Test + - +__plugoo__ = "DNST" +__desc__ = "DNS censorship detection test" + ++class Top1MAsset(Asset): ++ """ ++ Class for parsing top-1m.txt as an asset. ++ """ ++ def __init__(self, file=None): ++ self = Asset.__init__(self, file) ++ ++ def parse_line(self, line): ++ self = Asset.parse_line(self, line) ++ return line.split(',')[1].replace('\n','') ++ +class DNSTAsset(Asset): ++ """ ++ Creates DNS testing specific Assets. ++ """ + def __init__(self, file=None): - self = asset.__init__(self, file) ++ self = Asset.__init__(self, file) + +class DNST(Test): + def lookup(self, hostname, ns): ++ """ ++ Resolves a hostname through a DNS nameserver, ns, to the corresponding ++ IP address(es). ++ """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + answer = res.query(hostname) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + ++ def reverse_lookup(self, ip, ns): ++ """ ++ Attempt to do a reverse DNS lookup to determine if the control and exp ++ sets from a positive result resolve to the same domain, in order to ++ remove false positives due to GeoIP load balancing. ++ """ ++ res = resolver.Resolver(configure=False) ++ res.nameservers = [ns] ++ n = reversename.from_address(ip) ++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] ++ ++ return revn ++ + def experiment(self, *a, **kw): ++ """ ++ Compares the lookup() sets of the control and experiment groups. ++ """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config ++ ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) ++ control = self.lookup(address, ctrl_ns) ++ ++ result = [] + + if len(set(exp) & set(control)) > 0: - print "%s : no tampering on %s" % (address, ns) - return (address, ns, False) ++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns) ++ result = (address, ns, exp, control, False) ++ return result + else: - print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control) - return (address, ns, exp, control, True) ++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) ++ result = (address, ns, exp, control, True) ++ ++ if config.tests.dns_reverse_lookup: ++ ++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp] ++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) ++ for ip in control] ++ ++ if len(set(exprevn) & set(ctrlrevn)) > 0: ++ print "Further testing has eliminated this as a false positive." ++ else: ++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) ++ return result ++ ++ else: ++ print "\n" ++ return result + +def run(ooni): - """Run the test ++ """ ++ Run the test. + """ + config = ooni.config + urls = [] + - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ ++ if (config.tests.dns_experiment == "top-1m.txt"): ++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ else: ++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment_dns)) + + assets = [dns_experiment, dns_experiment_dns] + + dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - ++ ooni.logger.info("Beginning dnstamper test...") ++ dnstest.run(assets, {'index': 1}) ++ ooni.logger.info("Dnstamper test completed!") + diff --cc ooni/oonitests/httphost.py index 6446e1f,0000000..25adff2 mode 100644,000000..100644 --- a/ooni/oonitests/httphost.py +++ b/ooni/oonitests/httphost.py @@@ -1,132 -1,0 +1,135 @@@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). +""" +import os +from datetime import datetime +from gevent import monkey + +import urllib2 +import httplib +# WARNING! Using gevent's socket +# introduces the 0x20 DNS "feature". +# This will result is weird DNS requests +# appearing on the wire. +monkey.patch_socket() + - from BeautifulSoup import BeautifulSoup ++try: ++ from BeautifulSoup import BeautifulSoup ++except: ++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "HTTP Host" +__desc__ = "This detects HTTP Host field based filtering" + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + +class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + +def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + +