commit 1f5c521ebe1bededb16d70082ae4f4617df0785e Author: Isis Lovecruft isis@patternsinthevoid.net Date: Sat Mar 24 01:07:41 2012 -0700

Completed attempt to use reverse DNS to account for geoIP load balancing, added option in ooni-conf to disable this feature. --- HACKING | 2 +- ooni-probe.conf | 7 +++- plugoo/reports.py | 2 +- tests/dnstamper.py | 110 ++++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 102 insertions(+), 19 deletions(-)

diff --git a/HACKING b/HACKING index 57b6c9b..4451882 100644 --- a/HACKING +++ b/HACKING @@ -170,7 +170,7 @@ Method definitions inside of class are separated by a single blank line. Encoding ........

-Always use UTF-8 encoding. This can be specified by add the encoding cookie +Always use UTF-8 encoding. This can be specified by adding the encoding cookie to the beginning of your python files:

# -*- coding: UTF-8 -*- diff --git a/ooni-probe.conf b/ooni-probe.conf index 8702353..161a008 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -31,7 +31,12 @@ dns_experiment = top-1m.txt dns_experiment_dns = dns_servers.txt

# This is the control known good DNS server -dns_control_server = 8.8.8.8 +dns_control_server = 91.191.136.152 + +# Specify whether the dnstamper test should attempt to remove +# GeoIP-based false positives by doing a reverse DNS resolve +# on positive results. +dns_reverse_lookup = true

### traceroute testing related config parameters

diff --git a/plugoo/reports.py b/plugoo/reports.py index 90de498..009e86b 100644 --- a/plugoo/reports.py +++ b/plugoo/reports.py @@ -42,7 +42,7 @@ class Report: import paramiko except: self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") + self.logger.warn("Could not import paramiko. SCP will be disabled")

def __call__(self, data): """ diff --git a/tests/dnstamper.py b/tests/dnstamper.py index 4064831..4fc3c71 100644 --- a/tests/dnstamper.py +++ b/tests/dnstamper.py @@ -1,9 +1,38 @@ +# -*- coding: utf-8 -*- +""" + dnstamper + ********* + + This test resolves DNS for a list of domain names, one per line, in the + file specified in the ooni-config under the setting "dns_experiment". If + the file is top-1m.txt, the test will be run using Amazon's list of top + one million domains. The experimental dns servers to query should + be specified one per line in assets/dns_servers.txt. + + The test reports censorship if the cardinality of the intersection of + the query result set from the control server and the query result set + from the experimental server is zero, which is to say, if the two sets + have no matching results whatsoever. + + NOTE: This test frequently results in false positives due to GeoIP-based + load balancing on major global sites such as google, facebook, and + youtube, etc. + + :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details +""" + +try: + from dns import resolver, reversename +except: + print "Error: dnspython is not installed! (http://www.dnspython.org/)" try: - from dns import resolver + import gevent except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" -import gevent + print "Error: gevent is not installed! (http://www.gevent.org/)" + import os + import plugoo from plugoo.assets import Asset from plugoo.tests import Test @@ -12,6 +41,9 @@ __plugoo__ = "DNST" __desc__ = "DNS censorship detection test"

class Top1MAsset(Asset): + """ + Class for parsing top-1m.txt as an asset. + """ def __init__(self, file=None): self = Asset.__init__(self, file)

@@ -20,11 +52,18 @@ class Top1MAsset(Asset): return line.split(',')[1].replace('\n','')

class DNSTAsset(Asset): + """ + Creates DNS testing specific Assets. + """ def __init__(self, file=None): self = Asset.__init__(self, file)

class DNST(Test): def lookup(self, hostname, ns): + """ + Resolves a hostname through a DNS nameserver, ns, to the corresponding + IP address(es). + """ res = resolver.Resolver(configure=False) res.nameservers = [ns] answer = res.query(hostname) @@ -36,42 +75,81 @@ class DNST(Test):

return ret

+ def reverse_lookup(self, ip, ns): + """ + Attempt to do a reverse DNS lookup to determine if the control and exp + sets from a positive result resolve to the same domain, in order to + remove false positives due to GeoIP load balancing. + """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + n = reversename.from_address(ip) + revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] + + return revn + def experiment(self, *a, **kw): + """ + Compares the lookup() sets of the control and experiment groups. + """ # this is just a dirty hack address = kw['data'][0] ns = kw['data'][1]

config = self.config + ctrl_ns = config.tests.dns_control_server

print "ADDRESS: %s" % address print "NAMESERVER: %s" % ns

exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) + control = self.lookup(address, ctrl_ns) + + result = []

if len(set(exp) & set(control)) > 0: print "Address %s has not tampered with on DNS server %s\n" % (address, ns) - return (address, ns, False) + result = (address, ns, exp, control, False) + return result else: - print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s\n" % (address, ns, ns, exp, control) - return (address, ns, exp, control, True) + print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) + result = (address, ns, exp, control, True) + + if config.tests.dns_reverse_lookup: + + exprevn = [self.reverse_lookup(ip, ns) for ip in exp] + ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) + for ip in control] + + if len(set(exprevn) & set(ctrlrevn)) > 0: + print "Further testing has eliminated this as a false positive." + else: + print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) + return result + + else: + print "\n" + return result

def run(ooni): - """Run the test + """ + Run the test. """ config = ooni.config urls = []

- dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ + if (config.tests.dns_experiment == "top-1m.txt"): + dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment)) + else: + dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment)) + dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, config.tests.dns_experiment_dns))

assets = [dns_experiment, dns_experiment_dns]

dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - - + ooni.logger.info("Beginning dnstamper test...") + dnstest.run(assets, {'index': 1}) + ooni.logger.info("Dnstamper test completed!")