[tor-commits] [ooni-probe/master] Merge branch 'master' into mastermerge
art at torproject.org
art at torproject.org
Thu May 31 03:01:43 UTC 2012
commit c7145bb3177eefcfcebb013db379f34b4c8baf35
Merge: a319db3 abfcf7c
Author: Arturo Filastò <hellais at torproject.org>
Date: Thu May 31 04:44:39 2012 +0200
Merge branch 'master' into mastermerge
Conflicts:
ooni/plugoo/reports.py
HACKING | 2 +-
LICENSE | 26 +++++++++
TODO | 5 ++
ooni/ooni-probe.conf | 7 ++-
ooni/oonitests/dnstamper.py | 122 ++++++++++++++++++++++++++++++++++++------
ooni/oonitests/httphost.py | 5 ++-
6 files changed, 146 insertions(+), 21 deletions(-)
diff --cc ooni/ooni-probe.conf
index 336af99,0000000..d95e410
mode 100644,000000..100644
--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,72 -1,0 +1,77 @@@
+# ooni-probe
+#
+# These are the global configuration parameters necessary to
+# make ooni-probe work
+[main]
+reportdir = reports/
+logfile = ooniprobe.log
+assetdir = assets/
+testdir = oonitests/
+
+loglevel = DEBUG
+consoleloglevel = DEBUG
+proxyaddress = 127.0.0.1:9050
+
+# The following configurations are for searching for PlanetLab
+# nodes, adding them to a slice, and PlanetLab general API
+# authentication:
+pl_username = yourusername
+pl_password = yourpassword
+
+# These are configurations specific to the tests that should be
+# run by ooni-probe
+[tests]
+run = dnstamper
+### DNS testing related config parameters
+
+# This is the list of hostnames that must be looked up
+dns_experiment = top-1m.txt
+
+# This is the dns servers to be tested
+dns_experiment_dns = dns_servers.txt
+
+# This is the control known good DNS server
- dns_control_server = 8.8.8.8
++dns_control_server = 91.191.136.152
++
++# Specify whether the dnstamper test should attempt to remove
++# GeoIP-based false positives by doing a reverse DNS resolve
++# on positive results.
++dns_reverse_lookup = true
+
+### traceroute testing related config parameters
+
+# This is the list of ips to traceroute to
+traceroute = example_exp_list.txt
+
+# This is the list of ports that should be used
+# src_x,src_y,src_z|dst_x,dst_y,dst_z
+traceroute_ports = 0,53,80,123,443|0,53,80,123,443
+
+# The protocol to be used in the scan
+traceroute_proto = UDP, TCP, ICMP
+
+### keyword injection related tests
+
+# List of keywords
+keywords = keywordlist.txt
+
+# hosts
+keywords_hosts = hostslist.txt
+
+# Methods to be used for testing
+keyword_method = http,telnet
+
+### Tor bridge testing
+
+tor_bridges = bridgetests.txt
+tor_bridges_timeout = 40
+
+[report]
+file = report.log
+timestamp = true
+#ssh = 127.0.0.1:22
+#ssh_user = theusername
+#ssh_password = thepassword
+#ssh_keyfile = ~/.ssh/mykey_rsa
+#ssh_rpath = ~/ooni-probe/
+#tcp = "127.0.0.1:9088"
diff --cc ooni/oonitests/dnstamper.py
index 68be12d,0000000..498ba04
mode 100644,000000..100644
--- a/ooni/oonitests/dnstamper.py
+++ b/ooni/oonitests/dnstamper.py
@@@ -1,70 -1,0 +1,156 @@@
++# -*- coding: utf-8 -*-
++"""
++ dnstamper
++ *********
++
++ This test resolves DNS for a list of domain names, one per line, in the
++ file specified in the ooni-config under the setting "dns_experiment". If
++ the file is top-1m.txt, the test will be run using Amazon's list of top
++ one million domains. The experimental dns servers to query should
++ be specified one per line in assets/dns_servers.txt.
++
++ The test reports censorship if the cardinality of the intersection of
++ the query result set from the control server and the query result set
++ from the experimental server is zero, which is to say, if the two sets
++ have no matching results whatsoever.
++
++ NOTE: This test frequently results in false positives due to GeoIP-based
++ load balancing on major global sites such as google, facebook, and
++ youtube, etc.
++
++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
++ :license: see LICENSE for more details
++"""
++
++try:
++ from dns import resolver, reversename
++except:
++ print "Error: dnspython is not installed! (http://www.dnspython.org/)"
+try:
- from dns import resolver
++ import gevent
+except:
- print "Error dnspython is not installed! (http://www.dnspython.org/)"
- import gevent
++ print "Error: gevent is not installed! (http://www.gevent.org/)"
++
+import os
++
+import plugoo
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
-
+__plugoo__ = "DNST"
+__desc__ = "DNS censorship detection test"
+
++class Top1MAsset(Asset):
++ """
++ Class for parsing top-1m.txt as an asset.
++ """
++ def __init__(self, file=None):
++ self = Asset.__init__(self, file)
++
++ def parse_line(self, line):
++ self = Asset.parse_line(self, line)
++ return line.split(',')[1].replace('\n','')
++
+class DNSTAsset(Asset):
++ """
++ Creates DNS testing specific Assets.
++ """
+ def __init__(self, file=None):
- self = asset.__init__(self, file)
++ self = Asset.__init__(self, file)
+
+class DNST(Test):
+ def lookup(self, hostname, ns):
++ """
++ Resolves a hostname through a DNS nameserver, ns, to the corresponding
++ IP address(es).
++ """
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [ns]
+ answer = res.query(hostname)
+
+ ret = []
+
+ for data in answer:
+ ret.append(data.address)
+
+ return ret
+
++ def reverse_lookup(self, ip, ns):
++ """
++ Attempt to do a reverse DNS lookup to determine if the control and exp
++ sets from a positive result resolve to the same domain, in order to
++ remove false positives due to GeoIP load balancing.
++ """
++ res = resolver.Resolver(configure=False)
++ res.nameservers = [ns]
++ n = reversename.from_address(ip)
++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
++
++ return revn
++
+ def experiment(self, *a, **kw):
++ """
++ Compares the lookup() sets of the control and experiment groups.
++ """
+ # this is just a dirty hack
+ address = kw['data'][0]
+ ns = kw['data'][1]
+
+ config = self.config
++ ctrl_ns = config.tests.dns_control_server
+
+ print "ADDRESS: %s" % address
+ print "NAMESERVER: %s" % ns
+
+ exp = self.lookup(address, ns)
- control = self.lookup(address, config.tests.dns_control_server)
++ control = self.lookup(address, ctrl_ns)
++
++ result = []
+
+ if len(set(exp) & set(control)) > 0:
- print "%s : no tampering on %s" % (address, ns)
- return (address, ns, False)
++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
++ result = (address, ns, exp, control, False)
++ return result
+ else:
- print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control)
- return (address, ns, exp, control, True)
++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
++ result = (address, ns, exp, control, True)
++
++ if config.tests.dns_reverse_lookup:
++
++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
++ for ip in control]
++
++ if len(set(exprevn) & set(ctrlrevn)) > 0:
++ print "Further testing has eliminated this as a false positive."
++ else:
++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
++ return result
++
++ else:
++ print "\n"
++ return result
+
+def run(ooni):
- """Run the test
++ """
++ Run the test.
+ """
+ config = ooni.config
+ urls = []
+
- dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \
- config.tests.dns_experiment))
- dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \
++ if (config.tests.dns_experiment == "top-1m.txt"):
++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
++ config.tests.dns_experiment))
++ else:
++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
++ config.tests.dns_experiment))
++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
+ config.tests.dns_experiment_dns))
+
+ assets = [dns_experiment, dns_experiment_dns]
+
+ dnstest = DNST(ooni)
- ooni.logger.info("starting test")
- dnstest.run(assets)
- ooni.logger.info("finished")
-
++ ooni.logger.info("Beginning dnstamper test...")
++ dnstest.run(assets, {'index': 1})
++ ooni.logger.info("Dnstamper test completed!")
+
diff --cc ooni/oonitests/httphost.py
index 6446e1f,0000000..25adff2
mode 100644,000000..100644
--- a/ooni/oonitests/httphost.py
+++ b/ooni/oonitests/httphost.py
@@@ -1,132 -1,0 +1,135 @@@
+"""
+ HTTP Host based filtering
+ *************************
+
+ This test detect HTTP Host field
+ based filtering.
+ It is used to detect censorship on
+ performed with Web Guard (used by
+ T-Mobile US).
+"""
+import os
+from datetime import datetime
+from gevent import monkey
+
+import urllib2
+import httplib
+# WARNING! Using gevent's socket
+# introduces the 0x20 DNS "feature".
+# This will result is weird DNS requests
+# appearing on the wire.
+monkey.patch_socket()
+
- from BeautifulSoup import BeautifulSoup
++try:
++ from BeautifulSoup import BeautifulSoup
++except:
++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/"
+
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
+__plugoo__ = "HTTP Host"
+__desc__ = "This detects HTTP Host field based filtering"
+
+class HTTPHostAsset(Asset):
+ """
+ This is the asset that should be used by the Test. It will
+ contain all the code responsible for parsing the asset file
+ and should be passed on instantiation to the test.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ return line.split(',')[1].replace('\n','')
+
+class HTTPHost(Test):
+ """
+ The main Test class
+ """
+
+ def check_response(self, response):
+ soup = BeautifulSoup(response)
+ if soup.head.title.string == "Content Filtered":
+ # Response indicates censorship
+ return True
+ else:
+ # Response does not indicate censorship
+ return False
+
+
+ def is_censored(self, response):
+ if response:
+ soup = BeautifulSoup(response)
+ censored = self.check_response(response)
+ else:
+ censored = "unreachable"
+ return censored
+
+ def urllib2_test(self, control_server, host):
+ req = urllib2.Request(control_server)
+ req.add_header('Host', host)
+ try:
+ r = urllib2.urlopen(req)
+ response = r.read()
+ censored = self.is_censored(response)
+ except Exception, e:
+ censored = "Error! %s" % e
+
+ return censored
+
+ def httplib_test(self, control_server, host):
+ try:
+ conn = httplib.HTTPConnection(control_server)
+ conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
+ conn.putheader("Host", host)
+ conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
+ conn.endheaders()
+ r = conn.getresponse()
+ response = r.read()
+ censored = self.is_censored(response)
+ except Exception, e:
+ censored = "Error! %s" % e
+
+ return censored
+
+
+ def experiment(self, *a, **kw):
+ """
+ Try to connect to the control server with
+ the specified host field.
+ """
+ host = kw['data']
+ control_server = kw['control_server']
+ self.logger.info("Testing %s (%s)" % (host, control_server))
+
+ #censored = self.urllib2_test(control_server, host)
+ censored = self.httplib_test(control_server, host)
+
+ self.logger.info("%s: %s" % (host, censored))
+ return {'Time': datetime.now(),
+ 'Host': host,
+ 'Censored': censored}
+
+
+def run(ooni):
+ """
+ This is the function that will be called by OONI
+ and it is responsible for instantiating and passing
+ the arguments to the Test class.
+ """
+ config = ooni.config
+
+ # This the assets array to be passed to the run function of
+ # the test
+ assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
+ "top-1m.csv"))]
+
+ # Instantiate the Test
+ thetest = HTTPHost(ooni)
+ ooni.logger.info("starting HTTP Host Test...")
+ # Run the test with argument assets
+ thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
+ ooni.logger.info("finished.")
+
+
More information about the tor-commits
mailing list