commit 4b735fbff7733bd05a0b75f16306147030fd4abb Author: Isis Lovecruft isis@torproject.org Date: Thu Nov 1 09:56:10 2012 +0000

* Organising unported code to a single directory. --- ooni/hack_this/TO_BE_PORTED | 14 +++ ooni/hack_this/dnstamper.py | 200 ++++++++++++++++++++++++++++++++++++++++++ ooni/hack_this/tcpscan.py | 84 ++++++++++++++++++ ooni/hack_this/traceroute.py | 108 +++++++++++++++++++++++ 4 files changed, 406 insertions(+), 0 deletions(-)

diff --git a/ooni/hack_this/TO_BE_PORTED b/ooni/hack_this/TO_BE_PORTED new file mode 100644 index 0000000..49ce5e0 --- /dev/null +++ b/ooni/hack_this/TO_BE_PORTED @@ -0,0 +1,14 @@ + +The tests in this directory are very old, and have neither been ported to +Twisted, nor to the new twisted.trial API framework. Although, they are not +old in the sense of the *seriously old* OONI code which was written two years +ago. + +These tests should be updated at least to use Twisted. + +If you want to hack on something care free, feel free to mess with these files +because it would be difficult to not improve on them. + +<(A)3 +isis +0x2cdb8b35 diff --git a/ooni/hack_this/dnstamper.py b/ooni/hack_this/dnstamper.py new file mode 100644 index 0000000..d6f87a6 --- /dev/null +++ b/ooni/hack_this/dnstamper.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +""" + dnstamper + ********* + + This test resolves DNS for a list of domain names, one per line, in the + file specified in the ooni-config under the setting "dns_experiment". If + the file is top-1m.txt, the test will be run using Amazon's list of top + one million domains. The experimental dns servers to query should + be specified one per line in assets/dns_servers.txt. + + The test reports censorship if the cardinality of the intersection of + the query result set from the control server and the query result set + from the experimental server is zero, which is to say, if the two sets + have no matching results whatsoever. + + NOTE: This test frequently results in false positives due to GeoIP-based + load balancing on major global sites such as google, facebook, and + youtube, etc. + + :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details + + TODO: + * Switch to using Twisted's DNS builtins instead of dnspython + * +""" + +import os + +from twisted.names import client +from twisted.internet import reactor +from twisted.internet.protocol import Factory, Protocol +from twisted.python import usage +from twisted.plugin import IPlugin +from zope.interface import implements + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni import log + +class Top1MAsset(Asset): + """ + Class for parsing the Alexa top-1m.txt as an asset. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.split(',')[1].replace('\n','') + +class DNSTamperAsset(Asset): + """ + Creates DNS testing specific Assets. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + +class DNSTamperArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], + ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], + ['testservers', 't', None, 'Asset file of the DNS servers to test'], + ['resume', 'r', 0, 'Resume at this index in the asset file']] +''' + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): +''' + +class DNSTamperTest(OONITest): + implements(IPlugin, ITest) + + shortName = "DNSTamper" + description = "DNS censorship detection test" + requirements = None + options = DNSTamperArgs + blocking = False + + def load_assets(self): + if self.local_options: + if self.local_options['asset']: + assetf = self.local_options['asset'] + if assetf == 'top-1m.txt': + return {'asset': Top1MAsset(assetf)} + else: + return {'asset': DNSTamperAsset(assetf)} + else: + return {} + + def lookup(self, hostname, nameserver): + """ + Resolves a hostname through a DNS nameserver to the corresponding + IP addresses. + """ + def got_result(result): + #self.logger.log(result) + print result + reactor.stop() + + def got_failure(failure): + failure.printTraceback() + reactor.stop() + + res = client.createResolver(servers=[(nameserver, 53)]) + d = res.getHostByName(hostname) + d.addCallbacks(got_result, got_failure) + + ## XXX MAY ALSO BE: + #answer = res.getAddress(servers=[('nameserver', 53)]) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + + def reverse_lookup(self, ip, nameserver): + """ + Attempt to do a reverse DNS lookup to determine if the control and exp + sets from a positive result resolve to the same domain, in order to + remove false positives due to GeoIP load balancing. + """ + res = client.createResolver(servers=nameserver) + n = reversename.from_address(ip) + revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] + + return revn + + def experiment(self, *a, **kw): + """ + Compares the lookup() sets of the control and experiment groups. + """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config + ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) + control = self.lookup(address, ctrl_ns) + + result = [] + + if len(set(exp) & set(control)) > 0: + print "Address %s has not tampered with on DNS server %s\n" % (address, ns) + result = (address, ns, exp, control, False) + return result + else: + print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) + result = (address, ns, exp, control, True) + + if config.tests.dns_reverse_lookup: + + exprevn = [self.reverse_lookup(ip, ns) for ip in exp] + ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) + for ip in control] + + if len(set(exprevn) & set(ctrlrevn)) > 0: + print "Further testing has eliminated this as a false positive." + else: + print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) + return result + + else: + print "\n" + return result + +#def run(ooni): +# """ +# Run the test. +# """ +# config = ooni.config +# urls = [] +# +# if (config.tests.dns_experiment == "top-1m.txt"): +# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# else: +# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment_dns)) +# +# assets = [dns_experiment, dns_experiment_dns] +# +# dnstest = DNST(ooni) +# ooni.logger.info("Beginning dnstamper test...") +# dnstest.run(assets, {'index': 1}) +# ooni.logger.info("Dnstamper test completed!") + +dnstamper = DNSTamperTest(None, None, None) diff --git a/ooni/hack_this/tcpscan.py b/ooni/hack_this/tcpscan.py new file mode 100644 index 0000000..b371c88 --- /dev/null +++ b/ooni/hack_this/tcpscan.py @@ -0,0 +1,84 @@ +""" + TCP Port Scanner + **************** + + Does a TCP connect scan on the IP:port pairs. + +""" +import os +from gevent import socket +from datetime import datetime +import socks + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "TCP Port Scanner" +__desc__ = "This a test template to be used to build your own tests" + +class TCPScanAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + +class TCPScan(Test): + """ + The main Test class + """ + + def experiment(self, *a, **kw): + """ + Fill this up with the tasks that should be performed + on the "dirty" network and should be compared with the + control. + """ + addr = kw['data'] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + res = False + try: + self.logger.debug('Doing a connection to %s' % addr) + s.connect((addr.split(':')[0], int(addr.split(':')[1]))) + res = True + except socket.error, msg: + self.logger.debug('Connection failed to %s: %s' % (addr, msg)) + + finally: + s.close() + + return {'Time': datetime.now(), + 'Address': addr, + 'Status': res} + + def control(self): + """ + Fill this up with the control related code. + """ + return True + +def run(ooni, asset=None): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + if asset: + assets = [TCPScanAsset(asset)] + else: + assets = [TCPScanAsset(os.path.join(config.main.assetdir, \ + "tcpscan.txt"))] + + # Instantiate the Test + thetest = TCPScan(ooni) + ooni.logger.info("starting TCP Scan...") + # Run the test with argument assets + thetest.run(assets) + ooni.logger.info("finished.") diff --git a/ooni/hack_this/traceroute.py b/ooni/hack_this/traceroute.py new file mode 100644 index 0000000..e8252c1 --- /dev/null +++ b/ooni/hack_this/traceroute.py @@ -0,0 +1,108 @@ +try: + from dns import resolver +except: + print "Error: dnspython is not installed (http://www.dnspython.org/)" +import gevent +import os +import plugoo + +try: + import scapy +except: + print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)" + +from plugoo.assets import Asset +from plugoo.tests import Test + +import socket + +__plugoo__ = "Traceroute" +__desc__ = "Performs TTL walking tests" + +class TracerouteAsset(Asset): + def __init__(self, file=None): + self = Asset.__init__(self, file) + + +class Traceroute(Test): + """A *very* quick and dirty traceroute implementation, UDP and TCP + """ + def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30): + dest_addr = socket.gethostbyname(dst) + print "Doing traceroute on %s" % dst + + recv = socket.getprotobyname('icmp') + send = socket.getprotobyname(proto) + ttl = 1 + while True: + recv_sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, recv) + if proto == "tcp": + send_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, send) + else: + send_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, send) + recv_sock.settimeout(10) + send_sock.settimeout(10) + + send_sock.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl) + recv_sock.bind(("", src_port)) + if proto == "tcp": + try: + send_sock.settimeout(2) + send_sock.connect((dst, dst_port)) + except socket.timeout: + pass + + except Exception, e: + print "Error doing connect %s" % e + else: + send_sock.sendto("", (dst, dst_port)) + + curr_addr = None + try: + print "receiving data..." + _, curr_addr = recv_sock.recvfrom(512) + curr_addr = curr_addr[0] + + except socket.error, e: + print "SOCKET ERROR: %s" % e + + except Exception, e: + print "ERROR: %s" % e + + finally: + send_sock.close() + recv_sock.close() + + if curr_addr is not None: + curr_host = "%s" % curr_addr + else: + curr_host = "*" + + print "%d\t%s" % (ttl, curr_host) + + if curr_addr == dest_addr or ttl > max_hops: + break + + ttl += 1 + + + def experiment(self, *a, **kw): + # this is just a dirty hack + address = kw['data'][0] + + self.traceroute(address) + +def run(ooni): + """Run the test""" + config = ooni.config + urls = [] + + traceroute_experiment = TracerouteAsset(os.path.join(config.main.assetdir, \ + config.tests.traceroute)) + + assets = [traceroute_experiment] + + traceroute = Traceroute(ooni) + ooni.logger.info("starting traceroute test") + traceroute.run(assets) + ooni.logger.info("finished")