[tor-commits] [ooni-probe/master] * Organising unported code to a single directory.
isis at torproject.org
isis at torproject.org
Thu Nov 1 12:14:02 UTC 2012
commit 4b735fbff7733bd05a0b75f16306147030fd4abb
Author: Isis Lovecruft <isis at torproject.org>
Date: Thu Nov 1 09:56:10 2012 +0000
* Organising unported code to a single directory.
---
ooni/hack_this/TO_BE_PORTED | 14 +++
ooni/hack_this/dnstamper.py | 200 ++++++++++++++++++++++++++++++++++++++++++
ooni/hack_this/tcpscan.py | 84 ++++++++++++++++++
ooni/hack_this/traceroute.py | 108 +++++++++++++++++++++++
4 files changed, 406 insertions(+), 0 deletions(-)
diff --git a/ooni/hack_this/TO_BE_PORTED b/ooni/hack_this/TO_BE_PORTED
new file mode 100644
index 0000000..49ce5e0
--- /dev/null
+++ b/ooni/hack_this/TO_BE_PORTED
@@ -0,0 +1,14 @@
+
+The tests in this directory are very old, and have neither been ported to
+Twisted, nor to the new twisted.trial API framework. Although, they are not
+old in the sense of the *seriously old* OONI code which was written two years
+ago.
+
+These tests should be updated at least to use Twisted.
+
+If you want to hack on something care free, feel free to mess with these files
+because it would be difficult to not improve on them.
+
+<(A)3
+isis
+0x2cdb8b35
diff --git a/ooni/hack_this/dnstamper.py b/ooni/hack_this/dnstamper.py
new file mode 100644
index 0000000..d6f87a6
--- /dev/null
+++ b/ooni/hack_this/dnstamper.py
@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+"""
+ dnstamper
+ *********
+
+ This test resolves DNS for a list of domain names, one per line, in the
+ file specified in the ooni-config under the setting "dns_experiment". If
+ the file is top-1m.txt, the test will be run using Amazon's list of top
+ one million domains. The experimental dns servers to query should
+ be specified one per line in assets/dns_servers.txt.
+
+ The test reports censorship if the cardinality of the intersection of
+ the query result set from the control server and the query result set
+ from the experimental server is zero, which is to say, if the two sets
+ have no matching results whatsoever.
+
+ NOTE: This test frequently results in false positives due to GeoIP-based
+ load balancing on major global sites such as google, facebook, and
+ youtube, etc.
+
+ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
+ :license: see LICENSE for more details
+
+ TODO:
+ * Switch to using Twisted's DNS builtins instead of dnspython
+ *
+"""
+
+import os
+
+from twisted.names import client
+from twisted.internet import reactor
+from twisted.internet.protocol import Factory, Protocol
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from zope.interface import implements
+
+from ooni.plugoo.assets import Asset
+from ooni.plugoo.tests import ITest, OONITest
+from ooni import log
+
+class Top1MAsset(Asset):
+ """
+ Class for parsing the Alexa top-1m.txt as an asset.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ self = Asset.parse_line(self, line)
+ return line.split(',')[1].replace('\n','')
+
+class DNSTamperAsset(Asset):
+ """
+ Creates DNS testing specific Assets.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+class DNSTamperArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'],
+ ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'],
+ ['testservers', 't', None, 'Asset file of the DNS servers to test'],
+ ['resume', 'r', 0, 'Resume at this index in the asset file']]
+'''
+ def control(self, experiment_result, args):
+ print "Experiment Result:", experiment_result
+ print "Args", args
+ return experiment_result
+
+ def experiment(self, args):
+'''
+
+class DNSTamperTest(OONITest):
+ implements(IPlugin, ITest)
+
+ shortName = "DNSTamper"
+ description = "DNS censorship detection test"
+ requirements = None
+ options = DNSTamperArgs
+ blocking = False
+
+ def load_assets(self):
+ if self.local_options:
+ if self.local_options['asset']:
+ assetf = self.local_options['asset']
+ if assetf == 'top-1m.txt':
+ return {'asset': Top1MAsset(assetf)}
+ else:
+ return {'asset': DNSTamperAsset(assetf)}
+ else:
+ return {}
+
+ def lookup(self, hostname, nameserver):
+ """
+ Resolves a hostname through a DNS nameserver to the corresponding
+ IP addresses.
+ """
+ def got_result(result):
+ #self.logger.log(result)
+ print result
+ reactor.stop()
+
+ def got_failure(failure):
+ failure.printTraceback()
+ reactor.stop()
+
+ res = client.createResolver(servers=[(nameserver, 53)])
+ d = res.getHostByName(hostname)
+ d.addCallbacks(got_result, got_failure)
+
+ ## XXX MAY ALSO BE:
+ #answer = res.getAddress(servers=[('nameserver', 53)])
+
+ ret = []
+
+ for data in answer:
+ ret.append(data.address)
+
+ return ret
+
+ def reverse_lookup(self, ip, nameserver):
+ """
+ Attempt to do a reverse DNS lookup to determine if the control and exp
+ sets from a positive result resolve to the same domain, in order to
+ remove false positives due to GeoIP load balancing.
+ """
+ res = client.createResolver(servers=nameserver)
+ n = reversename.from_address(ip)
+ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
+
+ return revn
+
+ def experiment(self, *a, **kw):
+ """
+ Compares the lookup() sets of the control and experiment groups.
+ """
+ # this is just a dirty hack
+ address = kw['data'][0]
+ ns = kw['data'][1]
+
+ config = self.config
+ ctrl_ns = config.tests.dns_control_server
+
+ print "ADDRESS: %s" % address
+ print "NAMESERVER: %s" % ns
+
+ exp = self.lookup(address, ns)
+ control = self.lookup(address, ctrl_ns)
+
+ result = []
+
+ if len(set(exp) & set(control)) > 0:
+ print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
+ result = (address, ns, exp, control, False)
+ return result
+ else:
+ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
+ result = (address, ns, exp, control, True)
+
+ if config.tests.dns_reverse_lookup:
+
+ exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
+ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
+ for ip in control]
+
+ if len(set(exprevn) & set(ctrlrevn)) > 0:
+ print "Further testing has eliminated this as a false positive."
+ else:
+ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
+ return result
+
+ else:
+ print "\n"
+ return result
+
+#def run(ooni):
+# """
+# Run the test.
+# """
+# config = ooni.config
+# urls = []
+#
+# if (config.tests.dns_experiment == "top-1m.txt"):
+# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
+# config.tests.dns_experiment))
+# else:
+# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
+# config.tests.dns_experiment))
+# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
+# config.tests.dns_experiment_dns))
+#
+# assets = [dns_experiment, dns_experiment_dns]
+#
+# dnstest = DNST(ooni)
+# ooni.logger.info("Beginning dnstamper test...")
+# dnstest.run(assets, {'index': 1})
+# ooni.logger.info("Dnstamper test completed!")
+
+dnstamper = DNSTamperTest(None, None, None)
diff --git a/ooni/hack_this/tcpscan.py b/ooni/hack_this/tcpscan.py
new file mode 100644
index 0000000..b371c88
--- /dev/null
+++ b/ooni/hack_this/tcpscan.py
@@ -0,0 +1,84 @@
+"""
+ TCP Port Scanner
+ ****************
+
+ Does a TCP connect scan on the IP:port pairs.
+
+"""
+import os
+from gevent import socket
+from datetime import datetime
+import socks
+
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
+__plugoo__ = "TCP Port Scanner"
+__desc__ = "This a test template to be used to build your own tests"
+
+class TCPScanAsset(Asset):
+ """
+ This is the asset that should be used by the Test. It will
+ contain all the code responsible for parsing the asset file
+ and should be passed on instantiation to the test.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+
+class TCPScan(Test):
+ """
+ The main Test class
+ """
+
+ def experiment(self, *a, **kw):
+ """
+ Fill this up with the tasks that should be performed
+ on the "dirty" network and should be compared with the
+ control.
+ """
+ addr = kw['data']
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ res = False
+ try:
+ self.logger.debug('Doing a connection to %s' % addr)
+ s.connect((addr.split(':')[0], int(addr.split(':')[1])))
+ res = True
+ except socket.error, msg:
+ self.logger.debug('Connection failed to %s: %s' % (addr, msg))
+
+ finally:
+ s.close()
+
+ return {'Time': datetime.now(),
+ 'Address': addr,
+ 'Status': res}
+
+ def control(self):
+ """
+ Fill this up with the control related code.
+ """
+ return True
+
+def run(ooni, asset=None):
+ """
+ This is the function that will be called by OONI
+ and it is responsible for instantiating and passing
+ the arguments to the Test class.
+ """
+ config = ooni.config
+
+ # This the assets array to be passed to the run function of
+ # the test
+ if asset:
+ assets = [TCPScanAsset(asset)]
+ else:
+ assets = [TCPScanAsset(os.path.join(config.main.assetdir, \
+ "tcpscan.txt"))]
+
+ # Instantiate the Test
+ thetest = TCPScan(ooni)
+ ooni.logger.info("starting TCP Scan...")
+ # Run the test with argument assets
+ thetest.run(assets)
+ ooni.logger.info("finished.")
diff --git a/ooni/hack_this/traceroute.py b/ooni/hack_this/traceroute.py
new file mode 100644
index 0000000..e8252c1
--- /dev/null
+++ b/ooni/hack_this/traceroute.py
@@ -0,0 +1,108 @@
+try:
+ from dns import resolver
+except:
+ print "Error: dnspython is not installed (http://www.dnspython.org/)"
+import gevent
+import os
+import plugoo
+
+try:
+ import scapy
+except:
+ print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)"
+
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
+import socket
+
+__plugoo__ = "Traceroute"
+__desc__ = "Performs TTL walking tests"
+
+class TracerouteAsset(Asset):
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+
+class Traceroute(Test):
+ """A *very* quick and dirty traceroute implementation, UDP and TCP
+ """
+ def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30):
+ dest_addr = socket.gethostbyname(dst)
+ print "Doing traceroute on %s" % dst
+
+ recv = socket.getprotobyname('icmp')
+ send = socket.getprotobyname(proto)
+ ttl = 1
+ while True:
+ recv_sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, recv)
+ if proto == "tcp":
+ send_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, send)
+ else:
+ send_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, send)
+ recv_sock.settimeout(10)
+ send_sock.settimeout(10)
+
+ send_sock.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
+ recv_sock.bind(("", src_port))
+ if proto == "tcp":
+ try:
+ send_sock.settimeout(2)
+ send_sock.connect((dst, dst_port))
+ except socket.timeout:
+ pass
+
+ except Exception, e:
+ print "Error doing connect %s" % e
+ else:
+ send_sock.sendto("", (dst, dst_port))
+
+ curr_addr = None
+ try:
+ print "receiving data..."
+ _, curr_addr = recv_sock.recvfrom(512)
+ curr_addr = curr_addr[0]
+
+ except socket.error, e:
+ print "SOCKET ERROR: %s" % e
+
+ except Exception, e:
+ print "ERROR: %s" % e
+
+ finally:
+ send_sock.close()
+ recv_sock.close()
+
+ if curr_addr is not None:
+ curr_host = "%s" % curr_addr
+ else:
+ curr_host = "*"
+
+ print "%d\t%s" % (ttl, curr_host)
+
+ if curr_addr == dest_addr or ttl > max_hops:
+ break
+
+ ttl += 1
+
+
+ def experiment(self, *a, **kw):
+ # this is just a dirty hack
+ address = kw['data'][0]
+
+ self.traceroute(address)
+
+def run(ooni):
+ """Run the test"""
+ config = ooni.config
+ urls = []
+
+ traceroute_experiment = TracerouteAsset(os.path.join(config.main.assetdir, \
+ config.tests.traceroute))
+
+ assets = [traceroute_experiment]
+
+ traceroute = Traceroute(ooni)
+ ooni.logger.info("starting traceroute test")
+ traceroute.run(assets)
+ ooni.logger.info("finished")
More information about the tor-commits
mailing list