- tor-commits - lists.torproject.org

[ooni-probe/master] Still sorting out that darned resolver...
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 13077468d151d931c4de38961be3dbb76a4eb88c Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Jul 28 01:33:23 2012 +0000 Still sorting out that darned resolver... --- ooni/plugins/dnstamper.py | 212 ++++++++++++++++++++++++++------------------- 1 files changed, 121 insertions(+), 91 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index e768dab..8e8563a 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -22,10 +22,10 @@ :license: see LICENSE for more details TODO: - * Use OONI log instead of print * Finish porting to twisted * Finish the client.Resolver() subclass and test it * Use the DNS tests from captiveportal + * Use plugoo/reports.py for final data """ import os @@ -56,11 +56,11 @@ class AlexaAsset(Asset): class DNSTamperArgs(usage.Options): optParameters = [['hostnames', 'h', None, 'Asset file of hostnames to resolve'], - ['controlserver', 'c', '8.8.8.8', + ['controlresolver', 'c', '8.8.8.8', 'Known good DNS server'], - ['testservers', 't', None, + ['testresolvers', 't', None, 'Asset file of DNS servers to test'], - ['localservers', 'l', False, + ['localresolvers', 'l', False, 'Also test local servers'], ['port', 'p', None, 'Local UDP port to send queries over'], @@ -120,6 +120,9 @@ class DNSTamperResolver(client.Resolver): return proto class DNSTamperTest(OONITest): + """ + XXX fill me in + """ implements(IPlugin, ITest) shortName = "dnstamper" @@ -127,136 +130,163 @@ class DNSTamperTest(OONITest): requirements = None options = DNSTamperArgs blocking = False - - #def __init__(self, local_options, global_options, - # report, ooninet=None, reactor=None): - # super(DNSTamperTest, self).__init__(local_options, global_options, - # report, ooninet, reactor) - # - # if self.reactor is None: - # self.reactor = reactor - # - # if self.local_options: - # if self.local_options['localservers']: - # ## client.createResolver() turns None into '/etc/resolv.conf' - # ## on posix systems, ignored on Windows. - # self.resolvconf = None - # else: - # self.resolvconf = '' + + def __init__(self, local_options, global_options, + report, ooninet=None, reactor=None): + super(DNSTamperTest, self).__init__(local_options, global_options, + report, ooninet, reactor) + + def __repr__(self): + represent = "DNSTamperTest(OONITest): local_options=%r, " \ + "global_options=%r, assets=%r" % (self.local_options, + self.global_options, + self.assets) + return represent def initialize(self): if self.local_options: - ## client.createResolver() turns 'None' into '/etc/resolv.conf' on ## posix systems, ignored on Windows. - if self.local_options['localservers']: + if self.local_options['localresolvers']: self.resolvconf = None else: self.resolvconf = '' - else: - pass - - self.d = defer.Deferred() def load_assets(self): assets = {} - if self.local_options: + default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', + 'ooni.nu', 'google.com', 'torproject.org'] + default_resolvers = ['209.244.0.3', '208.67.222.222'] - if self.local_options['hostnames']: - assetf = self.local_options['hostnames'] - if assetf == 'top-1m.txt': - assets['hostnames'] = AlexaAsset(assetf) - #assets.update({'asset': AlexaAsset(assetf)}) - else: - assets['hostnames'] = Asset(assetf) - #assets.update({'asset': Asset(assetf)}) + def asset_file(asset_option): + return self.local_options[asset_option] + + def list_to_asset(list_): + def next(list_): + host = list_.pop() + if host is not None: + yield str(host) + while len(list_) > 0: + next(list_) + + if self.local_options: + if asset_file('hostnames'): + with asset_file('hostnames') as hosts_file: + ## The default filename for the Alexa Top 1 Million: + if hosts_file is 'top-1m.txt': + assets.update({'hostnames': AlexaAsset(hosts_file)}) + else: + assets.update({'hostnames': Asset(hosts_file)}) else: - #default_hostnames = ['google.com', 'torrentz.eu', 'ooni.nu', - # 'twitter.com', 'baidu.com'] - #assets.update({'asset': [host for host in default_hostnames]}) - print "Error! We need a file containing the hostnames that we should test DNS for!" - - if self.local_options['testservers']: - #assets['testservers'] = Asset(self.local_options['testservers']) - self.testservers = Asset(self.local_options['testservers']) + log.msg("Error! We need an asset file containing the " + + "hostnames that we should test DNS with! Please use " + + "the '-h' option. Using pre-defined hostnames...") + assets.update({'hostnames': list_to_asset(default_hostnames)}) + + if asset_file('testresolvers'): + with asset_file('testresolvers') as resolver_file: + assets.update({'testresolvers': Asset(resolver_file)}) else: - self.testservers = ['209.244.0.3', '208.67.222.222', '156.154.70.1'] + assets.update({'testresolvers': + list_to_asset(default_resolvers)}) return assets - def lookup(self, hostname, nameserver): + def lookup(self, hostname, resolver): """ - Resolves a hostname through a DNS nameserver to the corresponding - IP addresses. + Resolves a hostname through a DNS nameserver to the corresponding IP + addresses. """ - def got_result(result): + def got_result(result, hostname, resolver): + ## XXX is there a report class that we should be using? log.msg('Resolved %s through %s to %s' - % (hostname, nameserver, result)) - return {'resolved': True, - 'domain': hostname, - 'nameserver': nameserver, - 'address': result} - - def got_error(err): + % (hostname, resolver, result)) + outcome = {'resolved': True, + 'domain': hostname, + 'nameserver': resolver, + 'address': result } + log.msg(outcome) + return result + + def got_error(err, hostname, resolver): log.msg(err.printTraceback()) - return {'resolved': False, - 'domain': hostname, - 'nameserver': nameserver, - 'address': err} + outcome = {'resolved': False, + 'domain': hostname, + 'nameserver': resolver, + 'address': err } + log.msg(outcome) + return err res = client.createResolver(resolvconf=self.resolvconf, - servers=[(nameserver, 53)]) - d = res.getHostByName(hostname) - d.addCallbacks(got_result, got_error) + servers=[(resolver, 53)]) + + ## XXX should we do self.d.addCallback(resHostByName, hostname)? + #d = res.getHostByName(hostname) + #d.addCallbacks(got_result, got_error) + + #d = defer.Deferred() + #d.addCallback(res.getHostByName, hostname) + + #d = res.getHostByName(hostname) + #d.addCallback(got_result, result, hostname, resolver) + #d.addErrback(got_error, err, hostname, resolver) + + res.addCallback(getHostByName, hostname) + res.addCallback(got_result, result, hostname, resolver) + res.addErrback(got_error, err, hostname, resolver) + + if self.local_options['usereverse']: + #d.addCallback(self.reverse_lookup, result, resolver) + #d.addErrback(log.msg(err.printTraceback())) + + #d.addCallback(self.reverse_lookup, result, resolver) + #d.addErrback(log.msg(err.printTraceback())) + + res.addCallback(self.reverse_lookup, result, resolver) + res.addErraback(log.msg(err.printTraceback())) - return d + return res - def reverse_lookup(self, address, nameserver): + def reverse_lookup(self, address, resolver): """ Attempt to do a reverse DNS lookup to determine if the control and exp sets from a positive result resolve to the same domain, in order to remove false positives due to GeoIP load balancing. """ res = client.createResolver(resolvconf=self.resolvconf, - servers=[(nameserver, 53)]) + servers=[(resolver, 53)]) ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa' - d = res.lookupPointer(ptr) - d.addCallback(lambda (ans, auth, add): util.println(ans[0].payload.name)) - d.addErrback(log.err) - d.addBoth(lambda r: reactor.stop()) - return d + reverse = res.lookupPointer(ptr) + reverse.addCallback(lambda (address, auth, add): + util.println(address[0].payload.name)) + reverse.addErrback(log.err) + + ## XXX do we need to stop the reactor? + #d.addBoth(lambda r: reactor.stop()) + + return reverse def experiment(self, args): """ Compares the lookup() sets of the control and experiment groups. """ - hostnames = args + for hostname in args: + for testresolver in self.assets['testresolvers']: + addressd = defer.Deferred() + addressd.addCallback(self.lookup, hostname, testresolver) + addressd.addErrback(log.err) - for hostname in hostnames: - for testserver in self.testservers: - #exp_address = self.lookup(hostname, testserver) - self.d.addCallback(self.lookup, hostname, testserver) + #addressd = self.lookup(hostname, testresolver) - #print self.assets['hostnames'] - #hostname = args - #exp_address = self.lookup(hostname, testserver) + #self.d.addCallback(self.lookup, hostname, testserver) - #return {'control': control_server, - # 'domain': args['asset'], - # 'experiment_address': address} + print "%s" % type(addressd) - if self.local_options['usereverse']: - exp_reversed = self.reverse_lookup(exp_address, testserver) - - ## XXX trying to fix errors: - #d = defer.Deferred() - - return (exp_address, hostname, testserver, exp_reversed) - else: - return (exp_address, hostname, testserver, False) + return addressd - def control(self, experiment_result): + def control(self, experiment_result, args): + print "EXPERIMENT RESULT IS %s" % experiment_result (exp_address, hostname, testserver, exp_reversed) = experiment_result control_server = self.local_options['controlserver'] ctrl_address = self.lookup(hostname, control_server)

1 0

[ooni-probe/master] Updating the ooni/lib structure and git submodules for imports, and I'm trying
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 4dcc00fdefef051ab27a03df94fa96730ece581b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 28 12:02:44 2012 +0000 Updating the ooni/lib structure and git submodules for imports, and I'm trying to keep the syntax the same. Updated the README with info on dev interaction with and user cloning of submodules. --- .gitmodules | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 07a4c56..daddf8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "ooni/lib/txscapy"] path = ooni/lib/txscapy - url = git://github.com/hellais/txscapy.git + url = https://github.com/isislovecruft/txscapy.git [submodule "ooni/lib/txtraceroute"] path = ooni/lib/txtraceroute - url = git://github.com/hellais/txtraceroute.git + url = https://github.com/isislovecruft/txtraceroute.git [submodule "ooni/lib/txtorcon"] path = ooni/lib/txtorcon - url = git://github.com/meejah/txtorcon.git + url = https://github.com/isislovecruft/txtorcon.git

1 0

[ooni-probe/master] Still getting the error from ../plugoo/test.py that "d" is a NoneType.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 2004db08a6c8f2c93bf3485864a3d6eba93b8ad0 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sun Aug 26 09:53:14 2012 +0000 Still getting the error from ../plugoo/test.py that "d" is a NoneType. I'll try merging master and see if that fixes it, else I'll have to patch that before continuing. --- ooni/plugins/dnstamper.py | 42 +++++++++++++++++++++++------------------- 1 files changed, 23 insertions(+), 19 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index 8e8563a..54cfbdf 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -154,10 +154,10 @@ class DNSTamperTest(OONITest): def load_assets(self): assets = {} - - default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', - 'ooni.nu', 'google.com', 'torproject.org'] - default_resolvers = ['209.244.0.3', '208.67.222.222'] + + #default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', + # 'ooni.nu', 'google.com', 'torproject.org'] + #default_resolvers = ['209.244.0.3', '208.67.222.222'] def asset_file(asset_option): return self.local_options[asset_option] @@ -172,24 +172,25 @@ class DNSTamperTest(OONITest): if self.local_options: if asset_file('hostnames'): - with asset_file('hostnames') as hosts_file: - ## The default filename for the Alexa Top 1 Million: - if hosts_file is 'top-1m.txt': - assets.update({'hostnames': AlexaAsset(hosts_file)}) - else: - assets.update({'hostnames': Asset(hosts_file)}) + ## The default filename for the Alexa Top 1 Million: + if assets_file('hostnames') is 'top-1m.txt': + assets.update({'hostnames': + AlexaAsset(assets_file('hostnames'))}) + else: + assets.update({'hostnames': + Asset(assets_file('hostnames'))}) else: log.msg("Error! We need an asset file containing the " + "hostnames that we should test DNS with! Please use " + "the '-h' option. Using pre-defined hostnames...") - assets.update({'hostnames': list_to_asset(default_hostnames)}) + #assets.update({'hostnames': list_to_asset(default_hostnames)}) if asset_file('testresolvers'): - with asset_file('testresolvers') as resolver_file: - assets.update({'testresolvers': Asset(resolver_file)}) - else: assets.update({'testresolvers': - list_to_asset(default_resolvers)}) + Asset(asset_file('testresolvers'))}) + #else: + # assets.update({'testresolvers': + # list_to_asset(asset_file('testresolvers'))}) return assets @@ -273,9 +274,12 @@ class DNSTamperTest(OONITest): """ for hostname in args: for testresolver in self.assets['testresolvers']: - addressd = defer.Deferred() - addressd.addCallback(self.lookup, hostname, testresolver) - addressd.addErrback(log.err) + #addressd = defer.Deferred() + #addressd.addCallback(self.lookup, hostname, testresolver) + #addressd.addErrback(log.err) + + self.d.addCallback(self.lookup, hostname, testresolver) + self.d.addErrback(log.err) #addressd = self.lookup(hostname, testresolver) @@ -283,7 +287,7 @@ class DNSTamperTest(OONITest): print "%s" % type(addressd) - return addressd + return self.d def control(self, experiment_result, args): print "EXPERIMENT RESULT IS %s" % experiment_result

1 0

[ooni-probe/master] Actually, everything is not working. I started to fix it, but I need to work
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 9c5d5d4b39babb714df86089e87db36af6562085 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 30 01:04:53 2012 +0000 Actually, everything is not working. I started to fix it, but I need to work on bridget, so I'm going to clone tpo:ooni-probe.git into a new branch now. --- ooni/ooniprobe.py | 6 +++--- ooni/plugoo/tests.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 7e9eecc..e60b956 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -27,9 +27,9 @@ from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation -from plugoo import tests, work, assets, reports -from logo import getlogo -import plugins, log +from ooni.plugoo import tests, work, assets, reports +from ooni.logo import getlogo +from ooni import plugins, log __version__ = "0.0.1-prealpha" diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 544d82d..4be0b62 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -7,7 +7,7 @@ import itertools from twisted.internet import reactor, defer, threads from twisted.python import failure -import log +from ooni import log from plugoo import assets, work from plugoo.reports import Report

1 0

[ooni-probe/master] Changed the ooni/lib structure to use gitsubmodules so that OONI users do not
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 78f768540a0912619b566e1f75e518edcb29b7ff Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Aug 27 04:50:54 2012 +0000 Changed the ooni/lib structure to use gitsubmodules so that OONI users do not accidentally git-clone broken or incompatible upstreams. Each submodule has it's own .git with both hellais' and my forks configured, though each submodule is also configured with a special HEAD state so that unneeded files are not downloaded from upstream (this seems better than git-/hg- cloning an unknown version of a repo and copying out the files we do want). I still need to update the ooni-probe imports to reflect the slight directory structure change. Also, I rm'd the Makefile. Now, new OONI users should do a "git submodule init && git submodule update"...I'll probably write a script for it. I will add documentation for what to do with committed changes in submodule directories, so that OONI devs can make sure that users get the right versions. --- .gitmodules | 9 ++ ooni/lib/Makefile | 30 ---- ooni/lib/txscapy | 1 + ooni/lib/txscapy.py | 363 ------------------------------------------------- ooni/lib/txtorcon | 1 + ooni/lib/txtraceroute | 1 + 6 files changed, 12 insertions(+), 393 deletions(-) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..07a4c56 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "ooni/lib/txscapy"] + path = ooni/lib/txscapy + url = git://github.com/hellais/txscapy.git +[submodule "ooni/lib/txtraceroute"] + path = ooni/lib/txtraceroute + url = git://github.com/hellais/txtraceroute.git +[submodule "ooni/lib/txtorcon"] + path = ooni/lib/txtorcon + url = git://github.com/meejah/txtorcon.git diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile deleted file mode 100644 index 3b0c922..0000000 --- a/ooni/lib/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -all: txtorcon txtraceroute - -txtraceroute: - echo "Processing dependency txtraceroute..." - git clone https://github.com/hellais/txtraceroute.git txtraceroute.git - mv txtraceroute.git/txtraceroute.py txtraceroute.py - rm -rf txtraceroute.git - -txtorcon: - echo "Processing dependency txtorcon..." - git clone https://github.com/meejah/txtorcon.git txtorcon.git - mv txtorcon.git/txtorcon txtorcon - rm -rf txtorcon.git - -clean: - rm -rf txtorcon - rm -rf txtraceroute.py - -#txscapy: -# echo "Processing dependency txscapy" -# git clone https://github.com/hellais/txscapy.git txscapy.git -# mv txscapy.git/txscapy.py txscapy.py -# rm -rf txscapy.git - -#rfc3339: -# echo "Processing RFC3339 dependency" -# hg clone https://bitbucket.org/henry/rfc3339 rfc3339 -# mv rfc3339/rfc3339.py rfc3339.py -# rm -rf rfc3339 - diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy new file mode 160000 index 0000000..19fb281 --- /dev/null +++ b/ooni/lib/txscapy @@ -0,0 +1 @@ +Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py deleted file mode 100644 index 8996f75..0000000 --- a/ooni/lib/txscapy.py +++ /dev/null @@ -1,363 +0,0 @@ -# -*- coding:utf8 -*- -""" - txscapy - ****** - (c) 2012 Arturo Filastò - a twisted wrapper for scapys send and receive functions. - - This software has been written to be part of OONI, the Open Observatory of - Network Interference. More information on that here: http://ooni.nu/ - -""" - -import struct -import socket -import os -import sys -import time - -from twisted.internet import protocol, base, fdesc, error, defer -from twisted.internet import reactor, threads -from twisted.python import log -from zope.interface import implements - -from scapy.all import Gen -from scapy.all import SetGen - -LINUX=sys.platform.startswith("linux") -OPENBSD=sys.platform.startswith("openbsd") -FREEBSD=sys.platform.startswith("freebsd") -NETBSD=sys.platform.startswith("netbsd") -DARWIN=sys.platform.startswith("darwin") -SOLARIS=sys.platform.startswith("sunos") -WINDOWS=sys.platform.startswith("win32") - -from scapy.all import RawPcapWriter, MTU, BasePacketList, conf -class PcapWriter(RawPcapWriter): - def __init__(self, filename, linktype=None, gz=False, endianness="", - append=False, sync=False): - RawPcapWriter.__init__(self, filename, linktype=None, gz=False, - endianness="", append=False, sync=False) - fdesc.setNonBlocking(self.f) - - def _write_header(self, pkt): - if self.linktype == None: - if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList): - pkt = pkt[0] - try: - self.linktype = conf.l2types[pkt.__class__] - except KeyError: - self.linktype = 1 - RawPcapWriter._write_header(self, pkt) - - def _write_packet(self, packet): - sec = int(packet.time) - usec = int(round((packet.time-sec)*1000000)) - s = str(packet) - caplen = len(s) - RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen) - -class ScapySocket(object): - MTU = 1500 - def __init__(self, filter=None, iface=None, nofilter=None): - from scapy.all import conf - self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - - def fileno(self): - return self.ssocket.ins.fileno() - - def send(self, data): - return self.ssocket.send(data) - - def recv(self): - if FREEBSD or DARWIN: - return self.ssocket.nonblock_recv() - else: - return self.ssocket.recv(self.MTU) - -class Scapy(object): - """ - A twisted based wrapper for scapy send and receive functionality. - - It sends packets inside of a threadpool and receives packets using the - libdnet receive non blocking file descriptor. - """ - min = 2 - max = 6 - debug = True - write_only_answers = False - pcapwriter = None - recv = False - timeout = None - - def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, - iface=None, nofilter=None, pcapfile=None, timeout=None, *args, **kw): - - self.timeout = timeout - self.last_answer = None - if self.debug: - log.startLogging(sys.stdout) - - self.maxPacketSize = maxPacketSize - if not reactor: - from twisted.internet import reactor - - self._reactor = reactor - - if pkts: - self._buildPacketQueues(pkts) - self._buildSocket() - - self.cthreads = 0 - self.mthreads = 80 - - self.running = False - self.done = False - self.finished = False - - import thread - from twisted.python import threadpool - self.threadID = thread.get_ident - self.threadpool = threadpool.ThreadPool(self.min, self.max) - self.startID = self._reactor.callWhenRunning(self._start) - - self.deferred = defer.Deferred() - - if pcapfile: - self.pcapwriter = PcapWriter(pcapfile) - - def _buildSocket(self, filter=None, iface=None, nofilter=None): - self.socket = ScapySocket(filter, iface, nofilter) - if self.recv: - self._reactor.addReader(self) - - def _buildPacketQueues(self, pkts): - """ - Converts the list of packets to a Scapy generator and sets up all the - necessary attributes for understanding if all the needed responses have - been received. - """ - if not isinstance(pkts, Gen): - self.pkts = SetGen(pkts) - - self.outqueue = [p for p in pkts] - - self.total_count = len(self.outqueue) - self.answer_count = 0 - self.out_count = 0 - - self.hsent = {} - for p in self.outqueue: - h = p.hashret() - if h in self.hsent: - self.hsent[h].append(p) - else: - self.hsent[h] = [p] - - - def gotAnswer(self, answer, question): - """ - Got a packet that has been identified as an answer to one of the sent - out packets. - - If the answer count matches the sent count the finish callback is - fired. - - @param answer: the packet received on the wire. - - @param question: the sent packet that matches that response. - - """ - - if self.pcapwriter and self.write_only_answers: - self.pcapwriter.write(question) - self.pcapwriter.write(answer) - self.answer_count += 1 - self.last_answer = time.time() - - if self.answer_count >= self.total_count: - print "Got all the answers I need" - self.deferred.callback(None) - - - def processAnswer(self, pkt, hlst): - """ - Checks if the potential answer is in fact an answer to one of the - matched sent packets. Uses the scapy .answers() function to verify - this. - - @param pkt: The packet to be tested if is the answer to a sent packet. - - @param hlst: a list of packets that match the hash for an answer to - pkt. - """ - for i in range(len(hlst)): - if pkt.answers(hlst[i]): - self.gotAnswer(pkt, hlst[i]) - - def fileno(self): - """ - Returns a fileno for use by twisteds Reader. - """ - return self.socket.fileno() - - def processPacket(self, pkt): - """ - Override this method to process your packets. - - @param pkt: the packet that has been received. - """ - #pkt.show() - - - def doRead(self): - """ - There is something to be read on the wire. Do all the processing on the - received packet. - """ - if self.timeout and (time.time() - self.last_answer) > self.timeout and\ - not self.outqueue: - print "Timing out.." - self.deferred.callback(None) - - pkt = self.socket.recv() - if self.pcapwriter and not self.write_only_answers: - self.pcapwriter.write(pkt) - self.processPacket(pkt) - - h = pkt.hashret() - if h in self.hsent: - hlst = self.hsent[h] - self.processAnswer(pkt, hlst) - - def logPrefix(self): - """ - The prefix to be prepended in logging. - """ - return "txScapy" - - def _start(self): - """ - Start the twisted thread pool. - """ - self.startID = None - return self.start() - - def start(self): - """ - Actually start the thread pool. - """ - if not self.running: - self.threadpool.start() - self.shutdownID = self._reactor.addSystemEventTrigger( - 'during', 'shutdown', self.finalClose) - self.running = True - - def sendPkt(self, pkt): - """ - Send a packet to the wire. - - @param pkt: The packet to be sent. - """ - self.socket.send(pkt) - - def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): - """ - Wraps the scapy sr function. - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - self.timeout = timeout - self.recv = True - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): - """ - Wraps the scapy send function. Its the same as send and receive, except - it does not receive. Who would have ever guessed? ;) - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - self.recv = False - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0): - self._buildSocket(filter, iface, nofilter) - self._buildPacketQueues(pkts) - if not self.last_answer: - self.last_answer = time.time() - - def sent(cb): - if self.cthreads < self.mthreads and not self.done: - pkt = None - try: - pkt = self.outqueue.pop() - except: - self.done = True - if not self.recv: - self.deferred.callback(None) - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - for x in range(self.mthreads): - try: - pkt = self.outqueue.pop() - except: - self.done = True - return - if self.cthreads >= self.mthreads and self.done: - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - def connectionLost(self, why): - pass - - def finalClose(self): - """ - Clean all the thread related stuff up. - """ - self.shutdownID = None - self.threadpool.stop() - self.running = False - -def txsr(*args, **kw): - tr = Scapy(*args, **kw) - tr.sr(*args, **kw) - return tr.deferred - -def txsend(*arg, **kw): - tr = Scapy(*arg, **kw) - tr.send(*arg, **kw) - return tr.deferred diff --git a/ooni/lib/txtorcon b/ooni/lib/txtorcon new file mode 160000 index 0000000..ac791b4 --- /dev/null +++ b/ooni/lib/txtorcon @@ -0,0 +1 @@ +Subproject commit ac791b4ddfc29a2dd29a7044f34c3394df38653d diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute new file mode 160000 index 0000000..067a260 --- /dev/null +++ b/ooni/lib/txtraceroute @@ -0,0 +1 @@ +Subproject commit 067a2609390e77bf9187275638cf8786efef7e13

1 0

[ooni-probe/master] * Fix some bugs in Daphn3
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 14ade9b22b2d91dc3c85a7d246c76cc6bc935d22 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu Aug 2 17:42:43 2012 +0200 * Fix some bugs in Daphn3 * Add more details to Test interface * Remove dead code in Reporting system * Make some changes to bridget --- ooni/ooniprobe.py | 2 +- ooni/plugoo/reports.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 3b48339..7e9eecc 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -84,7 +84,7 @@ def runTest(test, options, global_options, reactor=reactor): return if test.ended: - print "Ending prematurely" + print "Ending because it's ended" return wgen = work.WorkGenerator(test, diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index eb2cf4f..a6b0ac9 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,11 @@ import os import yaml import itertools +<<<<<<< HEAD import log +======= +from ooni.utils import log, date, net +>>>>>>> * Fix some bugs in Daphn3 class Report: """This is the ooni-probe reporting mechanism. It allows @@ -81,13 +85,8 @@ class Report: """ This reports to a file in YAML format """ - if not file: - file = self.file - try: - with open(file, mode) as f: - f.write(data) - except Exception, e: - raise e + with open(self.file, 'a+') as f: + f.write(data) def send_report(self, data, type): """

1 0

[ooni-probe/master] Updating the ooni/lib structure and git submodules for imports, and I'm trying
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 174cef2f7f9bfb395fe4753280a881ca5950b504 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 28 12:02:44 2012 +0000 Updating the ooni/lib structure and git submodules for imports, and I'm trying to keep the syntax the same. Updated the README with info on dev interaction with and user cloning of submodules. --- README.md | 22 ++++++++++++++-------- ooni/lib/__init__.py | 41 ++++++++++++++++++++++++++++++++++++----- ooni/lib/txtorcon | 2 +- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b39cb2b..7d6e37a 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,33 @@ OONI, the Open Observatory of Network Interference, is a global observation network which aims is to collect high quality data using open methodologies, using Free and Open Source Software (FL/OSS) to share observations and data -about the various types, methods, and amounts of network tampering in the world. +about the various types, methods, and amounts of network tampering in the +world. # Let's get started with this already! -To run OONI-probe without having to install it you must tell python that it can -import modules from the root of ooni-probe. +To run OONI-probe without having to install it you must tell python that it +can import modules from the root of ooni-probe, as well as initialize the +included submodules. -You must therefore run from the root of the repo: +From the root directory of the repo (.../ooni-probe/), initialize the submodules by doing: - export PYTHONPATH=$PYTHONPATH:`pwd` + $ git submodule init && git submodule update + +Next, you will need to tell Python that OONI is part of its path: + + $ export PYTHONPATH=$PYTHONPATH:`pwd` Then to see what tests are available: - cd ooni - python ooniprobe.py + $ cd ooni + $ python ooniprobe.py If you see some errors see INSTALL to install the missing dependencies. To list the help for a specific test: - python ooniprobe.py httpt --help + $ python ooniprobe.py httpt --help # More details diff --git a/ooni/lib/__init__.py b/ooni/lib/__init__.py index fac7035..0fd36c5 100644 --- a/ooni/lib/__init__.py +++ b/ooni/lib/__init__.py @@ -1,9 +1,40 @@ +import pkgutil import sys -import os +from os import listdir, path -pwd = os.path.dirname(__file__) -#child_path = os.path.normpath(os.path.join(pwd, '..')) -#sys.path.append(child_path) -sys.path.append(pwd) +__all__ = ['txtorcon', 'txscapy', 'txtraceroute'] +__sub_modules__ = [ ] +def callback(arg, directory, files): + for file in listdir(directory): + fullpath = path.abspath(file) + if path.isdir(fullpath) and not path.islink(fullpath): + __sub_modules__.append(fullpath) + sys.path.append(fullpath) + +path.walk(".", callback, None) + +def load_submodules(init, list): + for subdir in list: + contents=[x for x in pkgutil.iter_modules(path=subdir, + prefix='ooni.lib.')] + for loader, module_name, ispkg in contents: + init_dot_module = init + "." + module_name + if init_dot_module in sys.modules: + module = sys.modules[module_name] + else: + if module_name in __all__: + grep = loader.find_module(module_name) + module = grep.load_module(module_name) + else: + module = None + + if module is not None: + globals()[module_name] = module + +load_submodules(__name__, __sub_modules__) + +print "system paths are: %s" % sys.path +print "globals are: %s" % globals() +print "system modules are: %s" % sys.modules diff --git a/ooni/lib/txtorcon b/ooni/lib/txtorcon index ac791b4..def1551 160000 --- a/ooni/lib/txtorcon +++ b/ooni/lib/txtorcon @@ -1 +1 @@ -Subproject commit ac791b4ddfc29a2dd29a7044f34c3394df38653d +Subproject commit def155177f4ba6fd0728922e5b6f3c0066006a94

1 0

[ooni-probe/master] Submodules are working, without modification to any of the old import
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 1dc7a6966d1e4ec69f6bab4667384e583cf65636 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 28 12:58:42 2012 +0000 Submodules are working, without modification to any of the old import statements. To initialize them, do "$ git submodules init && git submodules update". --- TODO | 26 ++++++++++++-------------- 1 files changed, 12 insertions(+), 14 deletions(-) diff --git a/TODO b/TODO index 8cc5b90..7f532fd 100644 --- a/TODO +++ b/TODO @@ -15,20 +15,6 @@ It should respect the design goals of the new ooni-probe model. * All the parts related to HTTP and Squid (old/ooni/http.py) effort: low, skill: medium -* All the captive portal detection tests (old/ooni/http.py - and old/ooni/dnsooni.py) -effort: low, skill: low - - The captiveportal test is a port of the old code, plus - a couple new methods for testing. - -* All the DNS censorship detection tests (old/ooni/dnsooni.py) -effort: medium, skill: medium - - The dnstamper test fulfills this requirement, although - there are several improvements which could be made, namely - figuring out if there is a pattern to the reverse resolved - names and parsing them in some manner to determine if that - server is authorized to serve the requested page. - * Misc scripts to detect censorship that are not written in any version of OONI (old/ooni/plugins/old_stuff_to_convert/) effort: low-medium, skill: low-medium @@ -45,3 +31,15 @@ or that we haven't yet figured out how they should work. * Design and implement the Network Node and the Code Exec node classes +New things to test +------------------ + +Please check that the submodules are working correctly. From ../ooni-probe/, +do: + +$ git submodules init && git submodules update + +Make sure that they clone correctly, and run a few ooni tests to make sure +the import statements haven't broken. + +Write unittests for *all of the things*. Can anyone suggest a good framework? \ No newline at end of file

1 0

[ooni-probe/master] Merge branch 'bridget-twisted'
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 62956ebab7779c1b61ce3d6e8ac750552fd1c988 Merge: 9c5d5d4 df76264 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 13 13:01:44 2012 +0000 Merge branch 'bridget-twisted' ooni/assets/bridgetests.txt | 8 +- ooni/ooniprobe.py | 14 +- ooni/plugins/bridget.py | 465 +++++++++++++++++++++++++++++++++++++------ ooni/plugins/domclass.py | 3 +- ooni/plugoo/tests.py | 17 +- ooni/utils/log.py | 116 ++++++++--- 6 files changed, 522 insertions(+), 101 deletions(-) diff --cc ooni/plugoo/tests.py index 4be0b62,a827093..19d42b2 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@@ -5,11 -5,14 +5,12 @@@ from zope.interface import Interface, A import logging import itertools from twisted.internet import reactor, defer, threads + ## XXX why is this imported and not used? from twisted.python import failure -from ooni.utils import log -from ooni.utils import date -from ooni.plugoo import assets, work -from ooni.plugoo.reports import Report -from ooni.plugoo.interface import ITest +from ooni import log +from plugoo import assets, work +from plugoo.reports import Report class OONITest(object):

1 0

[ooni-probe/master] BridgeT works again, but...
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit c09e7f9a98e7191332f39e6d70d7bb967c5e3e8a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Sep 3 10:55:51 2012 +0000 BridgeT works again, but... There seems to be a problem with it attempting to ControlPort to the same port as one of my local non-slave Tors, even though TorConfig() is still using random.randint(). I will investigate further after some much-needed sl33pzorzings. --- ooni/ooniprobe.log | 1 - ooni/plugins/bridget.py | 53 +++++++++++++++++++++++++++++++++------------- ooni/plugins/domclass.py | 3 +- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log deleted file mode 100644 index 700e820..0000000 --- a/ooni/ooniprobe.log +++ /dev/null @@ -1 +0,0 @@ -2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 22ff9a0..5562fb6 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -1,8 +1,17 @@ -""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +----------------------------------------------+ +# +--------| Use a slave Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +----------------------------------------------+ +# +# :authors: Arturo Filasto, Isis Lovecruft, Jacob Appelbaum +# :licence: see included LICENSE +# :version: 0.1.0-alpha + from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin @@ -21,34 +30,43 @@ class bridgetTest(OONITest): implements(IPlugin, ITest) shortName = "bridget" - description = "bridget" + description = "Use a slave Tor process to test RELAY_EXTEND to bridges/relays" requirements = None options = bridgetArgs blocking = False def experiment(self, args): - log.msg("Doing test") - # What you return here gets handed as input to control + log.msg("BridgeT: initiating test ... ") + from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) - return + + reactor = self.reactor def setup_failed(args): log.msg("Setup Failed.") report.update({'failed': args}) - return report + reactor.stop() + #return report def setup_complete(proto): - log.msg("Setup Complete.") + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_failed) report.update({'success': args}) - return report + #return report + + def bootstrap(c): + conf = TorConfig(c) + conf.post_bootstrap.addCallback(setup_complete).addErrback(setup_failed) + log.msg("Slave Tor process connected, bootstrapping ...") config = TorConfig() import random config.SocksPort = random.randint(1024, 2**16) config.ControlPort = random.randint(1024, 2**16) + #config.SocksPort = 12345 + #config.ControlPort = 12346 if 'bridge' in args: config.UseBridges = 1 @@ -57,6 +75,11 @@ class bridgetTest(OONITest): print config.create_torrc() report = {'tor_config': config.config} log.msg("Starting Tor") + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + #return + d = launch_tor(config, self.reactor, progress_updates=updates) d.addCallback(setup_complete) d.addErrback(setup_failed) @@ -68,7 +91,7 @@ class bridgetTest(OONITest): if self.local_options['bridges']: assets.update({'bridge': Asset(self.local_options['bridges'])}) elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relay'])}) + assets.update({'relay': Asset(self.local_options['relays'])}) return assets # We need to instantiate it otherwise getPlugins does not detect it diff --git a/ooni/plugins/domclass.py b/ooni/plugins/domclass.py index cdcd508..c24f2ce 100644 --- a/ooni/plugins/domclass.py +++ b/ooni/plugins/domclass.py @@ -1,4 +1,5 @@ -# +#!/usr/bin/env python +#-*- encoding: utf-8 -*- # # domclass # ********

1 0