September 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Added subclassed t.n.client.Resolver() to overcome query port randomization.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 5e748acc7f787d37274cbdab9a57b317919f2fa9 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Jun 16 09:21:53 2012 -0700 Added subclassed t.n.client.Resolver() to overcome query port randomization. --- ooni/plugins/dnstamper.py | 75 ++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 68 insertions(+), 7 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index 34aaa01..ca0e2e3 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -28,8 +28,9 @@ import os -from twisted.names import client +from twisted.names import client, dns from twisted.internet import reactor +from twisted.internet.error import CannotListenError from twisted.internet.protocol import Factory, Protocol from twisted.python import usage from twisted.plugin import IPlugin @@ -39,7 +40,7 @@ from ooni.plugoo.assets import Asset from ooni.plugoo.tests import ITest, OONITest from ooni import log -class Top1MAsset(Asset): +class AlexaAsset(Asset): """ Class for parsing the Alexa top-1m.txt as an asset. """ @@ -54,9 +55,60 @@ class DNSTamperArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], ['testservers', 't', None, 'Asset file of DNS servers to test'], + ['localservers', 'l', False, 'Also test local servers'], + ['port', 'p', None, 'Local UDP port to send queries over'], ['usereverse', 'r', False, 'Also try reverse DNS resolves'], ['resume', 's', 0, 'Resume at this index in the asset file']] +class DNSTamperResolver(client.Resolver): + """ + Twisted by default issues DNS queries over cryptographically random + UDP ports to mitigate the Berstein/Kaminsky attack on limited DNS + Transaction ID numbers.[1][2][3] + + This is fine, unless the client has external restrictions which require + DNS queries to be conducted over UDP port 53. Twisted does not provide + an easy way to change this, ergo subclassing client.Resolver.[4] It + would perhaps be wise to patch twisted.names.client and request a merge + into upstream. + + [1] https://twistedmatrix.com/trac/ticket/3342 + [2] http://blog.netherlabs.nl/articles/2008/07/09/ \ + some-thoughts-on-the-recent-dns-vulnerability + [3] http://www.blackhat.com/presentations/bh-dc-09/Kaminsky/ \ + BlackHat-DC-09-Kaminsky-DNS-Critical-Infrastructure.pdf + [4] http://comments.gmane.org/gmane.comp.python.twisted/22794 + """ + def __init__(self): + super(DNSTamperResolver, self).__init__() + #client.Resolver.__init__(self) + + if self.local_options['port']: + self.port = self.local_options['port'] + else: + self.port = '53' + + def _connectedProtocol(self): + """ + Return a new DNSDatagramProtocol bound to a specific port + rather than the default cryptographically-random port. + """ + if 'protocol' in self.__dict__: + return self.protocol + proto = dns.DNSDatagramProtocol(self) + + ## XXX We may need to remove the while loop, which was + ## originally implemented to safeguard against attempts to + ## bind to the same random port twice...but then the code + ## would be blocking... + while True: + try: + self._reactor.listenUDP(self.port, proto) + except error.CannotListenError: + pass + else: + return proto + class DNSTamperTest(OONITest): implements(IPlugin, ITest) @@ -66,13 +118,20 @@ class DNSTamperTest(OONITest): options = DNSTamperArgs blocking = False + if self.local_options['localservers']: + ## client.createResolver() turns None into '/etc/resolv.conf' + ## on posix systems, ignored on Windows. + self.resolvconf = None + else: + self.resolvconf = '' + def load_assets(self): assets = {} if self.local_options: if self.local_options['asset']: assetf = self.local_options['asset'] if assetf == 'top-1m.txt': - assets.update({'asset': Top1MAsset(assetf)}) + assets.update({'asset': AlexaAsset(assetf)}) else: assets.update({'asset': Asset(assetf)}) elif self.local_options['testservers']: @@ -88,7 +147,7 @@ class DNSTamperTest(OONITest): def got_result(result): log.msg('Resolved %s through %s to %s' % (hostname, nameserver, result)) - reactor.stop() + #reactor.stop() return {'resolved': True, 'domain': hostname, 'nameserver': nameserver, @@ -96,13 +155,14 @@ class DNSTamperTest(OONITest): def got_error(err): log.msg(err.printTraceback()) - reactor.stop() + #reactor.stop() return {'resolved': False, 'domain': hostname, 'nameserver': nameserver, 'address': err} - res = client.createResolver(servers=[(nameserver, 53)]) + res = client.createResolver(resolvconf=self.resolvconf, + servers=[(nameserver, 53)]) d = res.getHostByName(hostname) d.addCallbacks(got_result, got_error) return d @@ -116,7 +176,8 @@ class DNSTamperTest(OONITest): sets from a positive result resolve to the same domain, in order to remove false positives due to GeoIP load balancing. """ - res = client.createResolver(servers=[(nameserver, 53)]) + res = client.createResolver(resolvconf=self.resolvconf, + servers=[(nameserver, 53)]) ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa' d = res.lookupPointer(ptr) d.addCallback(lambda (ans, auth, add): util.println(ans[0].payload.name))

1 0

[ooni-probe/master] Added subclassed t.n.client.Resolver() to overcome query port randomization.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 507694c16715f76c17b061bc49bdb32f15c81983 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Jun 16 09:21:53 2012 -0700 Added subclassed t.n.client.Resolver() to overcome query port randomization. --- ooni/oonitests/dnstamper.py | 187 ------------------------------------------- 1 files changed, 0 insertions(+), 187 deletions(-) diff --git a/ooni/oonitests/dnstamper.py b/ooni/oonitests/dnstamper.py deleted file mode 100644 index 4ad7e7f..0000000 --- a/ooni/oonitests/dnstamper.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- coding: utf-8 -*- -""" - dnstamper - ********* - - This test resolves DNS for a list of domain names, one per line, in the - file specified in the ooni-config under the setting "dns_experiment". If - the file is top-1m.txt, the test will be run using Amazon's list of top - one million domains. The experimental dns servers to query should - be specified one per line in assets/dns_servers.txt. - - The test reports censorship if the cardinality of the intersection of - the query result set from the control server and the query result set - from the experimental server is zero, which is to say, if the two sets - have no matching results whatsoever. - - NOTE: This test frequently results in false positives due to GeoIP-based - load balancing on major global sites such as google, facebook, and - youtube, etc. - - :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft - :license: see LICENSE for more details - - TODO: - * Switch to using Twisted's DNS builtins instead of dnspython - * -""" - -import os - -from twisted.names import client -from twisted.python import usage -from twisted.plugin import IPlugin -from zope.interface import implements - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest -from ooni import log - -class Top1MAsset(Asset): - """ - Class for parsing the Alexa top-1m.txt as an asset. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - self = Asset.parse_line(self, line) - return line.split(',')[1].replace('\n','') - -class DNSTamperAsset(Asset): - """ - Creates DNS testing specific Assets. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - -class DNSTamperArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], - ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], - ['testservers', 't', None, 'Asset file of the DNS servers to test'], - ['resume', 'r', 0, 'Resume at this index in the asset file']] -''' - def control(self, experiment_result, args): - print "Experiment Result:", experiment_result - print "Args", args - return experiment_result - - def experiment(self, args): -''' - -class DNSTamperTest(OONITest): - implements(IPlugin, ITest) - - shortName = "DNSTamper" - description = "DNS censorship detection test" - requirements = None - options = DNSTamperArgs - blocking = False - - def load_asset(self): - if self.local_options: - if self.local_options['asset']: - assetf = self.local_options['asset'] - if assetf == 'top-1m.txt': - return {'asset': Top1MAsset(assetf)} - else: - return {'asset': DNSTamperAsset(assetf)} - else: - return {} - - def lookup(self, hostname, nameserver): - """ - Resolves a hostname through a DNS nameserver to the corresponding - IP addresses. - """ - res = client.createResolver(servers=[('nameserver', 53)]) - answer = res.getHostByName(hostname) - ## XXX MAY ALSO BE: - #answer = res.getAddress(servers=[('nameserver', 53)]) - - ret = [] - - for data in answer: - ret.append(data.address) - - return ret - - def reverse_lookup(self, ip, nameserver): - """ - Attempt to do a reverse DNS lookup to determine if the control and exp - sets from a positive result resolve to the same domain, in order to - remove false positives due to GeoIP load balancing. - """ - res = client.createResolver(servers=nameserver) - n = reversename.from_address(ip) - revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] - - return revn - - def experiment(self, *a, **kw): - """ - Compares the lookup() sets of the control and experiment groups. - """ - # this is just a dirty hack - address = kw['data'][0] - ns = kw['data'][1] - - config = self.config - ctrl_ns = config.tests.dns_control_server - - print "ADDRESS: %s" % address - print "NAMESERVER: %s" % ns - - exp = self.lookup(address, ns) - control = self.lookup(address, ctrl_ns) - - result = [] - - if len(set(exp) & set(control)) > 0: - print "Address %s has not tampered with on DNS server %s\n" % (address, ns) - result = (address, ns, exp, control, False) - return result - else: - print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) - result = (address, ns, exp, control, True) - - if config.tests.dns_reverse_lookup: - - exprevn = [self.reverse_lookup(ip, ns) for ip in exp] - ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) - for ip in control] - - if len(set(exprevn) & set(ctrlrevn)) > 0: - print "Further testing has eliminated this as a false positive." - else: - print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) - return result - - else: - print "\n" - return result - -#def run(ooni): -# """ -# Run the test. -# """ -# config = ooni.config -# urls = [] -# -# if (config.tests.dns_experiment == "top-1m.txt"): -# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment)) -# else: -# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment)) -# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment_dns)) -# -# assets = [dns_experiment, dns_experiment_dns] -# -# dnstest = DNST(ooni) -# ooni.logger.info("Beginning dnstamper test...") -# dnstest.run(assets, {'index': 1}) -# ooni.logger.info("Dnstamper test completed!") - -dnstamper = DNSTamperTest(None, None, None)

1 0

[ooni-probe/master] Adding old dnstamper.py file so git doesn't wipe it. Don't include this commit.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit eeed21131f3b955176ad7d0c1502f6f4551cc72a Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Jun 25 11:34:15 2012 -0700 Adding old dnstamper.py file so git doesn't wipe it. Don't include this commit. --- ooni/oonitests/dnstamper.py | 200 +++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 200 insertions(+), 0 deletions(-) diff --git a/ooni/oonitests/dnstamper.py b/ooni/oonitests/dnstamper.py new file mode 100644 index 0000000..f73f494 --- /dev/null +++ b/ooni/oonitests/dnstamper.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +""" + dnstamper + ********* + + This test resolves DNS for a list of domain names, one per line, in the + file specified in the ooni-config under the setting "dns_experiment". If + the file is top-1m.txt, the test will be run using Amazon's list of top + one million domains. The experimental dns servers to query should + be specified one per line in assets/dns_servers.txt. + + The test reports censorship if the cardinality of the intersection of + the query result set from the control server and the query result set + from the experimental server is zero, which is to say, if the two sets + have no matching results whatsoever. + + NOTE: This test frequently results in false positives due to GeoIP-based + load balancing on major global sites such as google, facebook, and + youtube, etc. + + :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details + + TODO: + * Switch to using Twisted's DNS builtins instead of dnspython + * +""" + +import os + +from twisted.names import client +from twisted.internet import reactor +from twisted.internet.protocol import Factory, Protocol +from twisted.python import usage +from twisted.plugin import IPlugin +from zope.interface import implements + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni import log + +class Top1MAsset(Asset): + """ + Class for parsing the Alexa top-1m.txt as an asset. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.split(',')[1].replace('\n','') + +class DNSTamperAsset(Asset): + """ + Creates DNS testing specific Assets. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + +class DNSTamperArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], + ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], + ['testservers', 't', None, 'Asset file of the DNS servers to test'], + ['resume', 'r', 0, 'Resume at this index in the asset file']] +''' + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): +''' + +class DNSTamperTest(OONITest): + implements(IPlugin, ITest) + + shortName = "DNSTamper" + description = "DNS censorship detection test" + requirements = None + options = DNSTamperArgs + blocking = False + + def load_assets(self): + if self.local_options: + if self.local_options['asset']: + assetf = self.local_options['asset'] + if assetf == 'top-1m.txt': + return {'asset': Top1MAsset(assetf)} + else: + return {'asset': DNSTamperAsset(assetf)} + else: + return {} + + def lookup(self, hostname, nameserver): + """ + Resolves a hostname through a DNS nameserver to the corresponding + IP addresses. + """ + def got_result(result): + #self.logger.log(result) + print result + reactor.stop() + + def got_failure(failure): + failure.printTraceback() + reactor.stop() + + res = client.createResolver(servers=[(nameserver, 53)]) + d = res.getHostByName(hostname) + d.addCallbacks(got_result, got_failure) + + ## XXX MAY ALSO BE: + #answer = res.getAddress(servers=[('nameserver', 53)]) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + + def reverse_lookup(self, ip, nameserver): + """ + Attempt to do a reverse DNS lookup to determine if the control and exp + sets from a positive result resolve to the same domain, in order to + remove false positives due to GeoIP load balancing. + """ + res = client.createResolver(servers=nameserver) + n = reversename.from_address(ip) + revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] + + return revn + + def experiment(self, *a, **kw): + """ + Compares the lookup() sets of the control and experiment groups. + """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config + ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) + control = self.lookup(address, ctrl_ns) + + result = [] + + if len(set(exp) & set(control)) > 0: + print "Address %s has not tampered with on DNS server %s\n" % (address, ns) + result = (address, ns, exp, control, False) + return result + else: + print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) + result = (address, ns, exp, control, True) + + if config.tests.dns_reverse_lookup: + + exprevn = [self.reverse_lookup(ip, ns) for ip in exp] + ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) + for ip in control] + + if len(set(exprevn) & set(ctrlrevn)) > 0: + print "Further testing has eliminated this as a false positive." + else: + print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) + return result + + else: + print "\n" + return result + +#def run(ooni): +# """ +# Run the test. +# """ +# config = ooni.config +# urls = [] +# +# if (config.tests.dns_experiment == "top-1m.txt"): +# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# else: +# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment_dns)) +# +# assets = [dns_experiment, dns_experiment_dns] +# +# dnstest = DNST(ooni) +# ooni.logger.info("Beginning dnstamper test...") +# dnstest.run(assets, {'index': 1}) +# ooni.logger.info("Dnstamper test completed!") + +dnstamper = DNSTamperTest(None, None, None)

1 0

[ooni-probe/master] Need to fix exp_address from d['address'] dict
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 0264e482d01dbbfc0a5b1f94bd98118270b7cdd6 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu Jul 26 00:17:00 2012 +0000 Need to fix exp_address from d['address'] dict --- ooni/ooniprobe.log | 1 - ooni/plugins/dnstamper.py | 92 ++++++++++++++++++++++++++++++--------------- ooni/plugoo/reports.py | 9 +++- 3 files changed, 69 insertions(+), 33 deletions(-) diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log deleted file mode 100644 index 700e820..0000000 --- a/ooni/ooniprobe.log +++ /dev/null @@ -1 +0,0 @@ -2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index a30cf27..e768dab 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -18,12 +18,14 @@ load balancing on major global sites such as google, facebook, and youtube, etc. - :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :author: Isis Lovecruft, Arturo Filastò :license: see LICENSE for more details - TODO: - * Switch to using Twisted's DNS builtins instead of dnspython - * + TODO: + * Use OONI log instead of print + * Finish porting to twisted + * Finish the client.Resolver() subclass and test it + * Use the DNS tests from captiveportal """ import os @@ -52,13 +54,20 @@ class AlexaAsset(Asset): return line.split(',')[1].replace('\n','') class DNSTamperArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], - ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], - ['testservers', 't', None, 'Asset file of DNS servers to test'], - ['localservers', 'l', False, 'Also test local servers'], - ['port', 'p', None, 'Local UDP port to send queries over'], - ['usereverse', 'r', False, 'Also try reverse DNS resolves'], - ['resume', 's', 0, 'Resume at this index in the asset file']] + optParameters = [['hostnames', 'h', None, + 'Asset file of hostnames to resolve'], + ['controlserver', 'c', '8.8.8.8', + 'Known good DNS server'], + ['testservers', 't', None, + 'Asset file of DNS servers to test'], + ['localservers', 'l', False, + 'Also test local servers'], + ['port', 'p', None, + 'Local UDP port to send queries over'], + ['usereverse', 'r', False, + 'Also try reverse DNS resolves'], + ['resume', 's', 0, + 'Resume at this index in the asset file']] class DNSTamperResolver(client.Resolver): """ @@ -137,23 +146,43 @@ class DNSTamperTest(OONITest): def initialize(self): if self.local_options: + + ## client.createResolver() turns 'None' into '/etc/resolv.conf' on + ## posix systems, ignored on Windows. if self.local_options['localservers']: self.resolvconf = None else: self.resolvconf = '' + else: + pass + + self.d = defer.Deferred() def load_assets(self): assets = {} + if self.local_options: - if self.local_options['asset']: - assetf = self.local_options['asset'] + + if self.local_options['hostnames']: + assetf = self.local_options['hostnames'] if assetf == 'top-1m.txt': - assets.update({'asset': AlexaAsset(assetf)}) + assets['hostnames'] = AlexaAsset(assetf) + #assets.update({'asset': AlexaAsset(assetf)}) else: - assets.update({'asset': Asset(assetf)}) - elif self.local_options['testservers']: - assets.update({'testservers': - Asset(self.local_options['testservers'])}) + assets['hostnames'] = Asset(assetf) + #assets.update({'asset': Asset(assetf)}) + else: + #default_hostnames = ['google.com', 'torrentz.eu', 'ooni.nu', + # 'twitter.com', 'baidu.com'] + #assets.update({'asset': [host for host in default_hostnames]}) + print "Error! We need a file containing the hostnames that we should test DNS for!" + + if self.local_options['testservers']: + #assets['testservers'] = Asset(self.local_options['testservers']) + self.testservers = Asset(self.local_options['testservers']) + else: + self.testservers = ['209.244.0.3', '208.67.222.222', '156.154.70.1'] + return assets def lookup(self, hostname, nameserver): @@ -164,7 +193,6 @@ class DNSTamperTest(OONITest): def got_result(result): log.msg('Resolved %s through %s to %s' % (hostname, nameserver, result)) - #reactor.stop() return {'resolved': True, 'domain': hostname, 'nameserver': nameserver, @@ -172,7 +200,6 @@ class DNSTamperTest(OONITest): def got_error(err): log.msg(err.printTraceback()) - #reactor.stop() return {'resolved': False, 'domain': hostname, 'nameserver': nameserver, @@ -182,11 +209,9 @@ class DNSTamperTest(OONITest): servers=[(nameserver, 53)]) d = res.getHostByName(hostname) d.addCallbacks(got_result, got_error) + return d - ## XXX MAY ALSO BE: - #answer = res.getAddress(servers=[('nameserver', 53)]) - def reverse_lookup(self, address, nameserver): """ Attempt to do a reverse DNS lookup to determine if the control and exp @@ -206,26 +231,33 @@ class DNSTamperTest(OONITest): """ Compares the lookup() sets of the control and experiment groups. """ - test_server = self.local_options['testservers'] - hostname = args['asset'] - exp_address = self.lookup(hostname, test_server) + hostnames = args + + for hostname in hostnames: + for testserver in self.testservers: + #exp_address = self.lookup(hostname, testserver) + self.d.addCallback(self.lookup, hostname, testserver) + + #print self.assets['hostnames'] + #hostname = args + #exp_address = self.lookup(hostname, testserver) #return {'control': control_server, # 'domain': args['asset'], # 'experiment_address': address} if self.local_options['usereverse']: - exp_reversed = self.reverse_lookup(exp_address, test_server) + exp_reversed = self.reverse_lookup(exp_address, testserver) ## XXX trying to fix errors: #d = defer.Deferred() - return (exp_address, hostname, test_server, exp_reversed) + return (exp_address, hostname, testserver, exp_reversed) else: - return (exp_address, hostname, test_server, False) + return (exp_address, hostname, testserver, False) def control(self, experiment_result): - (exp_address, hostname, test_server, exp_reversed) = experiment_result + (exp_address, hostname, testserver, exp_reversed) = experiment_result control_server = self.local_options['controlserver'] ctrl_address = self.lookup(hostname, control_server) diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index ef15a04..eb2cf4f 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -81,8 +81,13 @@ class Report: """ This reports to a file in YAML format """ - with open(self.file, 'a+') as f: - f.write(data) + if not file: + file = self.file + try: + with open(file, mode) as f: + f.write(data) + except Exception, e: + raise e def send_report(self, data, type): """

1 0

[ooni-probe/master] Unfinished, committing in order to merge changes from master. Fixed class initializations.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit f85da56f489198358d7d1d7fbd2338037af88ce9 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu Jul 19 18:00:00 2012 +0000 Unfinished, committing in order to merge changes from master. Fixed class initializations. --- ooni/plugins/dnstamper.py | 43 ++++++++++++++++++++++++++++++++----------- 1 files changed, 32 insertions(+), 11 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index ca0e2e3..a30cf27 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -29,7 +29,7 @@ import os from twisted.names import client, dns -from twisted.internet import reactor +from twisted.internet import reactor, defer from twisted.internet.error import CannotListenError from twisted.internet.protocol import Factory, Protocol from twisted.python import usage @@ -80,7 +80,8 @@ class DNSTamperResolver(client.Resolver): [4] http://comments.gmane.org/gmane.comp.python.twisted/22794 """ def __init__(self): - super(DNSTamperResolver, self).__init__() + super(DNSTamperResolver, self).__init__(self, resolv, servers, + timeout, reactor) #client.Resolver.__init__(self) if self.local_options['port']: @@ -117,13 +118,29 @@ class DNSTamperTest(OONITest): requirements = None options = DNSTamperArgs blocking = False - - if self.local_options['localservers']: - ## client.createResolver() turns None into '/etc/resolv.conf' - ## on posix systems, ignored on Windows. - self.resolvconf = None - else: - self.resolvconf = '' + + #def __init__(self, local_options, global_options, + # report, ooninet=None, reactor=None): + # super(DNSTamperTest, self).__init__(local_options, global_options, + # report, ooninet, reactor) + # + # if self.reactor is None: + # self.reactor = reactor + # + # if self.local_options: + # if self.local_options['localservers']: + # ## client.createResolver() turns None into '/etc/resolv.conf' + # ## on posix systems, ignored on Windows. + # self.resolvconf = None + # else: + # self.resolvconf = '' + + def initialize(self): + if self.local_options: + if self.local_options['localservers']: + self.resolvconf = None + else: + self.resolvconf = '' def load_assets(self): assets = {} @@ -199,9 +216,13 @@ class DNSTamperTest(OONITest): if self.local_options['usereverse']: exp_reversed = self.reverse_lookup(exp_address, test_server) - return exp_address, hostname, test_server, exp_reversed + + ## XXX trying to fix errors: + #d = defer.Deferred() + + return (exp_address, hostname, test_server, exp_reversed) else: - return exp_address, hostname, test_server, False + return (exp_address, hostname, test_server, False) def control(self, experiment_result): (exp_address, hostname, test_server, exp_reversed) = experiment_result

1 0

[ooni-probe/master] Fixed module import issues with directory structure.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 37c0e4e888416a6d0433a23a164c5bd565c357ca Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu May 31 12:36:50 2012 -0300 Fixed module import issues with directory structure. --- ooni/plugoo/reports.py | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index a6b0ac9..ef15a04 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,11 +4,7 @@ import os import yaml import itertools -<<<<<<< HEAD import log -======= -from ooni.utils import log, date, net ->>>>>>> * Fix some bugs in Daphn3 class Report: """This is the ooni-probe reporting mechanism. It allows

1 0

[ooni-probe/master] Still sorting out that darned resolver...
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 13077468d151d931c4de38961be3dbb76a4eb88c Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Jul 28 01:33:23 2012 +0000 Still sorting out that darned resolver... --- ooni/plugins/dnstamper.py | 212 ++++++++++++++++++++++++++------------------- 1 files changed, 121 insertions(+), 91 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index e768dab..8e8563a 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -22,10 +22,10 @@ :license: see LICENSE for more details TODO: - * Use OONI log instead of print * Finish porting to twisted * Finish the client.Resolver() subclass and test it * Use the DNS tests from captiveportal + * Use plugoo/reports.py for final data """ import os @@ -56,11 +56,11 @@ class AlexaAsset(Asset): class DNSTamperArgs(usage.Options): optParameters = [['hostnames', 'h', None, 'Asset file of hostnames to resolve'], - ['controlserver', 'c', '8.8.8.8', + ['controlresolver', 'c', '8.8.8.8', 'Known good DNS server'], - ['testservers', 't', None, + ['testresolvers', 't', None, 'Asset file of DNS servers to test'], - ['localservers', 'l', False, + ['localresolvers', 'l', False, 'Also test local servers'], ['port', 'p', None, 'Local UDP port to send queries over'], @@ -120,6 +120,9 @@ class DNSTamperResolver(client.Resolver): return proto class DNSTamperTest(OONITest): + """ + XXX fill me in + """ implements(IPlugin, ITest) shortName = "dnstamper" @@ -127,136 +130,163 @@ class DNSTamperTest(OONITest): requirements = None options = DNSTamperArgs blocking = False - - #def __init__(self, local_options, global_options, - # report, ooninet=None, reactor=None): - # super(DNSTamperTest, self).__init__(local_options, global_options, - # report, ooninet, reactor) - # - # if self.reactor is None: - # self.reactor = reactor - # - # if self.local_options: - # if self.local_options['localservers']: - # ## client.createResolver() turns None into '/etc/resolv.conf' - # ## on posix systems, ignored on Windows. - # self.resolvconf = None - # else: - # self.resolvconf = '' + + def __init__(self, local_options, global_options, + report, ooninet=None, reactor=None): + super(DNSTamperTest, self).__init__(local_options, global_options, + report, ooninet, reactor) + + def __repr__(self): + represent = "DNSTamperTest(OONITest): local_options=%r, " \ + "global_options=%r, assets=%r" % (self.local_options, + self.global_options, + self.assets) + return represent def initialize(self): if self.local_options: - ## client.createResolver() turns 'None' into '/etc/resolv.conf' on ## posix systems, ignored on Windows. - if self.local_options['localservers']: + if self.local_options['localresolvers']: self.resolvconf = None else: self.resolvconf = '' - else: - pass - - self.d = defer.Deferred() def load_assets(self): assets = {} - if self.local_options: + default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', + 'ooni.nu', 'google.com', 'torproject.org'] + default_resolvers = ['209.244.0.3', '208.67.222.222'] - if self.local_options['hostnames']: - assetf = self.local_options['hostnames'] - if assetf == 'top-1m.txt': - assets['hostnames'] = AlexaAsset(assetf) - #assets.update({'asset': AlexaAsset(assetf)}) - else: - assets['hostnames'] = Asset(assetf) - #assets.update({'asset': Asset(assetf)}) + def asset_file(asset_option): + return self.local_options[asset_option] + + def list_to_asset(list_): + def next(list_): + host = list_.pop() + if host is not None: + yield str(host) + while len(list_) > 0: + next(list_) + + if self.local_options: + if asset_file('hostnames'): + with asset_file('hostnames') as hosts_file: + ## The default filename for the Alexa Top 1 Million: + if hosts_file is 'top-1m.txt': + assets.update({'hostnames': AlexaAsset(hosts_file)}) + else: + assets.update({'hostnames': Asset(hosts_file)}) else: - #default_hostnames = ['google.com', 'torrentz.eu', 'ooni.nu', - # 'twitter.com', 'baidu.com'] - #assets.update({'asset': [host for host in default_hostnames]}) - print "Error! We need a file containing the hostnames that we should test DNS for!" - - if self.local_options['testservers']: - #assets['testservers'] = Asset(self.local_options['testservers']) - self.testservers = Asset(self.local_options['testservers']) + log.msg("Error! We need an asset file containing the " + + "hostnames that we should test DNS with! Please use " + + "the '-h' option. Using pre-defined hostnames...") + assets.update({'hostnames': list_to_asset(default_hostnames)}) + + if asset_file('testresolvers'): + with asset_file('testresolvers') as resolver_file: + assets.update({'testresolvers': Asset(resolver_file)}) else: - self.testservers = ['209.244.0.3', '208.67.222.222', '156.154.70.1'] + assets.update({'testresolvers': + list_to_asset(default_resolvers)}) return assets - def lookup(self, hostname, nameserver): + def lookup(self, hostname, resolver): """ - Resolves a hostname through a DNS nameserver to the corresponding - IP addresses. + Resolves a hostname through a DNS nameserver to the corresponding IP + addresses. """ - def got_result(result): + def got_result(result, hostname, resolver): + ## XXX is there a report class that we should be using? log.msg('Resolved %s through %s to %s' - % (hostname, nameserver, result)) - return {'resolved': True, - 'domain': hostname, - 'nameserver': nameserver, - 'address': result} - - def got_error(err): + % (hostname, resolver, result)) + outcome = {'resolved': True, + 'domain': hostname, + 'nameserver': resolver, + 'address': result } + log.msg(outcome) + return result + + def got_error(err, hostname, resolver): log.msg(err.printTraceback()) - return {'resolved': False, - 'domain': hostname, - 'nameserver': nameserver, - 'address': err} + outcome = {'resolved': False, + 'domain': hostname, + 'nameserver': resolver, + 'address': err } + log.msg(outcome) + return err res = client.createResolver(resolvconf=self.resolvconf, - servers=[(nameserver, 53)]) - d = res.getHostByName(hostname) - d.addCallbacks(got_result, got_error) + servers=[(resolver, 53)]) + + ## XXX should we do self.d.addCallback(resHostByName, hostname)? + #d = res.getHostByName(hostname) + #d.addCallbacks(got_result, got_error) + + #d = defer.Deferred() + #d.addCallback(res.getHostByName, hostname) + + #d = res.getHostByName(hostname) + #d.addCallback(got_result, result, hostname, resolver) + #d.addErrback(got_error, err, hostname, resolver) + + res.addCallback(getHostByName, hostname) + res.addCallback(got_result, result, hostname, resolver) + res.addErrback(got_error, err, hostname, resolver) + + if self.local_options['usereverse']: + #d.addCallback(self.reverse_lookup, result, resolver) + #d.addErrback(log.msg(err.printTraceback())) + + #d.addCallback(self.reverse_lookup, result, resolver) + #d.addErrback(log.msg(err.printTraceback())) + + res.addCallback(self.reverse_lookup, result, resolver) + res.addErraback(log.msg(err.printTraceback())) - return d + return res - def reverse_lookup(self, address, nameserver): + def reverse_lookup(self, address, resolver): """ Attempt to do a reverse DNS lookup to determine if the control and exp sets from a positive result resolve to the same domain, in order to remove false positives due to GeoIP load balancing. """ res = client.createResolver(resolvconf=self.resolvconf, - servers=[(nameserver, 53)]) + servers=[(resolver, 53)]) ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa' - d = res.lookupPointer(ptr) - d.addCallback(lambda (ans, auth, add): util.println(ans[0].payload.name)) - d.addErrback(log.err) - d.addBoth(lambda r: reactor.stop()) - return d + reverse = res.lookupPointer(ptr) + reverse.addCallback(lambda (address, auth, add): + util.println(address[0].payload.name)) + reverse.addErrback(log.err) + + ## XXX do we need to stop the reactor? + #d.addBoth(lambda r: reactor.stop()) + + return reverse def experiment(self, args): """ Compares the lookup() sets of the control and experiment groups. """ - hostnames = args + for hostname in args: + for testresolver in self.assets['testresolvers']: + addressd = defer.Deferred() + addressd.addCallback(self.lookup, hostname, testresolver) + addressd.addErrback(log.err) - for hostname in hostnames: - for testserver in self.testservers: - #exp_address = self.lookup(hostname, testserver) - self.d.addCallback(self.lookup, hostname, testserver) + #addressd = self.lookup(hostname, testresolver) - #print self.assets['hostnames'] - #hostname = args - #exp_address = self.lookup(hostname, testserver) + #self.d.addCallback(self.lookup, hostname, testserver) - #return {'control': control_server, - # 'domain': args['asset'], - # 'experiment_address': address} + print "%s" % type(addressd) - if self.local_options['usereverse']: - exp_reversed = self.reverse_lookup(exp_address, testserver) - - ## XXX trying to fix errors: - #d = defer.Deferred() - - return (exp_address, hostname, testserver, exp_reversed) - else: - return (exp_address, hostname, testserver, False) + return addressd - def control(self, experiment_result): + def control(self, experiment_result, args): + print "EXPERIMENT RESULT IS %s" % experiment_result (exp_address, hostname, testserver, exp_reversed) = experiment_result control_server = self.local_options['controlserver'] ctrl_address = self.lookup(hostname, control_server)

1 0

[ooni-probe/master] Updating the ooni/lib structure and git submodules for imports, and I'm trying
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 4dcc00fdefef051ab27a03df94fa96730ece581b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 28 12:02:44 2012 +0000 Updating the ooni/lib structure and git submodules for imports, and I'm trying to keep the syntax the same. Updated the README with info on dev interaction with and user cloning of submodules. --- .gitmodules | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 07a4c56..daddf8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "ooni/lib/txscapy"] path = ooni/lib/txscapy - url = git://github.com/hellais/txscapy.git + url = https://github.com/isislovecruft/txscapy.git [submodule "ooni/lib/txtraceroute"] path = ooni/lib/txtraceroute - url = git://github.com/hellais/txtraceroute.git + url = https://github.com/isislovecruft/txtraceroute.git [submodule "ooni/lib/txtorcon"] path = ooni/lib/txtorcon - url = git://github.com/meejah/txtorcon.git + url = https://github.com/isislovecruft/txtorcon.git

1 0

[ooni-probe/master] Still getting the error from ../plugoo/test.py that "d" is a NoneType.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 2004db08a6c8f2c93bf3485864a3d6eba93b8ad0 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sun Aug 26 09:53:14 2012 +0000 Still getting the error from ../plugoo/test.py that "d" is a NoneType. I'll try merging master and see if that fixes it, else I'll have to patch that before continuing. --- ooni/plugins/dnstamper.py | 42 +++++++++++++++++++++++------------------- 1 files changed, 23 insertions(+), 19 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py index 8e8563a..54cfbdf 100644 --- a/ooni/plugins/dnstamper.py +++ b/ooni/plugins/dnstamper.py @@ -154,10 +154,10 @@ class DNSTamperTest(OONITest): def load_assets(self): assets = {} - - default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', - 'ooni.nu', 'google.com', 'torproject.org'] - default_resolvers = ['209.244.0.3', '208.67.222.222'] + + #default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com', + # 'ooni.nu', 'google.com', 'torproject.org'] + #default_resolvers = ['209.244.0.3', '208.67.222.222'] def asset_file(asset_option): return self.local_options[asset_option] @@ -172,24 +172,25 @@ class DNSTamperTest(OONITest): if self.local_options: if asset_file('hostnames'): - with asset_file('hostnames') as hosts_file: - ## The default filename for the Alexa Top 1 Million: - if hosts_file is 'top-1m.txt': - assets.update({'hostnames': AlexaAsset(hosts_file)}) - else: - assets.update({'hostnames': Asset(hosts_file)}) + ## The default filename for the Alexa Top 1 Million: + if assets_file('hostnames') is 'top-1m.txt': + assets.update({'hostnames': + AlexaAsset(assets_file('hostnames'))}) + else: + assets.update({'hostnames': + Asset(assets_file('hostnames'))}) else: log.msg("Error! We need an asset file containing the " + "hostnames that we should test DNS with! Please use " + "the '-h' option. Using pre-defined hostnames...") - assets.update({'hostnames': list_to_asset(default_hostnames)}) + #assets.update({'hostnames': list_to_asset(default_hostnames)}) if asset_file('testresolvers'): - with asset_file('testresolvers') as resolver_file: - assets.update({'testresolvers': Asset(resolver_file)}) - else: assets.update({'testresolvers': - list_to_asset(default_resolvers)}) + Asset(asset_file('testresolvers'))}) + #else: + # assets.update({'testresolvers': + # list_to_asset(asset_file('testresolvers'))}) return assets @@ -273,9 +274,12 @@ class DNSTamperTest(OONITest): """ for hostname in args: for testresolver in self.assets['testresolvers']: - addressd = defer.Deferred() - addressd.addCallback(self.lookup, hostname, testresolver) - addressd.addErrback(log.err) + #addressd = defer.Deferred() + #addressd.addCallback(self.lookup, hostname, testresolver) + #addressd.addErrback(log.err) + + self.d.addCallback(self.lookup, hostname, testresolver) + self.d.addErrback(log.err) #addressd = self.lookup(hostname, testresolver) @@ -283,7 +287,7 @@ class DNSTamperTest(OONITest): print "%s" % type(addressd) - return addressd + return self.d def control(self, experiment_result, args): print "EXPERIMENT RESULT IS %s" % experiment_result

1 0

[ooni-probe/master] Actually, everything is not working. I started to fix it, but I need to work
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 9c5d5d4b39babb714df86089e87db36af6562085 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 30 01:04:53 2012 +0000 Actually, everything is not working. I started to fix it, but I need to work on bridget, so I'm going to clone tpo:ooni-probe.git into a new branch now. --- ooni/ooniprobe.py | 6 +++--- ooni/plugoo/tests.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 7e9eecc..e60b956 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -27,9 +27,9 @@ from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation -from plugoo import tests, work, assets, reports -from logo import getlogo -import plugins, log +from ooni.plugoo import tests, work, assets, reports +from ooni.logo import getlogo +from ooni import plugins, log __version__ = "0.0.1-prealpha" diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 544d82d..4be0b62 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -7,7 +7,7 @@ import itertools from twisted.internet import reactor, defer, threads from twisted.python import failure -import log +from ooni import log from plugoo import assets, work from plugoo.reports import Report

1 0