September 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Only lookup test decks if they have not already been specified.
by art＠torproject.org 28 Sep '13

28 Sep '13

commit b9bc4f0ed03c51fb4e82c57c65dacebbacd4360c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Sep 11 18:15:36 2013 +0200 Only lookup test decks if they have not already been specified. --- decks/before_i_commit.testdeck | 4 ++++ ooni/deck.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index 69ef16c..9a3bca7 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -1,5 +1,6 @@ - options: collector: null + no-collector: true help: 0 logfile: reports/before_i_commit.log pcapfile: null @@ -8,6 +9,7 @@ test_file: manipulation/captiveportal - options: collector: null + no-collector: true help: 0 logfile: reports/before_i_commit.log pcapfile: null @@ -16,6 +18,7 @@ test_file: blocking/dnsconsistency - options: collector: null + no-collector: true help: 0 logfile: reports/before_i_commit.log pcapfile: null @@ -23,6 +26,7 @@ subargs: [-b, 'http://93.95.227.200', -f, example_inputs/http_host_file.txt] test_file: manipulation/http_host - options: + no-collector: true collector: null help: 0 logfile: reports/before_i_commit.log diff --git a/ooni/deck.py b/ooni/deck.py index 5cd8773..8820d12 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -148,9 +148,14 @@ class Deck(InputFile): required_test_helpers = [] for net_test_loader in self.netTestLoaders: for th in net_test_loader.requiredTestHelpers: + if th['test_class'].localOptions[th['option']]: + continue # {'name':'', 'option':'', 'test_class':''} required_test_helpers.append(th['name']) + if not required_test_helpers: + defer.returnValue(None) + response = yield oonibclient.lookupTestHelpers(required_test_helpers) for net_test_loader in self.netTestLoaders:

1 0

[ooni-probe/master] Merge remote-tracking branch 'thetorproject/master' into feature/move_nettests
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 40f5789e926d9af0479bd1f879ad9ab6053f36c9 Merge: b9bc4f0 c027888 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Sep 11 18:20:01 2013 +0200 Merge remote-tracking branch 'thetorproject/master' into feature/move_nettests * thetorproject/master: Add encoding format Add a version and author to dnsspoof Write unittest for testing https://github.com/TheTorProject/ooni-backend/pull/16 URL must be utf-8 ooni/deck.py | 2 +- ooni/nettests/manipulation/dnsspoof.py | 9 +++++++-- ooni/tests/test_oonibclient.py | 9 +++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --cc ooni/nettests/manipulation/dnsspoof.py index c9120a4,0000000..598c41f mode 100644,000000..100644 --- a/ooni/nettests/manipulation/dnsspoof.py +++ b/ooni/nettests/manipulation/dnsspoof.py @@@ -1,70 -1,0 +1,75 @@@ ++# -*- encoding: utf-8 -*- ++# ++# :authors: Arturo Filastò ++# :licence: see LICENSE ++ +from twisted.internet import defer +from twisted.python import usage + +from scapy.all import IP, UDP, DNS, DNSQR + +from ooni.templates import scapyt +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [['resolver', 'r', None, + 'Specify the resolver that should be used for DNS queries (ip:port)'], + ['hostname', 'h', None, + 'Specify the hostname of a censored site'], + ['backend', 'b', '8.8.8.8:53', + 'Specify the IP address of a good DNS resolver (ip:port)'] + ] + + +class DNSSpoof(scapyt.ScapyTest): + name = "DNS Spoof" ++ author = "Arturo Filastò" ++ version = "0.0.1" + timeout = 2 + + usageOptions = UsageOptions + + requiredTestHelpers = {'backend': 'dns'} + requiredOptions = ['hostname', 'resolver'] + + def setUp(self): + self.resolverAddr, self.resolverPort = self.localOptions['resolver'].split(':') + self.resolverPort = int(self.resolverPort) + + self.controlResolverAddr, self.controlResolverPort = self.localOptions['backend'].split(':') + self.controlResolverPort = int(self.controlResolverPort) + + self.hostname = self.localOptions['hostname'] + + def postProcessor(self, report): + """ + This is not tested, but the concept is that if the two responses + match up then spoofing is occuring. + """ + try: + test_answer = report['test_a_lookup']['answered_packets'][0][1] + control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + except IndexError: + self.report['spoofing'] = 'no_answer' + return + + if test_answer[UDP] == control_answer[UDP]: + self.report['spoofing'] = True + else: + self.report['spoofing'] = False + return + + @defer.inlineCallbacks + def test_a_lookup(self): + question = IP(dst=self.resolverAddr)/UDP()/DNS(rd=1, + qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) + log.msg("Performing query to %s with %s:%s" % (self.hostname, self.resolverAddr, self.resolverPort)) + yield self.sr1(question) + + @defer.inlineCallbacks + def test_control_a_lookup(self): + question = IP(dst=self.controlResolverAddr)/UDP()/DNS(rd=1, + qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) + log.msg("Performing query to %s with %s:%s" % (self.hostname, + self.controlResolverAddr, self.controlResolverPort)) + yield self.sr1(question) - -

1 0

[ooni-probe/master] Implement retry logic inside of the oonibclient.
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 1c3a5067ae9a26fc5c537e956e8c0847ad85fe4a Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 12 15:00:31 2013 +0200 Implement retry logic inside of the oonibclient. --- ooni/oonibclient.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 24dc374..455aea8 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -42,26 +42,41 @@ class Collector(object): return False class OONIBClient(object): + retries = 3 + def __init__(self, address): self.address = address self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port) def _request(self, method, urn, genReceiver, bodyProducer=None): - finished = defer.Deferred() + attempts = 0 - uri = self.address + urn - headers = {} - d = self.agent.request(method, uri, bodyProducer=bodyProducer) + finished = defer.Deferred() - @d.addCallback - def callback(response): - content_length = int(response.headers.getRawHeaders('content-length')[0]) - response.deliverBody(genReceiver(finished, content_length)) + def perform_request(): + uri = self.address + urn + headers = {} + d = self.agent.request(method, uri, bodyProducer=bodyProducer) - @d.addErrback - def eb(err): - finished.errback(err) + @d.addCallback + def callback(response): + content_length = int(response.headers.getRawHeaders('content-length')[0]) + response.deliverBody(genReceiver(finished, content_length)) + + def errback(err, attempts): + # We we will recursively keep trying to perform a request until + # we have reached the retry count. + if attempts < self.retries: + log.err("Lookup failed. Retrying.") + attempts += 1 + perform_request() + else: + log.err("Failed. Giving up.") + finished.errback(err) + d.addErrback(errback, attempts) + + perform_request() return finished

1 0

[ooni-probe/master] Do not submit probe_ip by default
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 8a3c6dbbfe6b47f97b4a053be4dd1029ec2e49d8 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 12 13:21:36 2013 +0200 Do not submit probe_ip by default --- data/ooniprobe.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index b86ff69..d651bbd 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -7,7 +7,7 @@ basic: logfile: /var/log/ooniprobe.log privacy: # Should we include the IP address of the probe in the report? - includeip: true + includeip: false # Should we include the ASN of the probe in the report? includeasn: true # Should we include the country as reported by GeoIP in the report?

1 0

[ooni-probe/master] Let ooni-probe pick the ports automatically.
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 399154e5c794f215261168b67ca3e5fd893bf0a1 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 12 13:09:54 2013 +0200 Let ooni-probe pick the ports automatically. --- data/ooniprobe.conf.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index b86ff69..12189cb 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -49,8 +49,8 @@ advanced: data_dir: /usr/share/ooni/ oonid_api_port: 8042 tor: - socks_port: 8801 - control_port: 8802 + #socks_port: 8801 + #control_port: 8802 # Specify the absolute path to the Tor bridges to use for testing #bridges: bridges.list # Specify path of the tor datadirectory.

1 0

[ooni-probe/master] Handle exceptions in backend queries better.
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 5817a3d920b1ba24ba295ac331df8871c33a095c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 12 15:00:47 2013 +0200 Handle exceptions in backend queries better. --- ooni/oonibclient.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 455aea8..6be7c44 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -87,7 +87,10 @@ class OONIBClient(object): def genReceiver(finished, content_length): def process_response(s): - response = json.loads(s) + try: + response = json.loads(s) + except ValueError: + raise e.get_error(None) if 'error' in response: print "Got this backend error message %s" % response log.err("Got this backend error message %s" % response) @@ -216,7 +219,8 @@ class OONIBClient(object): test_helper = yield self.queryBackend('POST', '/bouncer', query={'test-helpers': test_helper_names}) - except Exception: + except Exception, e: + log.exception(e) raise e.CouldNotFindTestHelper if not test_helper:

1 0

[ooni-probe/master] Merge branch 'master' into feature/move_nettests
by art＠torproject.org 28 Sep '13

28 Sep '13

commit e4f5b47d2eaf278adc1cd47505afa73b23b79f27 Merge: 40f5789 ee3911f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 12 12:47:17 2013 +0200 Merge branch 'master' into feature/move_nettests * master: Enforce str on IP address of backend. Change the default ports used by Tor. data/ooniprobe.conf.sample | 4 ++-- ooni/nettests/blocking/dnsconsistency.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --cc ooni/nettests/blocking/dnsconsistency.py index 3c88cd2,0000000..d883434 mode 100644,000000..100644 --- a/ooni/nettests/blocking/dnsconsistency.py +++ b/ooni/nettests/blocking/dnsconsistency.py @@@ -1,175 -1,0 +1,175 @@@ +# -*- encoding: utf-8 -*- +# +# dnsconsistency +# ************** +# +# The test reports censorship if the cardinality of the intersection of +# the query result set from the control server and the query result set +# from the experimental server is zero, which is to say, if the two sets +# have no matching results whatsoever. +# +# NOTE: This test frequently results in false positives due to GeoIP-based +# load balancing on major global sites such as google, facebook, and +# youtube, etc. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + +import pdb + +from twisted.python import usage +from twisted.internet import defer + +from ooni.templates import dnst + +from ooni import nettest +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', '8.8.8.8:53', + 'The OONI backend that runs the DNS resolver'], + ['testresolvers', 'T', None, + 'File containing list of DNS resolvers to test against'], + ['testresolver', 't', None, + 'Specify a single test resolver to use for testing'] + ] + +class DNSConsistencyTest(dnst.DNSTest): + + name = "DNS Consistency" + description = "DNS censorship detection test" + version = "0.6" + authors = "Arturo Filastò, Isis Lovecruft" + requirements = None + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + requiredTestHelpers = {'backend': 'dns'} + + usageOptions = UsageOptions + requiredOptions = ['backend', 'file'] + + def setUp(self): + if (not self.localOptions['testresolvers'] and \ + not self.localOptions['testresolver']): + raise usage.UsageError("You did not specify a testresolver") + + elif self.localOptions['testresolvers']: + test_resolvers_file = self.localOptions['testresolvers'] + + elif self.localOptions['testresolver']: + self.test_resolvers = [self.localOptions['testresolver']] + + try: + with open(test_resolvers_file) as f: + self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()] + self.report['test_resolvers'] = self.test_resolvers + f.close() + + except IOError, e: + log.exception(e) + raise usage.UsageError("Invalid test resolvers file") + + except NameError: + log.debug("No test resolver file configured") + + dns_ip, dns_port = self.localOptions['backend'].split(':') - self.control_dns_server = (dns_ip, int(dns_port)) ++ self.control_dns_server = (str(dns_ip), int(dns_port)) + + self.report['control_resolver'] = self.control_dns_server + + @defer.inlineCallbacks + def test_a_lookup(self): + """ + We perform an A lookup on the DNS test servers for the domains to be + tested and an A lookup on the known good DNS server. + + We then compare the results from test_resolvers and that from + control_resolver and see if the match up. + If they match up then no censorship is happening (tampering: false). + + If they do not we do a reverse lookup (PTR) on the test_resolvers and + the control resolver for every IP address we got back and check to see + if anyone of them matches the control ones. + + If they do then we take not of the fact that censorship is probably not + happening (tampering: reverse-match). + + If they do not match then censorship is probably going on (tampering: + true). + """ + log.msg("Doing the test lookups on %s" % self.input) + list_of_ds = [] + hostname = self.input + + self.report['tampering'] = {} + + control_answers = yield self.performALookup(hostname, self.control_dns_server) + if not control_answers: + log.err("Got no response from control DNS server %s," \ + " perhaps the DNS resolver is down?" % self.control_dns_server[0]) + self.report['tampering'][self.control_dns_server] = 'no_answer' + return + + for test_resolver in self.test_resolvers: + log.msg("Testing resolver: %s" % test_resolver) + test_dns_server = (test_resolver, 53) + + try: + experiment_answers = yield self.performALookup(hostname, test_dns_server) + except Exception, e: + log.err("Problem performing the DNS lookup") + log.exception(e) + self.report['tampering'][test_resolver] = 'dns_lookup_error' + continue + + if not experiment_answers: + log.err("Got no response, perhaps the DNS resolver is down?") + self.report['tampering'][test_resolver] = 'no_answer' + continue + else: + log.debug("Got the following A lookup answers %s from %s" % (experiment_answers, test_resolver)) + + def lookup_details(): + """ + A closure useful for printing test details. + """ + log.msg("test resolver: %s" % test_resolver) + log.msg("experiment answers: %s" % experiment_answers) + log.msg("control answers: %s" % control_answers) + + log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) + if set(experiment_answers) & set(control_answers): + lookup_details() + log.msg("tampering: false") + self.report['tampering'][test_resolver] = False + else: + log.msg("Trying to do reverse lookup") + + experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server) + control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server) + + if experiment_reverse == control_reverse: + log.msg("Further testing has eliminated false positives") + lookup_details() + log.msg("tampering: reverse_match") + self.report['tampering'][test_resolver] = 'reverse_match' + else: + log.msg("Reverse lookups do not match") + lookup_details() + log.msg("tampering: true") + self.report['tampering'][test_resolver] = True + + def inputProcessor(self, filename=None): + """ + This inputProcessor extracts domain names from urls + """ + log.debug("Running dnsconsistency default processor") + if filename: + fp = open(filename) + for x in fp.readlines(): + yield x.strip().split('//')[-1].split('/')[0] + fp.close() + else: + pass

1 0

[ooni-probe/master] Drops the hierarchy by reportFormatVersion
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 1c8485b81c57d8613c17de2a1a1b69125db92d74 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 12 15:04:48 2013 +0200 Drops the hierarchy by reportFormatVersion See also: https://github.com/TheTorProject/ooni-backend/issues/20 This was not implemented in ooni-probe or ooni-backend, and is redundant to the report, though if we decide to change the report header in the future we may need to handle this case in parsers. Adding new fields should generally not be a problem, though ooni-backend presently expects a fixed length report header. --- docs/source/reports.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/reports.rst b/docs/source/reports.rst index 77d8cf6..ad97411 100644 --- a/docs/source/reports.rst +++ b/docs/source/reports.rst @@ -2,7 +2,7 @@ Reports ======= The reports collected by ooniprobe are stored on -https://ooni.torproject.org/reports/ ``reportFormatVersion`` / ``CC`` / +https://ooni.torproject.org/reports/ ``CC`` / Where ``CC`` is the two letter country code as specified by `ISO 31666-2 <http://en.wikipedia.org/wiki/ISO_3166-2>`_. @@ -50,11 +50,9 @@ some malicious report poisoning attack in progress. Report format version changelog =============================== -In here shall go details about he major changes to the reporting format. +In here shall go details about the major changes to the reporting format. version 0.1 ----------- Initial format version. - -

1 0

[ooni-probe/master] Only lookup and set collectors when it is needed.
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 23567662bea818287c26a448c88b43fe8d6c641c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 12 15:00:07 2013 +0200 Only lookup and set collectors when it is needed. --- ooni/deck.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 22ae066..98d90e0 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -140,27 +140,34 @@ class Deck(InputFile): if self.bouncer: log.msg("Looking up test helpers...") yield self.lookupTestHelpers() - + @defer.inlineCallbacks def lookupTestHelpers(self): from ooni.oonibclient import OONIBClient oonibclient = OONIBClient(self.bouncer) required_test_helpers = [] + requires_collector = [] for net_test_loader in self.netTestLoaders: + if not net_test_loader.collector: + requires_collector.append(net_test_loader) + for th in net_test_loader.requiredTestHelpers: + # {'name':'', 'option':'', 'test_class':''} if th['test_class'].localOptions[th['option']]: continue - # {'name':'', 'option':'', 'test_class':''} required_test_helpers.append(th['name']) - - if not required_test_helpers: + + if not required_test_helpers and not requires_collector: defer.returnValue(None) response = yield oonibclient.lookupTestHelpers(required_test_helpers) for net_test_loader in self.netTestLoaders: log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name']) - if not net_test_loader.requiredTestHelpers: + + # Only set the collector if the no collector has been specified + # from the command line or via the test deck. + if not net_test_loader.requiredTestHelpers and net_test_loader in requires_collector: log.msg("Using the default collector: %s" % response['default']['collector']) net_test_loader.collector = response['default']['collector'].encode('utf-8') @@ -168,7 +175,8 @@ class Deck(InputFile): test_helper = response[th['name']] log.msg("Using this helper: %s" % test_helper) th['test_class'].localOptions[th['option']] = test_helper['address'] - net_test_loader.collector = test_helper['collector'].encode('utf-8') + if net_test_loader in requires_collector: + net_test_loader.collector = test_helper['collector'].encode('utf-8') @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader):

1 0

[ooni-probe/master] Remove all references to directory hierarchy
by art＠torproject.org 28 Sep '13

28 Sep '13

commit 846baedb38944f9381bb90a70fe0bf539488e775 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 12 15:26:18 2013 +0200 Remove all references to directory hierarchy Removes the rest of the references to the reportFormatVersion hierarchy. --- docs/source/reports.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/source/reports.rst b/docs/source/reports.rst index ad97411..3214683 100644 --- a/docs/source/reports.rst +++ b/docs/source/reports.rst @@ -7,10 +7,9 @@ https://ooni.torproject.org/reports/ ``CC`` / Where ``CC`` is the two letter country code as specified by `ISO 31666-2 <http://en.wikipedia.org/wiki/ISO_3166-2>`_. -For example the reports for Italy (``CC`` is ``it``) of the ``reportVersion`` 0.1 may -be found in: +For example the reports for Italy (``CC`` is ``it``) of the may be found in: -https://ooni.torproject.org/reports/0.1/IT/ +https://ooni.torproject.org/reports/IT/ This directory shall contain the various reports for the test using the @@ -38,8 +37,8 @@ For example if two report that are created on the first of January 2012 at Noon :: - https://ooni.torproject.org/reports/0.1/US/2012-01-01T120000Z_AS3.yamloo - https://ooni.torproject.org/reports/0.1/US/2012-01-01T120000Z_AS3.1.yamloo + https://ooni.torproject.org/reports/US/2012-01-01T120000Z_AS3.yamloo + https://ooni.torproject.org/reports/US/2012-01-01T120000Z_AS3.1.yamloo Note: it is highly unlikely that reports get created with the same exact

1 0