April 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Add support for Ubuntu releases in setup script
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 18e32fbb1505ba2833a96546fe9c535295b9e77b Author: anadahz <kojgelo(a)inbox.com> Date: Tue Apr 29 03:33:37 2014 +0300 Add support for Ubuntu releases in setup script Add support for Ubuntu releases in setup-dependencies.sh: * Add support for Ubuntu releases 13.04 (#9050) (https://trac.torproject.org/projects/tor/ticket/9050) * Add support for Ubuntu releases 13.10, 14.04 --- setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) … [View More]

1 0

[ooni-probe/master] Merge pull request #310 from anadahz/trac_7761
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 1ecf348b4a552a5ac7ec115b389d738983456c80 Merge: ecba7cd 10ca73f Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Mon Apr 28 16:59:05 2014 +0200 Merge pull request #310 from anadahz/trac_7761 Add ThreatModel page to ooni documentation (#7761) docs/source/index.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)

1 0

[ooni-probe/master] Merge pull request #313 from anadahz/trac_9050
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 096c8261c8980570ccc5f8f6082d44af86948218 Merge: 8337903 18e32fb Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Tue Apr 29 17:29:52 2014 +0200 Merge pull request #313 from anadahz/trac_9050 Trac 9050 setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] Merge pull request #312 from anadahz/setup-dependencies-fix
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 83379032606b1f2f133ca754eeae2272f12e4dff Merge: 1ecf348 c00f369 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Tue Apr 29 17:29:36 2014 +0200 Merge pull request #312 from anadahz/setup-dependencies-fix Add required packages is setup-dependencies.sh setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] Merge pull request #292 from TheTorProject/feature/test_naming_consistency
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 772c8835976b1bdbf7fd4f65dcaa1011e343803b Merge: 096c826 550a2f0 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 18:26:27 2014 +0200 Merge pull request #292 from TheTorProject/feature/test_naming_consistency Feature/test naming consistency data/decks/complete.deck | 2 +- data/decks/complete_no_root.deck | 2 +- data/inputs/README | 4 +- docs/source/tests/dnsconsistency.rst … [View More]

1 0

[ooni-probe/master] Merge pull request #293 from TheTorProject/refactoring/geoip
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 4f5d9107750521d746640659af27fd563eabd651 Merge: 772c883 d46b879 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Apr 30 18:27:11 2014 +0200 Merge pull request #293 from TheTorProject/refactoring/geoip Refactoring of geoip related functionality to have it less coupled ooni/director.py | 3 +-- ooni/geoip.py | 41 +++++++++++++++++++++++++++++++++++------ ooni/nettest.py | 38 ++++---------------------------------- ooni/settings.py … [View More]

1 0

[ooni-probe/master] Add support for adding tor version to the bridge_reachability report.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit eccaec864777ea75dfea5f48db63468cc378a6c4 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 24 15:21:13 2014 +0100 Add support for adding tor version to the bridge_reachability report. Fail with a good error code if the version is unsupported for the given bridge. --- ooni/director.py | 4 ++- ooni/managers.py | 7 ++-- ooni/nettests/blocking/bridge_reachability.py | 33 +++++++++---------- ooni/… [View More]tests/test_onion.py | 8 +++++ ooni/utils/__init__.py | 3 -- ooni/utils/onion.py | 43 +++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 24 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index ada1b82..b95639a 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -208,7 +208,6 @@ class Director(object): self.activeNetTests.remove(net_test) if len(self.activeNetTests) == 0: self.allTestsDone.callback(None) - self.allTestsDone = defer.Deferred() @defer.inlineCallbacks def startNetTest(self, net_test_loader, reporters): @@ -219,6 +218,9 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader """ + if self.allTestsDone.called: + self.allTestsDone = defer.Deferred() + if config.privacy.includepcap: if not config.reports.pcap: config.reports.pcap = config.generate_pcap_filename(net_test_loader.testDetails) diff --git a/ooni/managers.py b/ooni/managers.py index f6af282..cd05d42 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -30,6 +30,8 @@ class TaskManager(object): to be re-run once all the currently scheduled tasks have run. """ log.err("Task %s has failed %s times" % (task, task.failures)) + if config.advanced.debug: + log.exception(failure) self._active_tasks.remove(task) self.failures = self.failures + 1 @@ -81,10 +83,11 @@ class TaskManager(object): """ self._active_tasks.remove(task) - self._fillSlots() - # Fires the done deferred when the task has completed task.done.callback(result) + + self._fillSlots() + self.succeeded(result, task) @property diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 892ed02..9e6bb9e 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -9,7 +9,7 @@ from twisted.internet import defer, reactor, error import txtorcon -from ooni.utils import log +from ooni.utils import log, onion from ooni import nettest class UsageOptions(usage.Options): @@ -35,8 +35,11 @@ class BridgeReachability(nettest.NetTestCase): self.tor_progress = 0 self.timeout = int(self.localOptions['timeout']) + self.report['error'] = None + self.report['success'] = None self.report['timeout'] = self.timeout self.report['transport_name'] = 'vanilla' + self.report['tor_version'] = str(onion.tor_details['version']) self.report['tor_progress'] = 0 self.report['tor_progress_tag'] = None self.report['tor_progress_summary'] = None @@ -90,36 +93,30 @@ class BridgeReachability(nettest.NetTestCase): print "Failing bridges: %s" % failing_bridges def test_full_tor_connection(self): - def getTransport(address): - """ - If the address of the bridge starts with a valid c identifier then - we consider it to be a bridge. - Returns: - The transport_name if it's a transport. - None if it's not a obfsproxy bridge. - """ - transport_name = address.split(' ')[0] - transport_name_chars = string.ascii_letters + string.digits - if all(c in transport_name_chars for c in transport_name): - return transport_name - else: - return None - config = txtorcon.TorConfig() config.ControlPort = random.randint(2**14, 2**16) config.SocksPort = random.randint(2**14, 2**16) - transport_name = getTransport(self.bridge) + transport_name = onion.transport_name(self.bridge) if transport_name and self.pyobfsproxy_bin: config.ClientTransportPlugin = "%s exec %s managed" % (transport_name, self.pyobfsproxy_bin) self.report['transport_name'] = transport_name self.report['bridge_address'] = self.bridge.split(' ')[1] elif transport_name and not self.pyobfsproxy_bin: log.err("Unable to test bridge because pyobfsproxy is not installed") - self.report['success'] = None + self.report['error'] = 'missing-pyobfsproxy' return else: self.report['bridge_address'] = self.bridge.split(' ')[0] + + if transport_name and transport_name == 'scramblesuit' and \ + onion.TorVersion('0.2.5') > onion.tor_details['version']: + self.report['error'] = 'unsupported-tor-version' + return + elif transport_name and \ + onion.TorVersion('0.2.4.1') > onion.tor_details['version']: + self.report['error'] = 'unsupported-tor-version' + return config.Bridge = self.bridge config.UseBridges = 1 diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py new file mode 100644 index 0000000..9830235 --- /dev/null +++ b/ooni/tests/test_onion.py @@ -0,0 +1,8 @@ +from twisted.trial import unittest +from ooni.utils import onion + +class TestOnion(unittest.TestCase): + def test_tor_details(self): + assert isinstance(onion.tor_details, dict) + assert onion.tor_details['version'] + assert onion.tor_details['binary'] diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 1b6a8e9..3f47bd1 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -82,7 +82,6 @@ def randomStr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length)) - def pushFilenameStack(filename): """ Takes as input a target filename and checks to see if a file by such name @@ -103,5 +102,3 @@ def pushFilenameStack(filename): new_filename = "%s.%s" % (c_filename, new_idx) os.rename(f, new_filename) os.rename(filename, filename+".1") - - diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py new file mode 100644 index 0000000..91d8a2d --- /dev/null +++ b/ooni/utils/onion.py @@ -0,0 +1,43 @@ +import string +import subprocess +from distutils.version import LooseVersion + +from txtorcon.util import find_tor_binary + +class TorVersion(LooseVersion): + pass + +def tor_version(): + tor_binary = find_tor_binary() + if not tor_binary: + return None + try: + proc = subprocess.Popen((tor_binary, '--version'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + pass + else: + stdout, _ = proc.communicate() + if proc.poll() == 0 and stdout != '': + return TorVersion(stdout.strip().split(' ')[2]) + return None + +def transport_name(address): + """ + If the address of the bridge starts with a valid c identifier then + we consider it to be a bridge. + Returns: + The transport_name if it's a transport. + None if it's not a obfsproxy bridge. + """ + transport_name = address.split(' ')[0] + transport_name_chars = string.ascii_letters + string.digits + if all(c in transport_name_chars for c in transport_name): + return transport_name + else: + return None + +tor_details = { + 'binary': find_tor_binary(), + 'version': tor_version() +} [View Less]

1 0

[ooni-probe/master] Enforce better consistency amongst test naming.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 550a2f0e5684522ec90a130ddd9ad42e63c5175e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 7 19:30:23 2014 +0100 Enforce better consistency amongst test naming. --- data/decks/complete.deck | 2 +- data/decks/complete_no_root.deck | 2 +- data/inputs/README | 4 +- docs/source/tests/dnsconsistency.rst | 8 +- ooni/nettests/blocking/dns_consistency.py | 182 +++++++++++++++++++++++++++++ ooni/nettests/… [View More]blocking/dnsconsistency.py | 182 ----------------------------- ooni/nettests/blocking/http_requests.py | 2 +- ooni/nettests/manipulation/dns_spoof.py | 78 +++++++++++++ ooni/nettests/manipulation/dnsspoof.py | 78 ------------- ooni/nettests/manipulation/traceroute.py | 5 +- ooni/tests/test_director.py | 2 +- ooni/tests/test_oonicli.py | 2 +- 12 files changed, 273 insertions(+), 274 deletions(-) diff --git a/data/decks/complete.deck b/data/decks/complete.deck index 770b401..7d9346c 100644 --- a/data/decks/complete.deck +++ b/data/decks/complete.deck @@ -20,7 +20,7 @@ reportfile: null resume: 0 subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dnsconsistency + test_file: blocking/dns_consistency testdeck: null - options: collector: null diff --git a/data/decks/complete_no_root.deck b/data/decks/complete_no_root.deck index 8dbc147..ba2c29b 100644 --- a/data/decks/complete_no_root.deck +++ b/data/decks/complete_no_root.deck @@ -20,7 +20,7 @@ reportfile: null resume: 0 subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dnsconsistency + test_file: blocking/dns_consistency testdeck: null - options: collector: null diff --git a/data/inputs/README b/data/inputs/README index fb3599d..13657b1 100644 --- a/data/inputs/README +++ b/data/inputs/README @@ -1,9 +1,9 @@ In here you will find some very simple input lists that are useful for testing the correct functionality of the various OONIProbe tests. -# DNS Tamper +# DNS Consistency -./bin/ooniprobe -o dns_tamper_test.yamloo data/nettests/blocking/dnsconsistency.py -t +./bin/ooniprobe -o dns_tamper_test.yamloo data/nettests/blocking/dns_consistency.py -t example_inputs/dns_tamper_test_resolvers.txt -f example_inputs/dns_tamper_file.txt less dns_tamper_test.yamloo diff --git a/docs/source/tests/dnsconsistency.rst b/docs/source/tests/dnsconsistency.rst index be88c82..7ae06f2 100644 --- a/docs/source/tests/dnsconsistency.rst +++ b/docs/source/tests/dnsconsistency.rst @@ -5,7 +5,7 @@ Details *Current version*: 0.4 -*NetTest*: DNS Consistency Test (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/ooni/nettests/block…) +*NetTest*: DNS Consistency Test (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/blocking/d…) *Test Helper*: DNS Test Helper (https://gitweb.torproject.org/oonib.git/blob/HEAD:/oonib/testhelpers/dns_he…) @@ -28,7 +28,7 @@ Youtube, etc. How to run the test =================== -`ooniprobe blocking/dnsconsistency [-t <test resolvers>|-T <test resolver file>-f <input file> -b IP:PORT` +`ooniprobe blocking/dns_consistency [-t <test resolvers>|-T <test resolver file>-f <input file> -b IP:PORT` *test resolvers* is a single test resolver (IP address) @@ -42,7 +42,7 @@ Sample report ============= From running: -`ooniprobe blocking/dnsconsistency -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/http_host_file.txt` +`ooniprobe blocking/dns_consistency -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/http_host_file.txt` :: @@ -59,7 +59,7 @@ From running: reportfile: null resume: 0 subargs: [-t, 8.8.8.8, -f, test_inputs/dns_tamper_file.txt] - test: nettests/blocking/dnsconsistency.py + test: nettests/blocking/dns_consistency.py probe_asn: null probe_cc: null probe_ip: 127.0.0.1 diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py new file mode 100644 index 0000000..5d17741 --- /dev/null +++ b/ooni/nettests/blocking/dns_consistency.py @@ -0,0 +1,182 @@ +# -*- encoding: utf-8 -*- +# +# dnsconsistency +# ************** +# +# The test reports censorship if the cardinality of the intersection of +# the query result set from the control server and the query result set +# from the experimental server is zero, which is to say, if the two sets +# have no matching results whatsoever. +# +# NOTE: This test frequently results in false positives due to GeoIP-based +# load balancing on major global sites such as google, facebook, and +# youtube, etc. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + +import pdb + +from twisted.python import usage +from twisted.internet import defer + +from ooni.templates import dnst + +from ooni import nettest +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', None, + 'The OONI backend that runs the DNS resolver'], + ['testresolvers', 'T', None, + 'File containing list of DNS resolvers to test against'], + ['testresolver', 't', None, + 'Specify a single test resolver to use for testing'] + ] + +class DNSConsistencyTest(dnst.DNSTest): + + name = "DNS Consistency" + description = "Checks to see if the DNS responses from a set of DNS resolvers are consistent." + version = "0.6" + authors = "Arturo Filastò, Isis Lovecruft" + requirements = None + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + requiredTestHelpers = {'backend': 'dns'} + requiresRoot = False + requiresTor = False + + usageOptions = UsageOptions + requiredOptions = ['backend', 'file'] + + def setUp(self): + if (not self.localOptions['testresolvers'] and \ + not self.localOptions['testresolver']): + self.test_resolvers = [] + with open('/etc/resolv.conf') as f: + for line in f: + if line.startswith('nameserver'): + self.test_resolvers.append(line.split(' ')[1].strip()) + self.report['test_resolvers'] = self.test_resolvers + + elif self.localOptions['testresolvers']: + test_resolvers_file = self.localOptions['testresolvers'] + + elif self.localOptions['testresolver']: + self.test_resolvers = [self.localOptions['testresolver']] + + try: + with open(test_resolvers_file) as f: + self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()] + self.report['test_resolvers'] = self.test_resolvers + f.close() + + except IOError, e: + log.exception(e) + raise usage.UsageError("Invalid test resolvers file") + + except NameError: + log.debug("No test resolver file configured") + + dns_ip, dns_port = self.localOptions['backend'].split(':') + self.control_dns_server = (str(dns_ip), int(dns_port)) + + self.report['control_resolver'] = "%s:%d" % self.control_dns_server + + @defer.inlineCallbacks + def test_a_lookup(self): + """ + We perform an A lookup on the DNS test servers for the domains to be + tested and an A lookup on the known good DNS server. + + We then compare the results from test_resolvers and that from + control_resolver and see if the match up. + If they match up then no censorship is happening (tampering: false). + + If they do not we do a reverse lookup (PTR) on the test_resolvers and + the control resolver for every IP address we got back and check to see + if anyone of them matches the control ones. + + If they do then we take not of the fact that censorship is probably not + happening (tampering: reverse-match). + + If they do not match then censorship is probably going on (tampering: + true). + """ + log.msg("Doing the test lookups on %s" % self.input) + list_of_ds = [] + hostname = self.input + + self.report['tampering'] = {} + + control_answers = yield self.performALookup(hostname, self.control_dns_server) + if not control_answers: + log.err("Got no response from control DNS server %s:%d," \ + " perhaps the DNS resolver is down?" % self.control_dns_server[0]) + self.report['tampering']["%s:%d" % self.control_dns_server] = 'no_answer' + return + + for test_resolver in self.test_resolvers: + log.msg("Testing resolver: %s" % test_resolver) + test_dns_server = (test_resolver, 53) + + try: + experiment_answers = yield self.performALookup(hostname, test_dns_server) + except Exception, e: + log.err("Problem performing the DNS lookup") + log.exception(e) + self.report['tampering'][test_resolver] = 'dns_lookup_error' + continue + + if not experiment_answers: + log.err("Got no response, perhaps the DNS resolver is down?") + self.report['tampering'][test_resolver] = 'no_answer' + continue + else: + log.debug("Got the following A lookup answers %s from %s" % (experiment_answers, test_resolver)) + + def lookup_details(): + """ + A closure useful for printing test details. + """ + log.msg("test resolver: %s" % test_resolver) + log.msg("experiment answers: %s" % experiment_answers) + log.msg("control answers: %s" % control_answers) + + log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) + if set(experiment_answers) & set(control_answers): + lookup_details() + log.msg("tampering: false") + self.report['tampering'][test_resolver] = False + else: + log.msg("Trying to do reverse lookup") + + experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server) + control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server) + + if experiment_reverse == control_reverse: + log.msg("Further testing has eliminated false positives") + lookup_details() + log.msg("tampering: reverse_match") + self.report['tampering'][test_resolver] = 'reverse_match' + else: + log.msg("Reverse lookups do not match") + lookup_details() + log.msg("tampering: true") + self.report['tampering'][test_resolver] = True + + def inputProcessor(self, filename=None): + """ + This inputProcessor extracts domain names from urls + """ + log.debug("Running dnsconsistency default processor") + if filename: + fp = open(filename) + for x in fp.readlines(): + yield x.strip().split('//')[-1].split('/')[0] + fp.close() + else: + pass diff --git a/ooni/nettests/blocking/dnsconsistency.py b/ooni/nettests/blocking/dnsconsistency.py deleted file mode 100644 index 5d17741..0000000 --- a/ooni/nettests/blocking/dnsconsistency.py +++ /dev/null @@ -1,182 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# dnsconsistency -# ************** -# -# The test reports censorship if the cardinality of the intersection of -# the query result set from the control server and the query result set -# from the experimental server is zero, which is to say, if the two sets -# have no matching results whatsoever. -# -# NOTE: This test frequently results in false positives due to GeoIP-based -# load balancing on major global sites such as google, facebook, and -# youtube, etc. -# -# :authors: Arturo Filastò, Isis Lovecruft -# :licence: see LICENSE - -import pdb - -from twisted.python import usage -from twisted.internet import defer - -from ooni.templates import dnst - -from ooni import nettest -from ooni.utils import log - -class UsageOptions(usage.Options): - optParameters = [['backend', 'b', None, - 'The OONI backend that runs the DNS resolver'], - ['testresolvers', 'T', None, - 'File containing list of DNS resolvers to test against'], - ['testresolver', 't', None, - 'Specify a single test resolver to use for testing'] - ] - -class DNSConsistencyTest(dnst.DNSTest): - - name = "DNS Consistency" - description = "Checks to see if the DNS responses from a set of DNS resolvers are consistent." - version = "0.6" - authors = "Arturo Filastò, Isis Lovecruft" - requirements = None - - inputFile = ['file', 'f', None, - 'Input file of list of hostnames to attempt to resolve'] - - requiredTestHelpers = {'backend': 'dns'} - requiresRoot = False - requiresTor = False - - usageOptions = UsageOptions - requiredOptions = ['backend', 'file'] - - def setUp(self): - if (not self.localOptions['testresolvers'] and \ - not self.localOptions['testresolver']): - self.test_resolvers = [] - with open('/etc/resolv.conf') as f: - for line in f: - if line.startswith('nameserver'): - self.test_resolvers.append(line.split(' ')[1].strip()) - self.report['test_resolvers'] = self.test_resolvers - - elif self.localOptions['testresolvers']: - test_resolvers_file = self.localOptions['testresolvers'] - - elif self.localOptions['testresolver']: - self.test_resolvers = [self.localOptions['testresolver']] - - try: - with open(test_resolvers_file) as f: - self.test_resolvers = [x.split('#')[0].strip() for x in f.readlines()] - self.report['test_resolvers'] = self.test_resolvers - f.close() - - except IOError, e: - log.exception(e) - raise usage.UsageError("Invalid test resolvers file") - - except NameError: - log.debug("No test resolver file configured") - - dns_ip, dns_port = self.localOptions['backend'].split(':') - self.control_dns_server = (str(dns_ip), int(dns_port)) - - self.report['control_resolver'] = "%s:%d" % self.control_dns_server - - @defer.inlineCallbacks - def test_a_lookup(self): - """ - We perform an A lookup on the DNS test servers for the domains to be - tested and an A lookup on the known good DNS server. - - We then compare the results from test_resolvers and that from - control_resolver and see if the match up. - If they match up then no censorship is happening (tampering: false). - - If they do not we do a reverse lookup (PTR) on the test_resolvers and - the control resolver for every IP address we got back and check to see - if anyone of them matches the control ones. - - If they do then we take not of the fact that censorship is probably not - happening (tampering: reverse-match). - - If they do not match then censorship is probably going on (tampering: - true). - """ - log.msg("Doing the test lookups on %s" % self.input) - list_of_ds = [] - hostname = self.input - - self.report['tampering'] = {} - - control_answers = yield self.performALookup(hostname, self.control_dns_server) - if not control_answers: - log.err("Got no response from control DNS server %s:%d," \ - " perhaps the DNS resolver is down?" % self.control_dns_server[0]) - self.report['tampering']["%s:%d" % self.control_dns_server] = 'no_answer' - return - - for test_resolver in self.test_resolvers: - log.msg("Testing resolver: %s" % test_resolver) - test_dns_server = (test_resolver, 53) - - try: - experiment_answers = yield self.performALookup(hostname, test_dns_server) - except Exception, e: - log.err("Problem performing the DNS lookup") - log.exception(e) - self.report['tampering'][test_resolver] = 'dns_lookup_error' - continue - - if not experiment_answers: - log.err("Got no response, perhaps the DNS resolver is down?") - self.report['tampering'][test_resolver] = 'no_answer' - continue - else: - log.debug("Got the following A lookup answers %s from %s" % (experiment_answers, test_resolver)) - - def lookup_details(): - """ - A closure useful for printing test details. - """ - log.msg("test resolver: %s" % test_resolver) - log.msg("experiment answers: %s" % experiment_answers) - log.msg("control answers: %s" % control_answers) - - log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) - if set(experiment_answers) & set(control_answers): - lookup_details() - log.msg("tampering: false") - self.report['tampering'][test_resolver] = False - else: - log.msg("Trying to do reverse lookup") - - experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server) - control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server) - - if experiment_reverse == control_reverse: - log.msg("Further testing has eliminated false positives") - lookup_details() - log.msg("tampering: reverse_match") - self.report['tampering'][test_resolver] = 'reverse_match' - else: - log.msg("Reverse lookups do not match") - lookup_details() - log.msg("tampering: true") - self.report['tampering'][test_resolver] = True - - def inputProcessor(self, filename=None): - """ - This inputProcessor extracts domain names from urls - """ - log.debug("Running dnsconsistency default processor") - if filename: - fp = open(filename) - for x in fp.readlines(): - yield x.strip().split('//')[-1].split('/')[0] - fp.close() - else: - pass diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 84c80dc..c660400 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -27,7 +27,7 @@ class HTTPRequestsTest(httpt.HTTPTest): We check to see if the response headers match and if the response body lengths match. """ - name = "HTTP Requests Test" + name = "HTTP Requests" description = "Performs a HTTP GET request over Tor and one over the local network and compares the two results." author = "Arturo Filastò" version = "0.2.4" diff --git a/ooni/nettests/manipulation/dns_spoof.py b/ooni/nettests/manipulation/dns_spoof.py new file mode 100644 index 0000000..15aad07 --- /dev/null +++ b/ooni/nettests/manipulation/dns_spoof.py @@ -0,0 +1,78 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from twisted.internet import defer +from twisted.python import usage + +from scapy.all import IP, UDP, DNS, DNSQR + +from ooni.templates import scapyt +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [['resolver', 'r', None, + 'Specify the resolver that should be used for DNS queries (ip:port)'], + ['hostname', 'h', None, + 'Specify the hostname of a censored site'], + ['backend', 'b', None, + 'Specify the IP address of a good DNS resolver (ip:port)'] + ] + + +class DNSSpoof(scapyt.ScapyTest): + name = "DNS Spoof" + description = "Used to validate if the type of censorship happening is DNS spoofing or not." + author = "Arturo Filastò" + version = "0.0.1" + timeout = 2 + + usageOptions = UsageOptions + + requiredTestHelpers = {'backend': 'dns'} + requiredOptions = ['hostname', 'resolver'] + requiresRoot = True + requiresTor = False + + def setUp(self): + self.resolverAddr, self.resolverPort = self.localOptions['resolver'].split(':') + self.resolverPort = int(self.resolverPort) + + self.controlResolverAddr, self.controlResolverPort = self.localOptions['backend'].split(':') + self.controlResolverPort = int(self.controlResolverPort) + + self.hostname = self.localOptions['hostname'] + + def postProcessor(self, report): + """ + This is not tested, but the concept is that if the two responses + match up then spoofing is occuring. + """ + try: + test_answer = report['test_a_lookup']['answered_packets'][0][1] + control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + except IndexError: + self.report['spoofing'] = 'no_answer' + return + + if test_answer[UDP] == control_answer[UDP]: + self.report['spoofing'] = True + else: + self.report['spoofing'] = False + return + + @defer.inlineCallbacks + def test_a_lookup(self): + question = IP(dst=self.resolverAddr)/UDP()/DNS(rd=1, + qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) + log.msg("Performing query to %s with %s:%s" % (self.hostname, self.resolverAddr, self.resolverPort)) + yield self.sr1(question) + + @defer.inlineCallbacks + def test_control_a_lookup(self): + question = IP(dst=self.controlResolverAddr)/UDP()/DNS(rd=1, + qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) + log.msg("Performing query to %s with %s:%s" % (self.hostname, + self.controlResolverAddr, self.controlResolverPort)) + yield self.sr1(question) diff --git a/ooni/nettests/manipulation/dnsspoof.py b/ooni/nettests/manipulation/dnsspoof.py deleted file mode 100644 index 15aad07..0000000 --- a/ooni/nettests/manipulation/dnsspoof.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from twisted.internet import defer -from twisted.python import usage - -from scapy.all import IP, UDP, DNS, DNSQR - -from ooni.templates import scapyt -from ooni.utils import log - -class UsageOptions(usage.Options): - optParameters = [['resolver', 'r', None, - 'Specify the resolver that should be used for DNS queries (ip:port)'], - ['hostname', 'h', None, - 'Specify the hostname of a censored site'], - ['backend', 'b', None, - 'Specify the IP address of a good DNS resolver (ip:port)'] - ] - - -class DNSSpoof(scapyt.ScapyTest): - name = "DNS Spoof" - description = "Used to validate if the type of censorship happening is DNS spoofing or not." - author = "Arturo Filastò" - version = "0.0.1" - timeout = 2 - - usageOptions = UsageOptions - - requiredTestHelpers = {'backend': 'dns'} - requiredOptions = ['hostname', 'resolver'] - requiresRoot = True - requiresTor = False - - def setUp(self): - self.resolverAddr, self.resolverPort = self.localOptions['resolver'].split(':') - self.resolverPort = int(self.resolverPort) - - self.controlResolverAddr, self.controlResolverPort = self.localOptions['backend'].split(':') - self.controlResolverPort = int(self.controlResolverPort) - - self.hostname = self.localOptions['hostname'] - - def postProcessor(self, report): - """ - This is not tested, but the concept is that if the two responses - match up then spoofing is occuring. - """ - try: - test_answer = report['test_a_lookup']['answered_packets'][0][1] - control_answer = report['test_control_a_lookup']['answered_packets'][0][1] - except IndexError: - self.report['spoofing'] = 'no_answer' - return - - if test_answer[UDP] == control_answer[UDP]: - self.report['spoofing'] = True - else: - self.report['spoofing'] = False - return - - @defer.inlineCallbacks - def test_a_lookup(self): - question = IP(dst=self.resolverAddr)/UDP()/DNS(rd=1, - qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) - log.msg("Performing query to %s with %s:%s" % (self.hostname, self.resolverAddr, self.resolverPort)) - yield self.sr1(question) - - @defer.inlineCallbacks - def test_control_a_lookup(self): - question = IP(dst=self.controlResolverAddr)/UDP()/DNS(rd=1, - qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) - log.msg("Performing query to %s with %s:%s" % (self.hostname, - self.controlResolverAddr, self.controlResolverPort)) - yield self.sr1(question) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index 66e0a34..c6e167c 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -22,9 +22,8 @@ class UsageOptions(usage.Options): ['numPackets', 'n', None, 'Specify the number of packets to send per hop'], ] -class TracerouteTest(scapyt.BaseScapyTest): - name = "Multi Protocol Traceroute Test" - +class Traceroute(scapyt.BaseScapyTest): + name = "Traceroute" description = "Performs a UDP, TCP, ICMP traceroute with destination port number set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535" requiredTestHelpers = {'backend': 'traceroute'} diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 9d86720..23afe30 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -34,7 +34,7 @@ class TestDirector(unittest.TestCase): director = Director() nettests = director.getNetTests() assert 'http_requests' in nettests - assert 'dnsconsistency' in nettests + assert 'dns_consistency' in nettests assert 'http_header_field_manipulation' in nettests assert 'traceroute' in nettests diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 7050c5a..b53b510 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -93,7 +93,7 @@ class TestRunDirector(unittest.TestCase): assert 'control_resolver' in entry assert 'tampering' in entry assert len(entry['tampering']) == 1 - yield self.run_test('blocking/dnsconsistency', + yield self.run_test('blocking/dns_consistency', ['-b', '8.8.8.8:53', '-t', '8.8.8.8', '-f', 'example-input.txt'], [View Less]

1 0

[ooni-probe/master] Use the correct version to check compatibility with scramblesuit
by art＠torproject.org 30 Apr '14

30 Apr '14

commit facacd6855e2165848a8ab71c6dd70944721bd69 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 24 15:41:58 2014 +0100 Use the correct version to check compatibility with scramblesuit --- ooni/nettests/blocking/bridge_reachability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 9e6bb9e..54bce73 100644 --- a/ooni/nettests/blocking/bridge_reachability.… [View More]

1 0

[ooni-probe/master] Provide a useful error message if we fail to load the inputfile
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 7e4db43bc305d6dab716e5cb1f04a92388d58a34 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 25 17:14:40 2014 +0100 Provide a useful error message if we fail to load the inputfile --- ooni/errors.py | 3 +++ ooni/nettest.py | 11 +++++++---- ooni/oonicli.py | 10 ++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 4ee822e..3b62b2a 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -220,6 +220,9 @@ … [View More]class InvalidOption(Exception): class TaskTimedOut(Exception): pass +class InvalidInputFile(Exception): + pass + def get_error(error_key): if error_key == 'test-helpers-key-missing': return CouldNotFindTestHelper diff --git a/ooni/nettest.py b/ooni/nettest.py index 4db7af2..aba768b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -185,10 +185,13 @@ class NetTestLoader(object): input_file['hash'] = m.group(2) else: input_file['filename'] = filename - with open(filename) as f: - h = sha256() - for l in f: - h.update(l) + try: + with open(filename) as f: + h = sha256() + for l in f: + h.update(l) + except: + raise e.InvalidInputFile(filename) input_file['hash'] = h.hexdigest() input_files.append(input_file) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5c3c0ce..6f4ebe0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -181,7 +181,8 @@ def runWithDirector(logging=True, start_tor=True): r = failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, - errors.CouldNotFindTestCollector, errors.ProbeIPUnknown) + errors.CouldNotFindTestCollector, errors.ProbeIPUnknown, + errors.InvalidInputFile) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") @@ -210,11 +211,12 @@ def runWithDirector(logging=True, start_tor=True): log.err("Failed to lookup probe IP address.") log.msg("Check your internet connection.") + elif isinstance(failure.value, errors.InvalidInputFile): + log.err("Invalid input file \"%s\"" % failure.value) + if config.advanced.debug: log.exception(failure) - - reactor.stop() - + # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): [View Less]

1 0