November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Update .gitignore to ignore ooniprobe.conf
by art＠torproject.org 28 Nov '12

28 Nov '12

commit cd644817d49784eac341d4625a8eed17a5f328f6 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 18:18:33 2012 +0100 Update .gitignore to ignore ooniprobe.conf --- .gitignore | 2 +- ooniprobe.conf | 38 -------------------------------------- ooniprobe.conf.sample | 5 +++-- 3 files changed, 4 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 4cb0775..2dada89 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ pcaps/ report*.yaml docs/build/* data/*.dat -ooniprobe.conf +./ooniprobe.conf diff --git a/ooniprobe.conf b/ooniprobe.conf deleted file mode 100644 index ed77dfe..0000000 --- a/ooniprobe.conf +++ /dev/null @@ -1,38 +0,0 @@ -# This is the configuration file for OONIProbe -# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. - -basic: - # Where OONIProbe should be writing it's log file - logfile: ooniprobe.log -privacy: - # Should we include the IP address of the probe in the report? - includeip: false - # Should we include the ASN of the probe in the report? - includeasn: false - # Should we include the ASN of the probe in the report? - includecountry: false - # Should we include the ASN of the probe in the report? - includecity: false - # Should we collect a full packet capture on the client? - includepcap: false -advanced: - # XXX change this to point to the directory where you have stored the GeoIP - # database file. This should be the directory in which OONI is installed - # /path/to/ooni-probe/data/ - geoip_data_dir: /usr/share/GeoIP/ - debug: true - tor_binary: '/usr/sbin/tor' - # For auto detection - interface: auto - # Of specify a specific interface - #interface: wlan0 - # If you do not specify start_tor, you will have to have Tor running and - # explicitly set the control port and orport. - start_tor: true -tor: - #socks_port: 9050 - #control_port: 9051 - # Specify the absolute path to the Tor bridges to use for testing - bridges: bridges.list - diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index a032c0c..551e4f0 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -22,13 +22,14 @@ advanced: # /path/to/ooni-probe/data/ geoip_data_dir: /usr/share/GeoIP/ debug: true - threadpool_size: 10 tor_binary: '/usr/sbin/tor' # For auto detection interface: auto # Of specify a specific interface #interface: wlan0 - start_tor: true + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and orport. + start_tor: false tor: #socks_port: 9050 #control_port: 9051

1 0

[ooni-probe/master] Add instructions to redirect low ports with iptables
by art＠torproject.org 28 Nov '12

28 Nov '12

commit b85aa044fc3d54fa41f9cbd42c45ba48f3d2152a Author: aagbsn <aagbsn(a)extc.org> Date: Wed Nov 28 17:05:34 2012 +0000 Add instructions to redirect low ports with iptables --- oonib/README.md | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/oonib/README.md b/oonib/README.md index f70c6db..27c1163 100644 --- a/oonib/README.md +++ b/oonib/README.md @@ -15,4 +15,9 @@ The extra dependencies necessary to run OONIB are: openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt rm private.key.org +# Redirect low ports with iptables + # Map port 80 to config.helpers.http_return_request.port (default: 57001) + iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 + # Map port 443 to config.helpers.ssl.port (default: 57006) + iptables -t nat -A PREROUTING -p tcp -m tcp --dport 443 -j REDIRECT --to-ports 57006

1 0

[ooni-probe/master] Merge patch from aagbsn for setup of iptables rules
by art＠torproject.org 28 Nov '12

28 Nov '12

commit d4642eee506dd5e9f0a1cec408dc8e45e3273820 Merge: cd64481 0849bf9 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 18:20:50 2012 +0100 Merge patch from aagbsn for setup of iptables rules oonib/README.md | 5 +++++ oonib/config.py | 2 +- 2 files changed, 6 insertions(+), 1 deletions(-)

1 0

[ooni-probe/master] Fix before I commit script
by art＠torproject.org 28 Nov '12

28 Nov '12

commit 37dd647d6def02f080c343454cb51239dc01b832 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 18:04:11 2012 +0100 Fix before I commit script --- before_i_commit.testdeck | 22 +++------------------- ooni/oonicli.py | 2 ++ ooni/runner.py | 2 +- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck index 7fb33f9..af420be 100644 --- a/before_i_commit.testdeck +++ b/before_i_commit.testdeck @@ -5,7 +5,7 @@ pcapfile: null reportfile: captive_portal_test.yamloo subargs: [] - test: nettests/core/captiveportal.py + test: nettests/manipulation/captiveportal.py - options: collector: null help: 0 @@ -13,7 +13,7 @@ pcapfile: null reportfile: dns_tamper_test.yamloo subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt] - test: nettests/core/dnstamper.py + test: nettests/blocking/dnstamper.py - options: collector: null help: 0 @@ -21,23 +21,7 @@ pcapfile: null reportfile: http_host.yamloo subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt] - test: nettests/core/http_host.py -- options: - collector: null - help: 0 - logfile: before_i_commit.log - pcapfile: null - reportfile: http_keyword_filtering.yamloo - subargs: [-b, 'http://ooni.nu/test/', -f, test_inputs/keyword_filtering_file.txt] - test: nettests/core/http_keyword_filtering.py -- options: - collector: null - help: 0 - logfile: before_i_commit.log - pcapfile: null - reportfile: http_url_lists.yamloo - subargs: [-f, test_inputs/url_lists_file.txt] - test: nettests/core/http_url_list.py + test: nettests/manipulation/http_host.py # XXX this is disabled because it requires oonib to be running #- options: # collector: null diff --git a/ooni/oonicli.py b/ooni/oonicli.py index ca80ad7..f706ee3 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -166,6 +166,8 @@ def run(): cmd_line_options = test['options'] if resume: cmd_line_options['resume'] = True + else: + cmd_line_options['resume'] = False test_list.append(runner.loadTest(cmd_line_options)) else: log.msg("No test deck detected") diff --git a/ooni/runner.py b/ooni/runner.py index 9bdedfc..4f59d17 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -442,7 +442,7 @@ def runTestCases(test_cases, options, cmd_line_options): log.debug("Resuming %s" % test_filename) resumeTest(test_filename, input_unit_factory) else: - log.debug("Nothing to resume for %s" % test_filename) + log.debug("Not going to resume %s" % test_filename) config.stateDict[test_filename] = 0 updateProgressMeters(test_filename, input_unit_factory, len(test_cases))

1 0

[ooni-probe/master] Set the required Tor version 2.2.x
by art＠torproject.org 28 Nov '12

28 Nov '12

commit 47174997095cd4fa0ae2c828944e07a31aa8b65c Author: Arturo Filastò <hellais(a)gmail.com> Date: Tue Nov 27 15:48:28 2012 +0100 Set the required Tor version 2.2.x This is needed because we want to use OwningController: https://trac.torproject.org/projects/tor/ticket/3049 --- README.md | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/README.md b/README.md index 9f7cfb1..4c3c916 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ On debian based systems these can be installed with: The python dependencies required for running ooniprobe are: - * Tor: http://torproject.org/ + * Tor (>2.2.x): http://torproject.org/ * Twisted (>12.1.0): http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/

1 0

[ooni-probe/master] Add tor to the list of dependencies
by art＠torproject.org 28 Nov '12

28 Nov '12

commit 86954cf1784ddbb7d645a3042158c7313d67a4a1 Author: Arturo Filastò <hellais(a)gmail.com> Date: Tue Nov 27 15:46:28 2012 +0100 Add tor to the list of dependencies --- README.md | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/README.md b/README.md index 4100541..9f7cfb1 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,11 @@ Basic requirements: On debian based systems these can be installed with: - sudo apt-get install git-core python python-pip python-dev build-essential + sudo apt-get install git-core python python-pip python-dev build-essential tor The python dependencies required for running ooniprobe are: + * Tor: http://torproject.org/ * Twisted (>12.1.0): http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/

1 0

[ooni-probe/master] Merge branch 'master' of github.com:hellais/ooni-probe
by art＠torproject.org 28 Nov '12

28 Nov '12

commit 76fc0f052e2d8f00b89f6250d0451b726eabb1e4 Merge: fe417b1 f72bfc1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 17:50:16 2012 +0100 Merge branch 'master' of github.com:hellais/ooni-probe * 'master' of github.com:hellais/ooni-probe: Add install instructions for latest version of Tor Set the required Tor version 2.2.x Add tor to the list of dependencies README.md | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-)

1 0

[ooni-probe/master] Add install instructions for latest version of Tor
by art＠torproject.org 28 Nov '12

28 Nov '12

commit f72bfc1167996be42b8b49af6b4c81382a161465 Author: Arturo Filastò <hellais(a)gmail.com> Date: Tue Nov 27 15:56:35 2012 +0100 Add install instructions for latest version of Tor --- README.md | 15 ++++++++++++++- 1 files changed, 14 insertions(+), 1 deletions(-) diff --git a/README.md b/README.md index 4c3c916..43bb6b0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Basic requirements: On debian based systems these can be installed with: - sudo apt-get install git-core python python-pip python-dev build-essential tor + sudo apt-get install git-core python python-pip python-dev build-essential The python dependencies required for running ooniprobe are: @@ -38,6 +38,19 @@ The python dependencies required for running ooniprobe are: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ * txtorcon: https://github.com/meejah/txtorcon +## Install Tor + +To get the latest version of Tor you should do the following (from: https://www.torproject.org/docs/debian) + + # put in here the value of lsb_release -c (ex. oneirc for ubuntu 11.10 or squeeze for debian 6.0) + export DISTRIBUTION="squeeze" + echo "deb http://deb.torproject.org/torproject.org $DISTRIBUTION main" >> /etc/apt/sources.list + gpg --keyserver keys.gnupg.net --recv 886DDD89 + gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - + apt-get update + apt-get install tor + + ## Configurating a virtual environment You are highly recommended to install python packages from inside of a virtual

1 0

[ooni-probe/master] Test DNS tamper test and move to blocking test directory.
by art＠torproject.org 28 Nov '12

28 Nov '12

commit 663be731d907c52d7fe9e8b7936ae5aaa12e04be Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 17:30:31 2012 +0100 Test DNS tamper test and move to blocking test directory. --- nettests/blocking/dnstamper.py | 136 ++++++++++++++++++++++++++++++++++++++++ nettests/core/dnstamper.py | 127 ------------------------------------- 2 files changed, 136 insertions(+), 127 deletions(-) diff --git a/nettests/blocking/dnstamper.py b/nettests/blocking/dnstamper.py new file mode 100644 index 0000000..9e0f69b --- /dev/null +++ b/nettests/blocking/dnstamper.py @@ -0,0 +1,136 @@ +# -*- encoding: utf-8 -*- +# +# dnstamper +# ********* +# +# The test reports censorship if the cardinality of the intersection of +# the query result set from the control server and the query result set +# from the experimental server is zero, which is to say, if the two sets +# have no matching results whatsoever. +# +# NOTE: This test frequently results in false positives due to GeoIP-based +# load balancing on major global sites such as google, facebook, and +# youtube, etc. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + +import pdb + +from twisted.python import usage +from twisted.internet import defer + +from ooni.templates import dnst + +from ooni import nettest +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', '8.8.8.8:53', + 'The OONI backend that runs the DNS resolver'], + ['testresolvers', 'T', None, + 'File containing list of DNS resolvers to test against'], + ['testresolver', 't', None, + 'Specify a single test resolver to use for testing'] + ] + +class DNSTamperTest(dnst.DNSTest): + + name = "DNS tamper" + description = "DNS censorship detection test" + version = "0.4" + authors = "Arturo Filastò, Isis Lovecruft" + requirements = None + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + usageOptions = UsageOptions + requiredOptions = ['backend', 'file'] + + def setUp(self): + if (not self.localOptions['testresolvers'] and \ + not self.localOptions['testresolver']): + raise usage.UsageError("You did not specify a testresolver") + + elif self.localOptions['testresolvers']: + test_resolvers_file = self.localOptions['testresolvers'] + + elif self.localOptions['testresolver']: + self.test_resolvers = [self.localOptions['testresolver']] + + try: + with open(test_resolvers_file) as f: + self.test_resolvers = [x.strip() for x in f.readlines()] + self.report['test_resolvers'] = self.test_resolvers + fp.close() + + except IOError: + log.exception(e) + raise usage.UsageError("Invalid test resolvers file") + + except NameError: + log.debug("No test resolver file configured") + + dns_ip, dns_port = self.localOptions['backend'].split(':') + self.control_dns_server = (dns_ip, int(dns_port)) + + self.report['control_resolver'] = self.control_dns_server + + @defer.inlineCallbacks + def test_a_lookup(self): + """ + We perform an A lookup on the DNS test servers for the domains to be + tested and an A lookup on the known good DNS server. + + We then compare the results from test_resolvers and that from + control_resolver and see if the match up. + If they match up then no censorship is happening (tampering: false). + + If they do not we do a reverse lookup (PTR) on the test_resolvers and + the control resolver for every IP address we got back and check to see + if anyone of them matches the control ones. + + If they do then we take not of the fact that censorship is probably not + happening (tampering: reverse-match). + + If they do not match then censorship is probably going on (tampering: + true). + """ + log.msg("Doing the test lookups on %s" % self.input) + list_of_ds = [] + hostname = self.input + + self.report['tampering'] = {} + + control_answers = yield self.performALookup(hostname, self.control_dns_server) + + for test_resolver in self.test_resolvers: + log.msg("Going for %s" % test_resolver) + test_dns_server = (test_resolver, 53) + + experiment_answers = yield self.performALookup(hostname, test_dns_server) + log.debug("Got these answers %s" % experiment_answers) + + if not experiment_answers: + log.err("Got no response, perhaps the DNS resolver is down?") + self.report['tampering'][test_resolver] = 'no_answer' + continue + + log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) + if set(experiment_answers) & set(control_answers): + log.msg("Address has not tampered with on DNS server") + self.report['tampering'][test_resolver] = False + else: + log.msg("Trying to do reverse lookup") + + experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server) + control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server) + + if experiment_reverse == control_reverse: + log.msg("Further testing has eliminated false positives") + self.report['tampering'][test_resolver] = 'reverse_match' + else: + log.msg("Reverse DNS on the results returned by returned") + log.msg("which does not match the expected domainname") + self.report['tampering'][test_resolver] = True diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py deleted file mode 100644 index 967ae26..0000000 --- a/nettests/core/dnstamper.py +++ /dev/null @@ -1,127 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# dnstamper -# ********* -# -# The test reports censorship if the cardinality of the intersection of -# the query result set from the control server and the query result set -# from the experimental server is zero, which is to say, if the two sets -# have no matching results whatsoever. -# -# NOTE: This test frequently results in false positives due to GeoIP-based -# load balancing on major global sites such as google, facebook, and -# youtube, etc. -# -# :authors: Arturo Filastò, Isis Lovecruft -# :licence: see LICENSE - -import pdb - -from twisted.python import usage -from twisted.internet import defer - -from ooni.templates import dnst - -from ooni import nettest -from ooni.utils import log - -class UsageOptions(usage.Options): - optParameters = [['backend', 'b', '8.8.8.8:53', - 'The OONI backend that runs the DNS resolver'], - ['testresolvers', 'T', None, - 'File containing list of DNS resolvers to test against'], - ['testresolver', 't', None, - 'Specify a single test resolver to use for testing'] - ] - -class DNSTamperTest(dnst.DNSTest): - - name = "DNS tamper" - description = "DNS censorship detection test" - version = "0.3" - authors = "Arturo Filastò, Isis Lovecruft" - requirements = None - - inputFile = ['file', 'f', None, - 'Input file of list of hostnames to attempt to resolve'] - - usageOptions = UsageOptions - requiredOptions = ['backend', 'file'] - - def setUp(self): - if not self.localOptions['testresolvers']: - raise usage.UsageError("You did not specify a file of DNS servers to test!" - "See the '--testresolvers' option.") - - try: - fp = open(self.localOptions['testresolvers']) - except: - raise usage.UsageError("Invalid test resolvers file") - - self.test_resolvers = [x.strip() for x in fp.readlines()] - fp.close() - - dns_ip, dns_port = self.localOptions['backend'].split(':') - self.control_dns_server = (dns_ip, int(dns_port)) - - self.report['test_resolvers'] = self.test_resolvers - self.report['control_resolver'] = self.control_dns_server - - @defer.inlineCallbacks - def test_a_lookup(self): - """ - We perform an A lookup on the DNS test servers for the domains to be - tested and an A lookup on the known good DNS server. - - We then compare the results from test_resolvers and that from - control_resolver and see if the match up. - If they match up then no censorship is happening (tampering: false). - - If they do not we do a reverse lookup (PTR) on the test_resolvers and - the control resolver for every IP address we got back and check to see - if anyone of them matches the control ones. - - If they do then we take not of the fact that censorship is probably not - happening (tampering: reverse-match). - - If they do not match then censorship is probably going on (tampering: - true). - """ - log.msg("Doing the test lookups on %s" % self.input) - list_of_ds = [] - hostname = self.input - - self.report['tampering'] = {} - - control_answers = yield self.performALookup(hostname, self.control_dns_server) - - for test_resolver in self.test_resolvers: - log.msg("Going for %s" % test_resolver) - test_dns_server = (test_resolver, 53) - - experiment_answers = yield self.performALookup(hostname, test_dns_server) - log.debug("Got these answers %s" % experiment_answers) - - if not experiment_answers: - log.err("Got no response, perhaps the DNS resolver is down?") - self.report['tampering'][test_resolver] = 'no_answer' - continue - - log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) - if set(experiment_answers) & set(control_answers): - log.msg("Address has not tampered with on DNS server") - self.report['tampering'][test_resolver] = False - else: - log.msg("Trying to do reverse lookup") - - experiment_reverse = yield self.performPTRLookup(experiment_answers[0], test_dns_server) - control_reverse = yield self.performPTRLookup(control_answers[0], self.control_dns_server) - - if experiment_reverse == control_reverse: - log.msg("Further testing has eliminated false positives") - self.report['tampering'][test_resolver] = 'reverse_match' - else: - log.msg("Reverse DNS on the results returned by returned") - log.msg("which does not match the expected domainname") - self.report['tampering'][test_resolver] = True -

1 0

[ooni-probe/master] Further work
by art＠torproject.org 28 Nov '12

28 Nov '12

commit ada2b83ee02a5a2b2ed197344498a30ebabd85d8 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 28 17:43:01 2012 +0100 Further work --- nettests/blocking/dnstamper.py | 22 +++++++++++++++++----- 1 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nettests/blocking/dnstamper.py b/nettests/blocking/dnstamper.py index 9e0f69b..f663276 100644 --- a/nettests/blocking/dnstamper.py +++ b/nettests/blocking/dnstamper.py @@ -106,20 +106,29 @@ class DNSTamperTest(dnst.DNSTest): control_answers = yield self.performALookup(hostname, self.control_dns_server) for test_resolver in self.test_resolvers: - log.msg("Going for %s" % test_resolver) + log.msg("Testing %s test resolver" % test_resolver) test_dns_server = (test_resolver, 53) experiment_answers = yield self.performALookup(hostname, test_dns_server) - log.debug("Got these answers %s" % experiment_answers) + log.debug("Got the following A lookup answers %s for %s" % (experiment_answers, test_resolver)) if not experiment_answers: log.err("Got no response, perhaps the DNS resolver is down?") self.report['tampering'][test_resolver] = 'no_answer' continue + def lookup_details(): + """ + A closure useful for printing test details. + """ + log.msg("test resolver: %s" % test_resolver) + log.msg("experiment answers: %s" % experiment_answers) + log.msg("control answers: %s" % control_answers) + log.debug("Comparing %s with %s" % (experiment_answers, control_answers)) if set(experiment_answers) & set(control_answers): - log.msg("Address has not tampered with on DNS server") + lookup_details() + log.msg("tampering: false") self.report['tampering'][test_resolver] = False else: log.msg("Trying to do reverse lookup") @@ -129,8 +138,11 @@ class DNSTamperTest(dnst.DNSTest): if experiment_reverse == control_reverse: log.msg("Further testing has eliminated false positives") + lookup_details() + log.msg("tampering: reverse_match") self.report['tampering'][test_resolver] = 'reverse_match' else: - log.msg("Reverse DNS on the results returned by returned") - log.msg("which does not match the expected domainname") + log.msg("Reverse lookups do not match") + lookup_details() + log.msg("tampering: true") self.report['tampering'][test_resolver] = True

1 0