April 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] changed parsing logic to use new Asset.parse_line method
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 022bf7374c0d16704754f441c442169b1ac7ff4a Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Mar 16 20:08:37 2012 -0700 changed parsing logic to use new Asset.parse_line method --- assets/top-1m.txt | 21 +++++++++++---------- tests/dnstamper.py | 15 ++++++--------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/assets/top-1m.txt b/assets/top-1m.txt index 9403460..293e661 100644 --- a/assets/top-1m.txt +++ b/assets/top-1m.txt @@ -1,10 +1,11 @@ -google.com -facebook.com -youtube.com -yahoo.com -baidu.com -wikipedia.org -live.com -blogspot.com -twitter.com -qq.com +1,torproject.org +2,google.com +3,facebook.com +4,youtube.com +5,yahoo.com +6,baidu.com +7,wikipedia.org +8,live.com +9,blogspot.com +10,twitter.com +11,qq.com diff --git a/tests/dnstamper.py b/tests/dnstamper.py index 1e59d26..4064831 100644 --- a/tests/dnstamper.py +++ b/tests/dnstamper.py @@ -7,7 +7,6 @@ import os import plugoo from plugoo.assets import Asset from plugoo.tests import Test -from __future__ import with_statement __plugoo__ = "DNST" __desc__ = "DNS censorship detection test" @@ -16,11 +15,9 @@ class Top1MAsset(Asset): def __init__(self, file=None): self = Asset.__init__(self, file) - def next_asset(self): - with self.fh as asset: - lines = asset.readlines() - for line in lines: - yield line.split(',')[1] + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.split(',')[1].replace('\n','') class DNSTAsset(Asset): def __init__(self, file=None): @@ -53,10 +50,10 @@ class DNST(Test): control = self.lookup(address, config.tests.dns_control_server) if len(set(exp) & set(control)) > 0: - print "%s : no tampering on %s" % (address, ns) + print "Address %s has not tampered with on DNS server %s\n" % (address, ns) return (address, ns, False) else: - print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control) + print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s\n" % (address, ns, ns, exp, control) return (address, ns, exp, control, True) def run(ooni): @@ -65,7 +62,7 @@ def run(ooni): config = ooni.config urls = [] - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \ + dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, \ config.tests.dns_experiment)) dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ config.tests.dns_experiment_dns))

1 0

[ooni-probe/master] Fixed invalid call on NoneType object in plugoo/tests.py, added exception catcher for import statement on BeautifulSoup module in tests/httphost.py
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 53e2da3a296ecbac3bbdd6d80c525d1fb21af4c2 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Mar 23 15:23:40 2012 -0700 Fixed invalid call on NoneType object in plugoo/tests.py, added exception catcher for import statement on BeautifulSoup module in tests/httphost.py --- plugoo/tests.py | 5 +++-- tests/httphost.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/plugoo/tests.py b/plugoo/tests.py index 3d6b36b..f7537de 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -78,8 +78,9 @@ class Test: if assets: self.logger.debug("Running through tests") - if extradata['index']: - index = extradata['index'] + if extradata is not None: + if extradata['index']: + index = extradata['index'] else: index = None diff --git a/tests/httphost.py b/tests/httphost.py index 6446e1f..0f33ca7 100644 --- a/tests/httphost.py +++ b/tests/httphost.py @@ -20,7 +20,10 @@ import httplib # appearing on the wire. monkey.patch_socket() -from BeautifulSoup import BeautifulSoup +try: + from BeautifulSoup import BeautifulSoup +except: + print "BeautifulSoup-3.2.1 is missing. Try doing: # easy_install beautifulsoup" from plugoo.assets import Asset from plugoo.tests import Test

1 0

[ooni-probe/master] Completed attempt to use reverse DNS to account for geoIP load balancing, added option in ooni-conf to disable this feature.
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 1f5c521ebe1bededb16d70082ae4f4617df0785e Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Mar 24 01:07:41 2012 -0700 Completed attempt to use reverse DNS to account for geoIP load balancing, added option in ooni-conf to disable this feature. --- HACKING | 2 +- ooni-probe.conf | 7 +++- plugoo/reports.py | 2 +- tests/dnstamper.py | 110 ++++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/HACKING b/HACKING index 57b6c9b..4451882 100644 --- a/HACKING +++ b/HACKING @@ -170,7 +170,7 @@ Method definitions inside of class are separated by a single blank line. Encoding ........ -Always use UTF-8 encoding. This can be specified by add the encoding cookie +Always use UTF-8 encoding. This can be specified by adding the encoding cookie to the beginning of your python files: # -*- coding: UTF-8 -*- diff --git a/ooni-probe.conf b/ooni-probe.conf index 8702353..161a008 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -31,7 +31,12 @@ dns_experiment = top-1m.txt dns_experiment_dns = dns_servers.txt # This is the control known good DNS server -dns_control_server = 8.8.8.8 +dns_control_server = 91.191.136.152 + +# Specify whether the dnstamper test should attempt to remove +# GeoIP-based false positives by doing a reverse DNS resolve +# on positive results. +dns_reverse_lookup = true ### traceroute testing related config parameters diff --git a/plugoo/reports.py b/plugoo/reports.py index 90de498..009e86b 100644 --- a/plugoo/reports.py +++ b/plugoo/reports.py @@ -42,7 +42,7 @@ class Report: import paramiko except: self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") + self.logger.warn("Could not import paramiko. SCP will be disabled") def __call__(self, data): """ diff --git a/tests/dnstamper.py b/tests/dnstamper.py index 4064831..4fc3c71 100644 --- a/tests/dnstamper.py +++ b/tests/dnstamper.py @@ -1,9 +1,38 @@ +# -*- coding: utf-8 -*- +""" + dnstamper + ********* + + This test resolves DNS for a list of domain names, one per line, in the + file specified in the ooni-config under the setting "dns_experiment". If + the file is top-1m.txt, the test will be run using Amazon's list of top + one million domains. The experimental dns servers to query should + be specified one per line in assets/dns_servers.txt. + + The test reports censorship if the cardinality of the intersection of + the query result set from the control server and the query result set + from the experimental server is zero, which is to say, if the two sets + have no matching results whatsoever. + + NOTE: This test frequently results in false positives due to GeoIP-based + load balancing on major global sites such as google, facebook, and + youtube, etc. + + :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details +""" + +try: + from dns import resolver, reversename +except: + print "Error: dnspython is not installed! (http://www.dnspython.org/)" try: - from dns import resolver + import gevent except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" -import gevent + print "Error: gevent is not installed! (http://www.gevent.org/)" + import os + import plugoo from plugoo.assets import Asset from plugoo.tests import Test @@ -12,6 +41,9 @@ __plugoo__ = "DNST" __desc__ = "DNS censorship detection test" class Top1MAsset(Asset): + """ + Class for parsing top-1m.txt as an asset. + """ def __init__(self, file=None): self = Asset.__init__(self, file) @@ -20,11 +52,18 @@ class Top1MAsset(Asset): return line.split(',')[1].replace('\n','') class DNSTAsset(Asset): + """ + Creates DNS testing specific Assets. + """ def __init__(self, file=None): self = Asset.__init__(self, file) class DNST(Test): def lookup(self, hostname, ns): + """ + Resolves a hostname through a DNS nameserver, ns, to the corresponding + IP address(es). + """ res = resolver.Resolver(configure=False) res.nameservers = [ns] answer = res.query(hostname) @@ -36,42 +75,81 @@ class DNST(Test): return ret + def reverse_lookup(self, ip, ns): + """ + Attempt to do a reverse DNS lookup to determine if the control and exp + sets from a positive result resolve to the same domain, in order to + remove false positives due to GeoIP load balancing. + """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + n = reversename.from_address(ip) + revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] + + return revn + def experiment(self, *a, **kw): + """ + Compares the lookup() sets of the control and experiment groups. + """ # this is just a dirty hack address = kw['data'][0] ns = kw['data'][1] config = self.config + ctrl_ns = config.tests.dns_control_server print "ADDRESS: %s" % address print "NAMESERVER: %s" % ns exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) + control = self.lookup(address, ctrl_ns) + + result = [] if len(set(exp) & set(control)) > 0: print "Address %s has not tampered with on DNS server %s\n" % (address, ns) - return (address, ns, False) + result = (address, ns, exp, control, False) + return result else: - print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s\n" % (address, ns, ns, exp, control) - return (address, ns, exp, control, True) + print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) + result = (address, ns, exp, control, True) + + if config.tests.dns_reverse_lookup: + + exprevn = [self.reverse_lookup(ip, ns) for ip in exp] + ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) + for ip in control] + + if len(set(exprevn) & set(ctrlrevn)) > 0: + print "Further testing has eliminated this as a false positive." + else: + print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) + return result + + else: + print "\n" + return result def run(ooni): - """Run the test + """ + Run the test. """ config = ooni.config urls = [] - dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ + if (config.tests.dns_experiment == "top-1m.txt"): + dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment)) + else: + dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment)) + dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, config.tests.dns_experiment_dns)) assets = [dns_experiment, dns_experiment_dns] dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - - + ooni.logger.info("Beginning dnstamper test...") + dnstest.run(assets, {'index': 1}) + ooni.logger.info("Dnstamper test completed!")

1 0

[ooni-probe/master] Removed nested if statements and pointed to crate.io for import exception
by art＠torproject.org 04 Apr '12

04 Apr '12

commit cd81a1d9f7d237d95b1b6c07414b493ee9f99cdd Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Mar 23 20:16:44 2012 -0700 Removed nested if statements and pointed to crate.io for import exception --- plugoo/tests.py | 5 ++--- tests/httphost.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugoo/tests.py b/plugoo/tests.py index f7537de..f94ba04 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -78,9 +78,8 @@ class Test: if assets: self.logger.debug("Running through tests") - if extradata is not None: - if extradata['index']: - index = extradata['index'] + if extradata and extradata['index']: + index = extradata['index'] else: index = None diff --git a/tests/httphost.py b/tests/httphost.py index 0f33ca7..25adff2 100644 --- a/tests/httphost.py +++ b/tests/httphost.py @@ -23,7 +23,7 @@ monkey.patch_socket() try: from BeautifulSoup import BeautifulSoup except: - print "BeautifulSoup-3.2.1 is missing. Try doing: # easy_install beautifulsoup" + print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" from plugoo.assets import Asset from plugoo.tests import Test

1 0

[ooni-probe/master] Updated TODO file
by art＠torproject.org 04 Apr '12

04 Apr '12

commit ea742983223b2033d39cfb692edd71698bbb4102 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Mar 24 01:31:40 2012 -0700 Updated TODO file --- TODO | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/TODO b/TODO index 9c2cd0a..9ecae4b 100644 --- a/TODO +++ b/TODO @@ -21,6 +21,11 @@ effort: low, skill: low * All the DNS censorship detection tests (old/ooni/dnsooni.py) effort: medium, skill: medium + - The dnstamper test fulfills this requirement, although + there are several improvements which could be made, namely + figuring out if there is a pattern to the reverse resolved + names and parsing them in some manner to determine if that + server is authorized to serve the requested page. * Misc scripts to detect censorship that are not written in any version of OONI (old/ooni/plugins/old_stuff_to_convert/)

1 0

[ooni-probe/master] rm ctrl from exp group
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 1a948c71d5c77c3c8938115f13a4383a36b809e1 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Tue Mar 13 05:13:19 2012 -0700 rm ctrl from exp group --- assets/dns_servers.txt | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/assets/dns_servers.txt b/assets/dns_servers.txt index 0453fe1..844e8d5 100644 --- a/assets/dns_servers.txt +++ b/assets/dns_servers.txt @@ -1,5 +1,3 @@ -8.8.8.8 -8.8.4.4 209.244.0.3 209.244.0.4 208.67.222.222

1 0

[ooni-probe/master] added ssh command and sftp methods
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 127f9410c4df41d14bb795386e72e6630de18683 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Wed Mar 7 21:01:26 2012 -0800 added ssh command and sftp methods --- plugoo/nodes.py | 78 +++++++++++++++++++++++++++++++++++++++--------------- 1 files changed, 56 insertions(+), 22 deletions(-) diff --git a/plugoo/nodes.py b/plugoo/nodes.py index 5b16d22..ccdaa44 100644 --- a/plugoo/nodes.py +++ b/plugoo/nodes.py @@ -12,6 +12,7 @@ """ +import cmd import os from binascii import hexlify @@ -93,6 +94,7 @@ class CodeExecNode(Node): class PlanetLab(Node, CodeExecNode): def __init__(self, address, auth_creds, ooni): + self.node = Node(address, 22) self.auth_creds = auth_creds self.config = ooni.config @@ -130,8 +132,10 @@ class PlanetLab(Node, CodeExecNode): print 'Adding nodes %s' % node['hostname'] def _auth_login(slicename, machinename): - """Attempt to authenticate to the given PL node, slicename and - machinename, using any of the private keys in ~/.ssh/ """ + """ + Attempt to authenticate to the given PL node, slicename and + machinename, using any of the private keys in ~/.ssh/ + """ agent = paramiko.Agent() agent_keys = agent.get_keys() @@ -147,27 +151,57 @@ class PlanetLab(Node, CodeExecNode): except paramiko.SSHException: print 'Public key authentication to PlanetLab node %s failed.' % machinename, - def _get_command: - pass - - def ssh_and_run_(slicename, machinename, command): - """Attempt to make a standard OpenSSH client to PL node, and run - commands from a .conf file.""" - - ## needs a way to specify 'ssh -l <slicename> <machinename>' - ## with public key authentication. - - command = PlanetLab.get_command() - + def _ssh_to_node(slicename, machinename, sshkeypasswd): + """ + Attempt to make an SSH2 connection to added PL nodes. + """ + + self.hosts = [] + self.connections = [] + + if slicename, machinename, sshkeypasswd: + Node(machinename, 22) + for node in self.node: + self.hosts.append([slicename, machinename, sshkeypasswd]) + + for host in self.hosts: + client = paramiko.SSHClient() + if (self.config.main.auto_add_new_hosts == 1): + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.load_system_host_keys() + client.connect(host[1], username=host[0], password=host[2]) + self.connections.append(client) + + return (self.hosts, self.connections) + + def _ssh_and_run_(slicename, machinename, sshkeypasswd, command): + """ + Attempt to make an SSH2 connection to PL nodes, and run + commands. + """ + + PlanetLab._ssh_to_node(slicename, machinename, sshkeypasswd) + + if command: + for host, conn in zip(self.hosts, self.connections): + stdin, stdout, stderr = conn.exec_command(command) + stdin.close() + for line in stdout.read().splitlines(): + print 'host: %s: %s' % (host[0], line) + + for conn in self.connections: + conn.close() + + def send_files_to_node(slicename, machinename, sshkeypasswd, + remotefile, localfile): + """Attempt to stfp files to the PL node.""" + + PlanetLab._ssh_to_node(slicename, machinename, sshkeypasswd) + client = paramiko.SSHClient() - client.load_system_host_keys() - client.connect(machinename) - - stdin, stdout, stderr = client.exec_command(command) - - def send_files_to_node(directory, files): - """Attempt to rsync a tree to the PL node.""" - pass + sftp = client.open_sftp() + sftp.get(localfile, remotefile) + sftp.close() def add_unit: pass

1 0

[ooni-probe/master] Fixing documentation in HACKING file and adding TAGS to .gitignore
by art＠torproject.org 04 Apr '12

04 Apr '12

commit d6f72bd1a8f0f6e055fcff91bebcc582230e90d8 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Mar 12 18:14:51 2012 -0700 Fixing documentation in HACKING file and adding TAGS to .gitignore --- .gitignore | 3 ++- HACKING | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 6433d66..1ec236d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ proxy-lists/ip-cc.txt proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt -private/* \ No newline at end of file +private/* +TAGS \ No newline at end of file diff --git a/HACKING b/HACKING index ed8d5dc..57b6c9b 100644 --- a/HACKING +++ b/HACKING @@ -93,7 +93,7 @@ Code Structure to write your own test based on the OONI paradigm. - tests/ - Contains all the "offical" OONI tests that are shipped. + Contains all the "official" OONI tests that are shipped. - utils.py Helper functions that don't fit into any place, but are not big enough to @@ -118,7 +118,7 @@ Indentation set expandtab - Continuation lines should be wrapper like this: + Continuation lines should be wrapped like this: foo = long_function_name(var_one, var_two, var_three, var_four) @@ -131,12 +131,12 @@ Indentation print(var_one) - They should NOT be wrapper like this: + They should NOT be wrapped like this: foo = long_function_name(var_one, var_two, var_three, var_four) - and NOT like this: + NOR like this: # See how it creates confusion with what is inside the function? def long_function_name(var_one, @@ -173,7 +173,7 @@ Encoding Always use UTF-8 encoding. This can be specified by add the encoding cookie to the beginning of your python files: - # -*- coding: UTF-8 + # -*- coding: UTF-8 -*- All identifiers should be ASCII-only. All doc strings and comments should also only be in ASCII. Non ASCII characters are allowed when they are related to @@ -205,11 +205,11 @@ Comments ........ Comments should always be up to date with the code. Don't have -comments that contraddict with the code. +comments that contradict with the code. Comments should always be written in English. -Blocks comments are indented to the same level of the code that +Block comments are indented to the same level of the code that they refer to. They start with # and are followed by a single space. Use inline comments sparingly. # Gotcha?

1 0

[ooni-probe/master] Testing dnstamper and adding assets for common DNS servers
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 5ab23640550789f56e8ad14d3c26b863e1e07229 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Mar 12 18:21:39 2012 -0700 Testing dnstamper and adding assets for common DNS servers --- assets/dns_servers.txt | 8 ++++++++ assets/top-1m.txt | 10 ++++++++++ tests/dnstamper.py | 2 +- 3 files changed, 19 insertions(+), 1 deletions(-) diff --git a/assets/dns_servers.txt b/assets/dns_servers.txt new file mode 100644 index 0000000..0453fe1 --- /dev/null +++ b/assets/dns_servers.txt @@ -0,0 +1,8 @@ +8.8.8.8 +8.8.4.4 +209.244.0.3 +209.244.0.4 +208.67.222.222 +208.67.220.220 +156.154.70.1 +156.154.71.1 diff --git a/assets/top-1m.txt b/assets/top-1m.txt new file mode 100644 index 0000000..9403460 --- /dev/null +++ b/assets/top-1m.txt @@ -0,0 +1,10 @@ +google.com +facebook.com +youtube.com +yahoo.com +baidu.com +wikipedia.org +live.com +blogspot.com +twitter.com +qq.com diff --git a/tests/dnstamper.py b/tests/dnstamper.py index 68be12d..23318b6 100644 --- a/tests/dnstamper.py +++ b/tests/dnstamper.py @@ -14,7 +14,7 @@ __desc__ = "DNS censorship detection test" class DNSTAsset(Asset): def __init__(self, file=None): - self = asset.__init__(self, file) + self = Asset.__init__(self, file) class DNST(Test): def lookup(self, hostname, ns):

1 0

[atlas/master] Update Onionoo URL.
by art＠torproject.org 04 Apr '12

04 Apr '12

commit 9e8c1ec430cd9061834adcc19bd4fc8f31de5725 Author: Arturo Filastò <hellais(a)gmail.com> Date: Wed Apr 4 09:32:42 2012 -0700 Update Onionoo URL. --- templates/about.html | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/templates/about.html b/templates/about.html index 876b49b..7297a33 100644 --- a/templates/about.html +++ b/templates/about.html @@ -43,7 +43,7 @@ This means that the amount of requests made to the server can be minimized and t this application can potentially run by being loaded locally. The server is only interrogated for JSON objects that do not manipulate the DOM of the page. The protocol used for retrieving relay data in Onionoo you can check it out -<a href="http://kloesing.github.com/Onionoo">here</a>. +<a href="https://onionoo.torproject.org/">here</a>. </p> </div>

1 0