March 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'master' into feature/add_scapy_traceroute_protocol_rebased
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 0ca415061de29139dd16b88038f113800862c00a Merge: de64930 aa45aca Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 28 17:59:56 2014 +0100 Merge branch 'master' into feature/add_scapy_traceroute_protocol_rebased * master: (53 commits) Install pyrex-real so that new pip does not give error Update index of documentation page. Bump to v1.0.0-rc-7 Add WARNING message to every run of ooniprobe. Expand the disclaimer in the README.md Update vagrant link to point to https site. Bump to version 1.0.0 Add instructions on setting up development dependencies. Make the geoip related tests make a less stronger assertion. Make GeoIP AS and GeoIP City soft dependencies. Minor fixes to comments DNS-host-resolution-for-tls_handshake Bug-Fixes-for-tls_handshake Properly expand the home directory to that of the user running ooniprobe. Fix import error in test_managers. Properly handle the cancelled failure. Silence the dummy protocol Write test that reproduces HTTP test stalling condition. Call connectionLost to avoid dirty reactor. Pass the mock super socket in the ScapyFactory constructor ... .coveragerc | 2 + .travis.yml | 3 +- README.md | 23 +- Vagrantfile | 26 +- docs/source/index.rst | 2 +- ooni/__init__.py | 2 +- ooni/deck.py | 31 +- ooni/director.py | 4 +- ooni/errors.py | 14 +- ooni/geoip.py | 35 +- ooni/nettest.py | 32 +- ooni/nettests/experimental/tls_handshake.py | 36 +- ooni/oonibclient.py | 35 +- ooni/oonicli.py | 3 + ooni/reporter.py | 4 +- ooni/settings.py | 5 +- ooni/tasks.py | 12 +- ooni/templates/httpt.py | 6 - ooni/tests/__init__.py | 3 + ooni/tests/disabled_test_utils.py | 21 - ooni/tests/test_deck.py | 130 +++++ ooni/tests/test_director.py | 47 ++ ooni/tests/test_geoip.py | 22 + ooni/tests/test_managers.py | 4 +- ooni/tests/test_nettest.py | 72 ++- ooni/tests/test_oonibclient.py | 43 +- ooni/tests/test_reporter.py | 91 ++++ ooni/tests/test_templates.py | 43 ++ ooni/tests/test_txscapy.py | 47 ++ ooni/tests/test_utils.py | 22 + ooni/utils/__init__.py | 9 +- ooni/utils/onion.py | 704 --------------------------- requirements-dev.txt | 1 + 33 files changed, 694 insertions(+), 840 deletions(-)

1 0

[ooni-probe/master] Make the getAddresses function more robust
by art＠torproject.org 12 Mar '14

12 Mar '14

commit c8d63868c393a903fa80c08fee3efa55cbfa19f7 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 7 13:41:58 2014 +0100 Make the getAddresses function more robust --- ooni/nettests/manipulation/parasitictraceroute.py | 2 +- ooni/utils/txscapy.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/manipulation/parasitictraceroute.py b/ooni/nettests/manipulation/parasitictraceroute.py index 7820721..c8aa3ed 100644 --- a/ooni/nettests/manipulation/parasitictraceroute.py +++ b/ooni/nettests/manipulation/parasitictraceroute.py @@ -33,7 +33,7 @@ class ParasiticTracerouteTest(scapyt.BaseScapyTest): if k in self.pt.matched_packets: ttl = self.pt.matched_packets[k]['ttl'] else: - ttl = 'Unknown' + ttl = 'unknown' hop = (ttl, packet.src) path = 'hops_%s' % packet[IPerror].dst if path in self.report['parasitic_traceroute']: diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 16d8e7e..a96a397 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -108,8 +108,14 @@ class IfaceError(Exception): def getAddresses(): from scapy.all import get_if_addr, get_if_list from ipaddr import IPAddress - addresses = set([get_if_addr(i) for i in get_if_list()]) - addresses.remove('0.0.0.0') + addresses = set() + for i in get_if_list(): + try: + addresses.add(get_if_addr(i)) + except: + pass + if '0.0.0.0' in addresses: + addresses.remove('0.0.0.0') return [IPAddress(addr) for addr in addresses] def getDefaultIface():

1 0

[ooni-probe/master] Move the parasitic traceroute function to the experimental tests.
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 2e5647cf4a5de7ba83c3c12596661f9007c216b5 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 7 13:42:52 2014 +0100 Move the parasitic traceroute function to the experimental tests. --- ooni/nettests/experimental/parasitictraceroute.py | 157 +++++---------------- ooni/nettests/manipulation/parasitictraceroute.py | 48 ------- 2 files changed, 38 insertions(+), 167 deletions(-) diff --git a/ooni/nettests/experimental/parasitictraceroute.py b/ooni/nettests/experimental/parasitictraceroute.py index 631c24b..c8aa3ed 100644 --- a/ooni/nettests/experimental/parasitictraceroute.py +++ b/ooni/nettests/experimental/parasitictraceroute.py @@ -1,129 +1,48 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - from twisted.python import usage -from twisted.internet import defer - +from twisted.internet import defer, reactor from ooni.templates import scapyt - -from scapy.all import * - from ooni.utils import log +from ooni.utils.txscapy import ParasiticTraceroute +from ooni.settings import config -class UsageOptions(usage.Options): - optParameters = [['backend', 'b', 'google.com', 'Test backend to use'], - ['timeout', 't', 5, 'The timeout for the traceroute test'], - ['maxttl', 'm', 64, 'The maximum value of ttl to set on packets'], - ['dstport', 'd', 80, 'Set the destination port of the traceroute test'], - ['srcport', 'p', None, 'Set the source port to a specific value']] +from scapy.all import TCPerror, IPerror -class ParasiticalTracerouteTest(scapyt.BaseScapyTest): - name = "Parasitic TCP Traceroute Test" - author = "Arturo Filastò" - version = "0.1" +class ParasiticTracerouteTest(scapyt.BaseScapyTest): + name = "Parasitic Traceroute Test" + description = "Injects duplicate TCP packets with varying TTL values by sniffing traffic" + version = '0.1' - usageOptions = UsageOptions + samplePeriod = 40 def setUp(self): - def get_sport(): - if self.localOptions['srcport']: - return int(self.localOptions['srcport']) + self.report['parasitic_traceroute'] = {} + + def test_parasitic_traceroute(self): + self.pt = ParasiticTraceroute() + config.scapyFactory.registerProtocol(self.pt) + d = defer.Deferred() + reactor.callLater(self.samplePeriod, d.callback, self.pt) + return d + + def postProcessor(self, *args, **kwargs): + self.pt.stopListening() + self.report['received_packets'] = self.pt.received_packets + + for packet in self.pt.received_packets: + k = (packet[IPerror].id, packet[TCPerror].sport, packet[TCPerror].dport, packet[TCPerror].seq) + if k in self.pt.matched_packets: + ttl = self.pt.matched_packets[k]['ttl'] else: - return random.randint(1024, 65535) - self.get_sport = get_sport - - self.dst_ip = socket.gethostbyaddr(self.localOptions['backend'])[2][0] - - self.dport = int(self.localOptions['dstport']) - self.max_ttl = int(self.localOptions['maxttl']) - - @defer.inlineCallbacks - def test_parasitic_tcp_traceroute(self): - """ - Establishes a TCP stream, then sequentially sends TCP packets with - increasing TTL until we reach the ttl of the destination. - - Requires the backend to respond with an ACK to our SYN packet (i.e. - the port must be open) - - XXX this currently does not work properly. The problem lies in the fact - that we are currently using the scapy layer 3 socket. This socket makes - packets received be trapped by the kernel TCP stack, therefore when we - send out a SYN and get back a SYN-ACK the kernel stack will reply with - a RST because it did not send a SYN. - - The quick fix to this would be to establish a TCP stream using socket - calls and then "cannibalizing" the TCP session with scapy. - - The real fix is to make scapy use libpcap instead of raw sockets - obviously as we previously did... arg. - """ - sport = self.get_sport() - dport = self.dport - ipid = int(RandShort()) - - ip_layer = IP(dst=self.dst_ip, - id=ipid, ttl=self.max_ttl) - - syn = ip_layer/TCP(sport=sport, dport=dport, flags="S", seq=0) - - log.msg("Sending...") - syn.show2() - - synack = yield self.sr1(syn) - - log.msg("Got response...") - synack.show2() - - if not synack: - log.err("Got no response. Try increasing max_ttl") - return - - if synack[TCP].flags == 11: - log.msg("Got back a FIN ACK. The destination port is closed") - return - - elif synack[TCP].flags == 18: - log.msg("Got a SYN ACK. All is well.") - else: - log.err("Got an unexpected result") - return - - ack = ip_layer/TCP(sport=synack.dport, - dport=dport, flags="A", - seq=synack.ack, ack=synack.seq + 1) - - yield self.send(ack) - - self.report['hops'] = [] - # For the time being we make the assumption that we are NATted and - # that the NAT will forward the packet to the destination even if the TTL has - for ttl in range(1, self.max_ttl): - log.msg("Sending packet with ttl of %s" % ttl) - ip_layer.ttl = ttl - empty_tcp_packet = ip_layer/TCP(sport=synack.dport, - dport=dport, flags="A", - seq=synack.ack, ack=synack.seq + 1) - - answer = yield self.sr1(empty_tcp_packet) - if not answer: - log.err("Got no response for ttl %s" % ttl) - continue - - try: - icmp = answer[ICMP] - report = {'ttl': empty_tcp_packet.ttl, - 'address': answer.src, - 'rtt': answer.time - empty_tcp_packet.time - } - log.msg("%s: %s" % (dport, report)) - self.report['hops'].append(report) - - except IndexError: - if answer.src == self.dst_ip: - answer.show() - log.msg("Reached the destination. We have finished the traceroute") - return + ttl = 'unknown' + hop = (ttl, packet.src) + path = 'hops_%s' % packet[IPerror].dst + if path in self.report['parasitic_traceroute']: + self.report['parasitic_traceroute'][path].append(hop) + else: + self.report['parasitic_traceroute'][path] = [hop] + for p in self.report['parasitic_traceroute'].keys(): + self.report['parasitic_traceroute'][p].sort(key=lambda x: x[0]) + + self.report['sent_packets'] = self.pt.sent_packets + return self.report diff --git a/ooni/nettests/manipulation/parasitictraceroute.py b/ooni/nettests/manipulation/parasitictraceroute.py deleted file mode 100644 index c8aa3ed..0000000 --- a/ooni/nettests/manipulation/parasitictraceroute.py +++ /dev/null @@ -1,48 +0,0 @@ -from twisted.python import usage -from twisted.internet import defer, reactor -from ooni.templates import scapyt -from ooni.utils import log -from ooni.utils.txscapy import ParasiticTraceroute -from ooni.settings import config - -from scapy.all import TCPerror, IPerror - -class ParasiticTracerouteTest(scapyt.BaseScapyTest): - name = "Parasitic Traceroute Test" - description = "Injects duplicate TCP packets with varying TTL values by sniffing traffic" - version = '0.1' - - samplePeriod = 40 - - def setUp(self): - self.report['parasitic_traceroute'] = {} - - def test_parasitic_traceroute(self): - self.pt = ParasiticTraceroute() - config.scapyFactory.registerProtocol(self.pt) - d = defer.Deferred() - reactor.callLater(self.samplePeriod, d.callback, self.pt) - return d - - def postProcessor(self, *args, **kwargs): - self.pt.stopListening() - self.report['received_packets'] = self.pt.received_packets - - for packet in self.pt.received_packets: - k = (packet[IPerror].id, packet[TCPerror].sport, packet[TCPerror].dport, packet[TCPerror].seq) - if k in self.pt.matched_packets: - ttl = self.pt.matched_packets[k]['ttl'] - else: - ttl = 'unknown' - hop = (ttl, packet.src) - path = 'hops_%s' % packet[IPerror].dst - if path in self.report['parasitic_traceroute']: - self.report['parasitic_traceroute'][path].append(hop) - else: - self.report['parasitic_traceroute'][path] = [hop] - for p in self.report['parasitic_traceroute'].keys(): - self.report['parasitic_traceroute'][p].sort(key=lambda x: x[0]) - - self.report['sent_packets'] = self.pt.sent_packets - return self.report -

1 0

[ooni-probe/master] Initialize all report entries in http requests test
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 59a9c96c81d5ac0e76808005fc4632a662be0666 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 7 17:03:24 2014 +0100 Initialize all report entries in http requests test --- ooni/nettests/blocking/http_requests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 790b554..17859f5 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -30,7 +30,7 @@ class HTTPRequestsTest(httpt.HTTPTest): name = "HTTP Requests Test" description = "Performs a HTTP GET request over Tor and one over the local network and compares the two results." author = "Arturo Filastò" - version = "0.2.3" + version = "0.2.4" usageOptions = UsageOptions @@ -56,6 +56,12 @@ class HTTPRequestsTest(httpt.HTTPTest): self.factor = self.localOptions['factor'] self.report['control_failure'] = None self.report['experiment_failure'] = None + self.report['body_length_match'] = None + self.report['body_proportion'] = None + self.report['factor'] = float(self.factor) + self.report['headers_diff'] = None + self.report['headers_match'] = None + self.headers = {'User-Agent': [random.choice(userAgents)]} def compare_body_lengths(self, body_length_a, body_length_b): @@ -71,7 +77,6 @@ class HTTPRequestsTest(httpt.HTTPTest): rel = 1/rel self.report['body_proportion'] = rel - self.report['factor'] = float(self.factor) if rel > float(self.factor): log.msg("The two body lengths appear to match") log.msg("censorship is probably not happening")

1 0

[ooni-probe/master] Merge pull request #290 from TheTorProject/feature/add_scapy_traceroute_protocol_rebased
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 4e529ec9a64ae6c26a7bedf99360c126a75ab1ce Merge: 3ccb5f9 2e5647c Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Fri Mar 7 16:55:18 2014 +0100 Merge pull request #290 from TheTorProject/feature/add_scapy_traceroute_protocol_rebased Feature/add scapy traceroute protocol rebased ooni/director.py | 14 +- ooni/errors.py | 4 + ooni/nettests/experimental/parasitictraceroute.py | 157 ++++--------- ooni/nettests/manipulation/traceroute.py | 165 +++++--------- ooni/reporter.py | 2 +- ooni/templates/scapyt.py | 2 +- ooni/utils/txscapy.py | 242 ++++++++++++++++++++- 7 files changed, 355 insertions(+), 231 deletions(-)

1 0

[ooni-probe/master] Remove the mlab_no_root deck and strip down the mlab deck to include only one test.
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 8f129d8ee6cf8ec7904c0aabd4bc6997b5eee7cd Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 12 22:48:49 2014 +0100 Remove the mlab_no_root deck and strip down the mlab deck to include only one test. --- data/decks/mlab.deck | 48 ------------------------------------------ data/decks/mlab_no_root.deck | 48 ------------------------------------------ 2 files changed, 96 deletions(-) diff --git a/data/decks/mlab.deck b/data/decks/mlab.deck index 9564d1b..59dc458 100644 --- a/data/decks/mlab.deck +++ b/data/decks/mlab.deck @@ -8,53 +8,5 @@ reportfile: null resume: 0 subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] test_file: manipulation/http_header_field_manipulation testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/traceroute - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: manipulation/http_host - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dnsconsistency - testdeck: null diff --git a/data/decks/mlab_no_root.deck b/data/decks/mlab_no_root.deck deleted file mode 100644 index 0ab459e..0000000 --- a/data/decks/mlab_no_root.deck +++ /dev/null @@ -1,48 +0,0 @@ -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_invalid_request_line - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [] - test_file: manipulation/http_header_field_manipulation - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: manipulation/http_host - testdeck: null -- options: - collector: null - help: 0 - logfile: null - no-default-reporter: 0 - parallelism: null - pcapfile: null - reportfile: null - resume: 0 - subargs: [-f, 'httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] - test_file: blocking/dnsconsistency - testdeck: null

1 0

[ooni-probe/master] Return the correct deferred in director start method.
by art＠torproject.org 12 Mar '14

12 Mar '14

commit 064f3e5e635d3ced2395702e66e5d07b222c177c Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 7 17:07:16 2014 +0100 Return the correct deferred in director start method. --- ooni/oonicli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index f75b31c..935d091 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -248,7 +248,7 @@ def runWithDirector(logging=True, start_tor=True): raise e netTestDone = director.startNetTest(net_test_loader, reporters) - return netTestDone + return director.allTestsDone def start(): d.addCallback(setup_nettest)

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 12 Mar '14

12 Mar '14

commit f4162e472de1bf533156c6a4b57c8b5999780149 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Mar 12 21:45:45 2014 +0000 Update translations for tails-misc --- mk.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk.po b/mk.po index 2fdd414..ef76ac4 100644 --- a/mk.po +++ b/mk.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2014-03-06 14:52+0100\n" -"PO-Revision-Date: 2014-03-07 08:49+0000\n" +"PO-Revision-Date: 2014-03-12 21:30+0000\n" "Last-Translator: runasand <runa.sandvik(a)gmail.com>\n" "Language-Team: Macedonian (http://www.transifex.com/projects/p/torproject/language/mk/)\n" "MIME-Version: 1.0\n"

1 0

[translation/tor-launcher-network-settings_completed] Update translations for tor-launcher-network-settings_completed
by translation＠torproject.org 12 Mar '14

12 Mar '14

commit 2aa05d67af263268334e7ce68101adc7f6839002 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Mar 12 21:45:43 2014 +0000 Update translations for tor-launcher-network-settings_completed --- es/network-settings.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/es/network-settings.dtd b/es/network-settings.dtd index 2a9c1e4..7b1a881 100644 --- a/es/network-settings.dtd +++ b/es/network-settings.dtd @@ -1,8 +1,8 @@ -<!ENTITY torsettings.dialog.title "Preferencias de red del Navegador Tor"> +<!ENTITY torsettings.dialog.title "Configuración de red de Tor">  -<!ENTITY torsettings.prompt "Antes de que el Navegador Tor intente conectar a la red Tor, necesita proporcionar información acerca de la conexión a Internet de esta computadora."> +<!ENTITY torsettings.prompt "Antes de que se conecte a la red Tor, necesita proporcionar información sobre la conexión a Internet de este equipo."> <!ENTITY torSettings.yes "Sí"> <!ENTITY torSettings.no "No">

1 0

[translation/tor-launcher-network-settings] Update translations for tor-launcher-network-settings
by translation＠torproject.org 12 Mar '14

12 Mar '14

commit 729c32bf6ec639cbac5a9cdb98ad95e82f7e2f0f Author: Translation commit bot <translation(a)torproject.org> Date: Wed Mar 12 21:45:40 2014 +0000 Update translations for tor-launcher-network-settings --- es/network-settings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es/network-settings.dtd b/es/network-settings.dtd index f6ca345..7b1a881 100644 --- a/es/network-settings.dtd +++ b/es/network-settings.dtd @@ -2,7 +2,7 @@  -<!ENTITY torsettings.prompt "Before you connect to the Tor network, you need to provide information about this computer's Internet connection."> +<!ENTITY torsettings.prompt "Antes de que se conecte a la red Tor, necesita proporcionar información sobre la conexión a Internet de este equipo."> <!ENTITY torSettings.yes "Sí"> <!ENTITY torSettings.no "No">

1 0