September 2014 - tor-commits - lists.torproject.org

[tor/master] + is not how we say concatenate
by arma＠torproject.org 23 Sep '14

23 Sep '14

commit 1987157d0c1e9acb0b88156e7104fbc8a11c5932 Author: Roger Dingledine <arma(a)torproject.org> Date: Mon Sep 22 20:09:03 2014 -0400 + is not how we say concatenate --- src/or/rendcommon.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index aca9da1..269cd65 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -155,7 +155,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, } /* Calculate current time-period. */ time_period = get_time_period(now, 0, service_id_binary); - /* Calculate secret-id-part = h(time-period + replica). */ + /* Calculate secret-id-part = h(time-period | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, replica); /* Calculate descriptor ID. */ @@ -556,7 +556,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, char desc_digest[DIGEST_LEN]; rend_encoded_v2_service_descriptor_t *enc = tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); - /* Calculate secret-id-part = h(time-period + cookie + replica). */ + /* Calculate secret-id-part = h(time-period | cookie | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, k); base32_encode(secret_id_part_base32, sizeof(secret_id_part_base32),

1 0

[ooni-probe/master] Close file descriptor as soon as we have created the temporary file.
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 9716b2196d7ef94d56ce7d79e894d32b473def41 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 18 12:54:00 2014 +0200 Close file descriptor as soon as we have created the temporary file. --- ooni/nettests/blocking/bridge_reachability.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index d4f6e13..f472f75 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -51,7 +51,8 @@ class BridgeReachability(nettest.NetTestCase): self.tor_progress = 0 self.timeout = int(self.localOptions['timeout']) - _, self.logfile = tempfile.mkstemp() + fd, self.logfile = tempfile.mkstemp() + os.close(fd) self.report['error'] = None self.report['success'] = None

1 0

[ooni-probe/master] Set the measurement timeout to the timeout of the nettest + 8 to avoid triggering the measurement timeout before the nettest one.
by art＠torproject.org 22 Sep '14

22 Sep '14

commit ba2a9f6c9d3276994d3a8f315541ae49297dd355 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Sep 13 21:00:27 2014 -0400 Set the measurement timeout to the timeout of the nettest + 8 to avoid triggering the measurement timeout before the nettest one. --- ooni/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index 650e0b0..9d281ab 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -118,7 +118,11 @@ class Measurement(TaskWithTimeout): if 'timeout' in dir(test_instance): if isinstance(test_instance.timeout, int) or isinstance(test_instance.timeout, float): - self.timeout = test_instance.timeout + # If the test has a timeout option set we set the measurement + # timeout to that value + 8 seconds to give it enough time to + # trigger it's internal timeout before we start trigger the + # measurement timeout. + self.timeout = test_instance.timeout + 8 elif config.advanced.measurement_timeout: self.timeout = config.advanced.measurement_timeout TaskWithTimeout.__init__(self)

1 0

[ooni-probe/master] Add support for detecting bridge lines in tcp_connect test.
by art＠torproject.org 22 Sep '14

22 Sep '14

commit f9eba6162df21c5c2e0c5446105a772091bcd9c1 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 22 19:11:45 2014 +0200 Add support for detecting bridge lines in tcp_connect test. --- ooni/nettests/blocking/tcp_connect.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/tcp_connect.py b/ooni/nettests/blocking/tcp_connect.py index 852be68..1666ce3 100644 --- a/ooni/nettests/blocking/tcp_connect.py +++ b/ooni/nettests/blocking/tcp_connect.py @@ -67,13 +67,24 @@ class TCPConnectTest(nettest.NetTestCase): if proto == 'https': return "%s:443" % host + pluggable_transports = ("obfs3", "obfs2", "fte", "scramblesuit") + def is_bridge_line(line): + first = line.split(" ")[0] + return first.lower() in pluggable_transports + ("bridge",) + def strip_bridge(line): + if line.lower().startswith("Bridge"): + return line.split(" ")[2] + return line.split(" ")[1] + if filename: fp = open(filename) for x in fp.readlines(): if x.startswith("http"): yield strip_url(x) + elif is_bridge_line(x): + yield strip_bridge(x) else: - yield x.strip() + yield x.split(" ")[0] fp.close() else: pass

1 0

[ooni-probe/master] Enable info level logging on obfsproxy
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 436a2323a92a79f198e77f384329b760ac1fca50 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 22 22:30:10 2014 +0200 Enable info level logging on obfsproxy --- ooni/nettests/blocking/bridge_reachability.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index f472f75..b9aec33 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -51,7 +51,9 @@ class BridgeReachability(nettest.NetTestCase): self.tor_progress = 0 self.timeout = int(self.localOptions['timeout']) - fd, self.logfile = tempfile.mkstemp() + fd, self.tor_logfile = tempfile.mkstemp() + os.close(fd) + fd, self.obfsproxy_logfile = tempfile.mkstemp() os.close(fd) self.report['error'] = None @@ -64,6 +66,7 @@ class BridgeReachability(nettest.NetTestCase): self.report['tor_progress_summary'] = None self.report['tor_log'] = None self.report['obfsproxy_version'] = str(onion.obfsproxy_details['version']) + self.report['obfsproxy_log'] = None self.report['bridge_address'] = None self.bridge = self.input @@ -144,8 +147,10 @@ class BridgeReachability(nettest.NetTestCase): self.report['error'] = 'missing-fteproxy' return elif transport_name and self.pyobfsproxy_bin: - config.ClientTransportPlugin = "%s exec %s managed" % ( - transport_name, self.pyobfsproxy_bin) + config.ClientTransportPlugin = ("%s exec %s --log-min-severity info " + "--log-file %s managed") % (transport_name, + self.pyobfsproxy_bin, + self.obfsproxy_logfile) if onion.OBFSProxyVersion('0.2') > onion.obfsproxy_details['version']: log.err( "The obfsproxy version you are using appears to be outdated." @@ -175,7 +180,7 @@ class BridgeReachability(nettest.NetTestCase): config.Bridge = self.bridge config.UseBridges = 1 - config.log = ['notice stdout', 'notice file %s' % self.logfile] + config.log = ['notice stdout', 'notice file %s' % self.tor_logfile] config.save() def updates(prog, tag, summary): @@ -205,8 +210,11 @@ class BridgeReachability(nettest.NetTestCase): @d.addCallback def write_log(_): - with open(self.logfile) as f: + with open(self.tor_logfile) as f: self.report['tor_log'] = f.read() - os.remove(self.logfile) + os.remove(self.tor_logfile) + with open(self.obfsproxy_logfile) as f: + self.report['obfsproxy_log'] = f.read() + os.remove(self.obfsproxy_logfile) return d

1 0

[ooni-probe/master] Update README.rst
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 6413bc031f52fdcb649a067dc345155e002b012f Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 18 14:00:15 2014 +0200 Update README.rst --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 95f21a2..7a0e749 100644 --- a/README.rst +++ b/README.rst @@ -60,6 +60,10 @@ On unix systems:: sudo pip install ooniprobe +To install it from the current master run:: + + sudo pip install https://github.com/TheTorProject/ooni-probe/archive/master.zip + The run:: mkdir my_decks

1 0

[ooni-probe/master] Merge pull request #350 from TheTorProject/feature/improve_bridge_reachability
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 26b8442d849933f6b2c63343bb000cad3afb0d2b Merge: 6413bc0 9716b21 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 18 14:33:30 2014 +0200 Merge pull request #350 from TheTorProject/feature/improve_bridge_reachability Feature/improve bridge reachability ooni/nettests/blocking/bridge_reachability.py | 16 ++++++++++++++-- ooni/tasks.py | 6 +++++- 2 files changed, 19 insertions(+), 3 deletions(-)

1 0

[ooni-probe/master] Record the log output of tor by creating a temporary logfile and reading from it.
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 02a78452d747648b97acc074d82287dd039b6971 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Sep 13 21:00:07 2014 -0400 Record the log output of tor by creating a temporary logfile and reading from it. --- ooni/nettests/blocking/bridge_reachability.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index e0d0d8a..d4f6e13 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -1,5 +1,7 @@ # -*- encoding: utf-8 -*- +import os import random +import tempfile from distutils.spawn import find_executable from twisted.python import usage @@ -49,6 +51,8 @@ class BridgeReachability(nettest.NetTestCase): self.tor_progress = 0 self.timeout = int(self.localOptions['timeout']) + _, self.logfile = tempfile.mkstemp() + self.report['error'] = None self.report['success'] = None self.report['timeout'] = self.timeout @@ -170,7 +174,7 @@ class BridgeReachability(nettest.NetTestCase): config.Bridge = self.bridge config.UseBridges = 1 - config.log = 'notice' + config.log = ['notice stdout', 'notice file %s' % self.logfile] config.save() def updates(prog, tag, summary): @@ -194,7 +198,14 @@ class BridgeReachability(nettest.NetTestCase): @d.addErrback def setup_failed(failure): log.msg("Failed to connect to %s" % self.bridge) - self.report['tor_log'] = failure.value.message self.report['success'] = False + self.report['error'] = 'timeout-reached' + return + + @d.addCallback + def write_log(_): + with open(self.logfile) as f: + self.report['tor_log'] = f.read() + os.remove(self.logfile) return d

1 0

[ooni-probe/master] Refactor Parasitictraceroute.
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 5a924d156bc567a3319241645df700d6f4b9023f Author: aagbsn <aagbsn(a)extc.org> Date: Wed Aug 20 14:23:04 2014 +0000 Refactor Parasitictraceroute. In particular, scapy.supersocket.L3RawSocket does not capture transmitted packets. --- ooni/nettests/experimental/parasitictraceroute.py | 13 +++++++----- ooni/utils/txscapy.py | 22 +++++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/experimental/parasitictraceroute.py b/ooni/nettests/experimental/parasitictraceroute.py index 9526455..882d25a 100644 --- a/ooni/nettests/experimental/parasitictraceroute.py +++ b/ooni/nettests/experimental/parasitictraceroute.py @@ -1,5 +1,6 @@ from twisted.python import usage from twisted.internet import defer, reactor +from ooni.errors import handleAllFailures from ooni.templates import scapyt from ooni.utils import log from ooni.utils.txscapy import ParasiticTraceroute @@ -14,19 +15,23 @@ class ParasiticTracerouteTest(scapyt.BaseScapyTest): samplePeriod = 40 requiresTor = False - requiresRoot = False def setUp(self): self.report['parasitic_traceroute'] = {} def test_parasitic_traceroute(self): self.pt = ParasiticTraceroute() + log.debug("Starting ParasiticTraceroute for up to %d hosts at inject " + "rate %d with %s" % (self.pt.numHosts, self.pt.rate, self.pt)) config.scapyFactory.registerProtocol(self.pt) d = defer.Deferred() - reactor.callLater(self.samplePeriod, d.callback, self.pt) + reactor.callLater(self.samplePeriod, d.callback, self) + d.addCallback(self.addToReport) + d.addErrback(handleAllFailures) return d - def postProcessor(self, *args, **kwargs): + def addToReport(self, result): + log.debug("Stopping ParasiticTraceroute") self.pt.stopListening() self.report['received_packets'] = self.pt.received_packets @@ -46,5 +51,3 @@ class ParasiticTracerouteTest(scapyt.BaseScapyTest): self.report['parasitic_traceroute'][p].sort(key=lambda x: x[0]) self.report['sent_packets'] = self.pt.sent_packets - return self.report - diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index ba90223..e8a27b6 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -160,7 +160,7 @@ class ScapyFactory(abstract.FileDescriptor): if not super_socket and sys.platform == 'darwin': super_socket = conf.L3socket(iface=interface, promisc=True, filter='') elif not super_socket: - super_socket = L3RawSocket(iface=interface, promisc=True) + super_socket = conf.L3socket(iface=interface) self.protocols = [] fdesc._setCloseOnExec(super_socket.ins.fileno()) @@ -325,6 +325,8 @@ class ParasiticTraceroute(ScapyProtocol): def sendPacket(self, packet): self.factory.send(packet) + self.sent_packets.append(packet) + log.debug("Sent packet to %s with ttl %d" % (packet.dst, packet.ttl)) def packetReceived(self, packet): try: @@ -332,24 +334,25 @@ class ParasiticTraceroute(ScapyProtocol): except IndexError: return + # Add TTL Expired responses. if isinstance(packet.getlayer(3), TCPerror): self.received_packets.append(packet) + # Live traceroute? + log.debug("%s replied with icmp-ttl-exceeded for %s" % (packet.src, packet[IPerror].dst)) return - elif packet.dst in self.hosts: if random.randint(1, 100) > self.rate: + # Don't send a packet this time return try: packet[IP].ttl = self.hosts[packet.dst]['ttl'].pop() del packet.chksum # XXX Why is this incorrect? - log.debug("Sent packet to %s with ttl %d" % (packet.dst, packet.ttl)) self.sendPacket(packet) k = (packet.id, packet[TCP].sport, packet[TCP].dport, packet[TCP].seq) self.matched_packets[k] = {'ttl': packet.ttl} return except IndexError: - pass - return + return def maxttl(packet=None): if packet: @@ -369,17 +372,16 @@ class ParasiticTraceroute(ScapyProtocol): self.hosts[packet.dst] = {'ttl': genttl()} log.debug("Tracing to %s" % packet.dst) - - elif packet.src not in self.hosts \ + return + if packet.src not in self.hosts \ and packet.src not in self.addresses \ and isinstance(packet.getlayer(1), TCP): - self.hosts[packet.src] = {'ttl': genttl(packet), 'ttl_max': maxttl(packet)} log.debug("Tracing to %s" % packet.src) - return + return - elif packet.src in self.hosts and not 'ttl_max' in self.hosts[packet.src]: + if packet.src in self.hosts and not 'ttl_max' in self.hosts[packet.src]: self.hosts[packet.src]['ttl_max'] = ttl_max = maxttl(packet) log.debug("set ttl_max to %d for host %s" % (ttl_max, packet.src)) ttl = []

1 0

[ooni-probe/master] Merge pull request #348 from TheTorProject/fix/parasitictraceroute
by art＠torproject.org 22 Sep '14

22 Sep '14

commit 041c5df81eff3da190fb94f794f04b0ad696690f Merge: 90aa08c 5a924d1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Sep 6 17:51:04 2014 +0200 Merge pull request #348 from TheTorProject/fix/parasitictraceroute Refactor Parasitictraceroute. ooni/nettests/experimental/parasitictraceroute.py | 13 +++++++----- ooni/utils/txscapy.py | 22 +++++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-)

1 0