November 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Start adding support for using txsocksx inside of oonibclient.
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 4e9dd2911274054b4fd467bfac53798e8d0e1987 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 16 13:16:12 2013 +0200 Start adding support for using txsocksx inside of oonibclient. There is some badness happening because we use the httpo (HTTP over Tor Hidden Service) as the scheme name. --- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/oonibclient.py | 10 ++++++++-- ooni/reporter.py | 20 +++++++++++++------- ooni/templates/httpt.py | 2 +- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 3423442..6717cc1 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -11,7 +11,7 @@ from twisted.python import usage from ooni.utils import log, net, randomStr from ooni.templates import httpt -from ooni.utils.txagentwithsocks import TrueHeaders +from ooni.utils.trueheaders import TrueHeaders def random_capitalization(string): output = "" diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 555d99c..f05c5e7 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -46,11 +46,16 @@ class OONIBClient(object): retries = 3 def __init__(self, address): - self.address = address - self.agent = SOCKS5Agent(reactor, + if address.startswith('httpo://'): + self.address = address.replace('httpo://', 'http://') + self.agent = SOCKS5Agent(reactor, proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', config.tor.socks_port)) + elif address.startswith('https://'): + log.err("HTTPS based bouncers are currently not supported.") + + def _request(self, method, urn, genReceiver, bodyProducer=None): attempts = 0 @@ -71,6 +76,7 @@ class OONIBClient(object): # we have reached the retry count. if attempts < self.retries: log.err("Lookup failed. Retrying.") + log.exception(err) attempts += 1 perform_request() else: diff --git a/ooni/reporter.py b/ooni/reporter.py index b5fcce8..7566ca5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -17,6 +17,8 @@ from twisted.trial import reporter from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError from twisted.python.failure import Failure +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.web.client import Agent from ooni.utils import log @@ -35,7 +37,6 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.settings import config from ooni.tasks import ReportEntry, TaskTimedOut, ReportTracker - class ReporterException(Exception): pass @@ -289,13 +290,18 @@ class OONIBReporter(OReporter): # do this with some deferred kung foo or instantiate the reporter after # tor is started. - from ooni.utils.txagentwithsocks import Agent + from ooni.utils.trueheaders import SOCKS5Agent from twisted.internet import reactor - try: - self.agent = Agent(reactor, sockshost="127.0.0.1", - socksport=int(config.tor.socks_port)) - except Exception, e: - log.exception(e) + + if self.collectorAddress.startswith('httpo://'): + self.collectorAddress.replace('httpo://', 'http://') + self.agent = SOCKS5Agent(reactor, + proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', + config.tor.socks_port)) + + elif self.collectorAddress.startswith('https://'): + # XXX add support for securely reporting to HTTPS collectors. + log.err("HTTPS based collectors are currently not supported.") url = self.collectorAddress + '/report' diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 9a44c18..b6272e9 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -19,7 +19,7 @@ from ooni.settings import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni.utils.txagentwithsocks import TrueHeaders +from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures

1 0

[ooni-probe/master] Add keys for each traceroute type to report
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 3cd285b20a2044f289be935f00df6e62b2ac5ae7 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Oct 18 12:17:38 2013 +0000 Add keys for each traceroute type to report --- ooni/nettests/manipulation/traceroute.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index 42d5f8d..f04360d 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -37,6 +37,9 @@ class TracerouteTest(scapyt.BaseScapyTest): return random.randint(1024, 65535) self.get_sport = get_sport + self.report['test_tcp_traceroute'] = {} + self.report['test_udp_traceroute'] = {} + self.report['test_icmp_traceroute'] = {} def max_ttl_and_timeout(self): max_ttl = int(self.localOptions['maxttl']) @@ -45,13 +48,6 @@ class TracerouteTest(scapyt.BaseScapyTest): self.report['timeout'] = timeout return max_ttl, timeout - - def postProcessor(self, report): - tcp_hops = report['test_tcp_traceroute'] - udp_hops = report['test_udp_traceroute'] - icmp_hops = report['test_icmp_traceroute'] - - def test_tcp_traceroute(self): """ Does a traceroute to the destination by sending TCP SYN packets @@ -60,7 +56,7 @@ class TracerouteTest(scapyt.BaseScapyTest): def finished(packets, port): log.debug("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets - self.report['hops_'+str(port)] = [] + self.report['test_tcp_traceroute']['hops_'+str(port)] = [] for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, @@ -68,7 +64,7 @@ class TracerouteTest(scapyt.BaseScapyTest): 'sport': snd[TCP].sport } log.debug("%s: %s" % (port, report)) - self.report['hops_'+str(port)].append(report) + self.report['test_tcp_traceroute']['hops_'+str(port)].append(report) dl = [] max_ttl, timeout = self.max_ttl_and_timeout() @@ -90,7 +86,7 @@ class TracerouteTest(scapyt.BaseScapyTest): def finished(packets, port): log.debug("Finished running UDP traceroute test on port %s" % port) answered, unanswered = packets - self.report['hops_'+str(port)] = [] + self.report['test_udp_traceroute']['hops_'+str(port)] = [] for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, @@ -98,7 +94,7 @@ class TracerouteTest(scapyt.BaseScapyTest): 'sport': snd[UDP].sport } log.debug("%s: %s" % (port, report)) - self.report['hops_'+str(port)].append(report) + self.report['test_udp_traceroute']['hops_'+str(port)].append(report) dl = [] max_ttl, timeout = self.max_ttl_and_timeout() for port in self.dst_ports: @@ -119,14 +115,14 @@ class TracerouteTest(scapyt.BaseScapyTest): def finished(packets): log.debug("Finished running ICMP traceroute test") answered, unanswered = packets - self.report['hops'] = [] + self.report['test_icmp_traceroute']['hops'] = [] for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, 'rtt': rcv.time - snd.time } log.debug("%s" % (report)) - self.report['hops'].append(report) + self.report['test_icmp_traceroute']['hops'].append(report) dl = [] max_ttl, timeout = self.max_ttl_and_timeout() packets = IP(dst=self.localOptions['backend'],

1 0

[ooni-probe/master] Use our TrueHeaders version of the SOCKS5Agent in oonibclient
by art＠torproject.org 06 Nov '13

06 Nov '13

commit d9a1be184d80899cd981103f653ec2802c88a364 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Oct 10 14:07:22 2013 +0000 Use our TrueHeaders version of the SOCKS5Agent in oonibclient --- ooni/oonibclient.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index f05c5e7..889dcd9 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -6,13 +6,12 @@ from hashlib import sha256 from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint -from txsocksx.http import SOCKS5Agent - from ooni.deck import Deck, InputFile from ooni import errors as e from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader +from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent class Collector(object): def __init__(self, address): @@ -48,7 +47,7 @@ class OONIBClient(object): def __init__(self, address): if address.startswith('httpo://'): self.address = address.replace('httpo://', 'http://') - self.agent = SOCKS5Agent(reactor, + self.agent = TrueHeadersSOCKS5Agent(reactor, proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', config.tor.socks_port))

1 0

[ooni-probe/master] str().replace() returns a copy and does not modify in place.
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 08d1c15424b9c85beadd00a562b25289cd4ff997 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Oct 10 14:07:59 2013 +0000 str().replace() returns a copy and does not modify in place. --- ooni/reporter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 7566ca5..f4521b6 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -290,11 +290,12 @@ class OONIBReporter(OReporter): # do this with some deferred kung foo or instantiate the reporter after # tor is started. - from ooni.utils.trueheaders import SOCKS5Agent + from txsocksx.http import SOCKS5Agent from twisted.internet import reactor if self.collectorAddress.startswith('httpo://'): - self.collectorAddress.replace('httpo://', 'http://') + self.collectorAddress = \ + self.collectorAddress.replace('httpo://', 'http://') self.agent = SOCKS5Agent(reactor, proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', config.tor.socks_port))

1 0

[ooni-probe/master] Merge pull request #235 from TheTorProject/fix/lookup_failures
by art＠torproject.org 06 Nov '13

06 Nov '13

commit a56c2fffd6804a85620aac746594df30b3877683 Merge: 2d3c12e 7dcd509 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 4 10:43:29 2013 -0800 Merge pull request #235 from TheTorProject/fix/lookup_failures Fix/lookup failures ooni/oonibclient.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)

1 0

[ooni-probe/master] Decode correct protocol
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 7df1f620074ed933b46c9361d7969d65184b81a5 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Oct 16 16:52:25 2013 +0000 Decode correct protocol No wonder it didn't work. --- ooni/nettests/manipulation/traceroute.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index 2db1826..42d5f8d 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -62,16 +62,10 @@ class TracerouteTest(scapyt.BaseScapyTest): answered, unanswered = packets self.report['hops_'+str(port)] = [] for snd, rcv in answered: - try: - sport = snd[UDP].sport - except IndexError: - log.err("Source port for this traceroute was not found. This is probably a bug") - sport = -1 - report = {'ttl': snd.ttl, 'address': rcv.src, 'rtt': rcv.time - snd.time, - 'sport': sport + 'sport': snd[TCP].sport } log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report)

1 0

[ooni-probe/master] Fix bug in exception printing.
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 7dcd509e3e81789362bd07e878aac2d04b26509b Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Nov 4 10:01:57 2013 +0100 Fix bug in exception printing. --- ooni/oonibclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 1b11347..4853047 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -219,8 +219,8 @@ class OONIBClient(object): test_helper = yield self.queryBackend('POST', '/bouncer', query={'test-helpers': test_helper_names}) - except Exception, e: - log.exception(e) + except Exception, exc: + log.exception(exc) raise e.CouldNotFindTestHelper if not test_helper:

1 0

[ooni-probe/master] Merge pull request #208 from TheTorProject/feature/use_txsocksx
by art＠torproject.org 06 Nov '13

06 Nov '13

commit 221372e3ebdc1d29c55620e6045e90d05423b71b Merge: a56c2ff 08d1c15 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 4 14:02:58 2013 -0800 Merge pull request #208 from TheTorProject/feature/use_txsocksx Feature/use txsocksx .../manipulation/http_header_field_manipulation.py | 2 +- ooni/oonibclient.py | 17 +- ooni/reporter.py | 21 +- ooni/templates/httpt.py | 20 +- ooni/tests/test_trueheaders.py | 2 +- ooni/utils/trueheaders.py | 159 +++++++++++++++ ooni/utils/txagentwithsocks.py | 215 -------------------- 7 files changed, 199 insertions(+), 237 deletions(-) diff --cc ooni/oonibclient.py index 4853047,889dcd9..5735155 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@@ -69,8 -75,9 +75,9 @@@ class OONIBClient(object) # we have reached the retry count. if attempts < self.retries: log.err("Lookup failed. Retrying.") + log.exception(err) attempts += 1 - perform_request() + perform_request(attempts) else: log.err("Failed. Giving up.") finished.errback(err) diff --cc ooni/reporter.py index 72c6975,f4521b6..d90c259 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -17,9 -17,11 +17,11 @@@ from twisted.trial import reporte from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError from twisted.python.failure import Failure + from twisted.internet.endpoints import TCP4ClientEndpoint + from twisted.web.client import Agent from ooni.utils import log - +from ooni.tasks import Measurement try: from scapy.packet import Packet except ImportError:

1 0

[ooni-probe/master] Merge pull request #230 from TheTorProject/fix/traceroute_decodes_wrong_protocol
by art＠torproject.org 06 Nov '13

06 Nov '13

commit bbd2e3143e8a45b3c3bbc2ba7013532b8945ee27 Merge: 221372e 3cd285b Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Wed Nov 6 00:59:23 2013 -0800 Merge pull request #230 from TheTorProject/fix/traceroute_decodes_wrong_protocol Decode correct protocol ooni/nettests/manipulation/traceroute.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-)

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 06 Nov '13

06 Nov '13

commit 6b467f956ae0777fb76614bef5dc289fed30a489 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Nov 6 01:46:13 2013 +0000 Update translations for tails-misc --- pl.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl.po b/pl.po index 62dc85b..e333be2 100644 --- a/pl.po +++ b/pl.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-09-09 12:35+0200\n" -"PO-Revision-Date: 2013-11-06 01:11+0000\n" +"PO-Revision-Date: 2013-11-06 01:20+0000\n" "Last-Translator: sebx\n" "Language-Team: Polish (http://www.transifex.com/projects/p/torproject/language/pl/)\n" "MIME-Version: 1.0\n"

1 0