May 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] state null check
by art＠torproject.org 09 May '14

09 May '14

commit 8452924ad680658ff4f0cebfe5b44311aa7f2db3 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Fri May 2 22:06:04 2014 +0530 state null check --- ooni/templates/httpt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 1263261..59b550e 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -327,7 +327,8 @@ class HTTPTest(NetTestCase): if use_tor: state = config.tor_state - state.add_stream_listener(StreamListener(request)) + if state: + state.add_stream_listener(StreamListener(request)) d = agent.request(request['method'], request['url'], headers, body_producer)

1 0

[ooni-probe/master] Merge pull request #299 from arunpandianp/enhancement/http_req
by art＠torproject.org 09 May '14

09 May '14

commit d17fe2556aa32b6fe6eed17d03bb82278348c907 Merge: 1d308e3 8452924 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Mon May 5 15:12:05 2014 +0200 Merge pull request #299 from arunpandianp/enhancement/http_req Enhancement/http_req ooni/templates/httpt.py | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-)

1 0

[ooni-probe/master] removed unnecessary imports
by art＠torproject.org 09 May '14

09 May '14

commit f8162d8f4a32f0e70625ef35042609bfb06fbb7d Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Apr 8 03:20:49 2014 +0530 removed unnecessary imports --- ooni/templates/httpt.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index c9e42bf..44d01be 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,20 +1,13 @@ -import copy import random -import struct from zope.interface import implements -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from twisted.internet.ssl import ClientContextFactory +from twisted.internet import defer from txtorcon.interface import IStreamListener from twisted.internet import reactor -from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.web._newclient import Request, Response, ResponseNeverReceived -from twisted.web.client import Agent from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.nettest import NetTestCase

1 0

[ooni-probe/master] Refactorings
by art＠torproject.org 09 May '14

09 May '14

commit 53f207416383b7782639d0ee2177580ba23bc5e7 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Apr 8 04:12:03 2014 +0530 Refactorings --- ooni/templates/httpt.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 44d01be..1263261 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,10 +1,8 @@ import random -from zope.interface import implements - from twisted.internet import defer -from txtorcon.interface import IStreamListener +from txtorcon.interface import StreamListenerMixin from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -22,33 +20,21 @@ from ooni.errors import handleAllFailures class InvalidSocksProxyOption(Exception): pass -class StreamListener(object): - implements(IStreamListener) +class StreamListener(StreamListenerMixin): def __init__(self, request): self.request = request - def stream_new(self, stream): - pass - def stream_succeeded(self, stream): host=self.request['url'].split('/')[2] try: if stream.target_host == host and len(self.request['tor']) == 1: self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip self.request['tor']['exit_name'] = stream.circuit.path[-1].name + config.tor_state.stream_listeners.remove(self) except: log.err("Tor Exit ip detection failed") - def stream_attach(self, stream, circuit): - pass - - def stream_closed(self, stream,**k): - pass - - def stream_failed(self, stream, reason, remote_reason): - pass - class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to

1 0

[ooni-probe/master] Store tor exit router's ip and name with every request
by art＠torproject.org 09 May '14

09 May '14

commit b57552d009e6d8af08a66a88e01897faed5ba945 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Apr 8 03:12:12 2014 +0530 Store tor exit router's ip and name with every request --- ooni/templates/httpt.py | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 403e222..c9e42bf 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -2,10 +2,14 @@ import copy import random import struct +from zope.interface import implements + from twisted.plugin import IPlugin from twisted.internet import protocol, defer from twisted.internet.ssl import ClientContextFactory +from txtorcon.interface import IStreamListener + from twisted.internet import reactor from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError from twisted.internet.endpoints import TCP4ClientEndpoint @@ -18,7 +22,6 @@ from ooni.utils import log from ooni.settings import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents - from ooni.utils.trueheaders import TrueHeaders from ooni.errors import handleAllFailures @@ -26,6 +29,33 @@ from ooni.errors import handleAllFailures class InvalidSocksProxyOption(Exception): pass +class StreamListener(object): + implements(IStreamListener) + + def __init__(self, request): + self.request = request + + def stream_new(self, stream): + pass + + def stream_succeeded(self, stream): + host=self.request['url'].split('/')[2] + try: + if stream.target_host == host and len(self.request['tor']) == 1: + self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip + self.request['tor']['exit_name'] = stream.circuit.path[-1].name + except: + log.err("Tor Exit ip detection failed") + + def stream_attach(self, stream, circuit): + pass + + def stream_closed(self, stream,**k): + pass + + def stream_failed(self, stream, reason, remote_reason): + pass + class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -228,7 +258,7 @@ class HTTPTest(NetTestCase): content_length = int(response.headers.getRawHeaders('content-length')[0]) except Exception: content_length = None - + finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) finished.addCallback(self._processResponseBody, request, @@ -285,9 +315,11 @@ class HTTPTest(NetTestCase): request['url'] = url request['headers'] = headers request['body'] = body - request['tor'] = False + request['tor'] = {} if use_tor: - request['tor'] = True + request['tor']['is_tor'] = True + else: + request['tor']['is_tor'] = False if self.randomizeUA: log.debug("Randomizing user agent") @@ -306,7 +338,7 @@ class HTTPTest(NetTestCase): headers = TrueHeaders(request['headers']) def errback(failure, request): - if request['tor']: + if request['tor']['is_tor']: log.err("Error performing torified request: %s" % request['url']) else: log.err("Error performing request: %s" % request['url']) @@ -314,6 +346,10 @@ class HTTPTest(NetTestCase): self.addToReport(request, failure_string=failure_string) return failure + if use_tor: + state = config.tor_state + state.add_stream_listener(StreamListener(request)) + d = agent.request(request['method'], request['url'], headers, body_producer) d.addErrback(errback, request)

1 0

[ooni-probe/master] Update Readme to point to correct file location.
by art＠torproject.org 09 May '14

09 May '14

commit 41ea9d71a65c195a08a21b05fe651eb77d12f89d Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Fri May 9 16:35:14 2014 +0200 Update Readme to point to correct file location. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a59481e..b596e1f 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ ooniprobe will be installed in `/ooni`. 3) You can run tests with: ``` -ooniprobe blocking/http_requests -f /ooni/inputs/input-pack/alexa-top-1k.txt +ooniprobe blocking/http_requests -f /ooni/example_inputs/alexa-top-1k.txt ``` ## Using ooniprobe

1 0

[ooni-probe/master] Merge branch 'master' of github.com:TheTorProject/ooni-probe
by art＠torproject.org 09 May '14

09 May '14

commit 0e3e52c68f5fa6ac6926f2eb5e6eec4351711beb Merge: c822750 41ea9d7 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri May 9 22:44:17 2014 +0200 Merge branch 'master' of github.com:TheTorProject/ooni-probe * 'master' of github.com:TheTorProject/ooni-probe: Update Readme to point to correct file location. state null check Refactorings removed unnecessary imports Store tor exit router's ip and name with every request README.md | 2 +- ooni/templates/httpt.py | 42 +++++++++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-)

1 0

[ooni-probe/master] Call .path on FilePath instead of using os.path.
by art＠torproject.org 09 May '14

09 May '14

commit c82275034f71f60276e58012dacf4c0293422c0f Author: Arturo Filastò <art(a)fuffa.org> Date: Fri May 9 22:42:21 2014 +0200 Call .path on FilePath instead of using os.path. --- ooni/deck.py | 26 ++++++++++++++------------ ooni/oonicli.py | 2 +- ooni/reporter.py | 6 +++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 8a26d0c..ad90d03 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -21,7 +21,7 @@ class InputFile(object): cache_path = os.path.join(os.path.abspath(base_path), input_hash) self.cached_file = cache_path self.cached_descriptor = cache_path + '.desc' - + @property def descriptorCached(self): if os.path.exists(self.cached_descriptor): @@ -30,7 +30,7 @@ class InputFile(object): self.load(descriptor) return True return False - + @property def fileCached(self): if os.path.exists(self.cached_file): @@ -52,7 +52,7 @@ class InputFile(object): 'date': self.date, 'description': self.description }, f) - + def load(self, descriptor): self.name = descriptor['name'] self.version = descriptor['version'] @@ -69,7 +69,7 @@ class InputFile(object): def nettest_to_path(path, allow_arbitrary_paths=False): """ Takes as input either a path or a nettest name. - + Args: allow_arbitrary_paths: @@ -81,13 +81,15 @@ def nettest_to_path(path, allow_arbitrary_paths=False): """ if allow_arbitrary_paths and os.path.exists(path): return path - elif FilePath(config.nettest_directory).preauthChild(path + '.py').exists(): - return os.path.join(config.nettest_directory, path + '.py') + + fp = FilePath(config.nettest_directory).preauthChild(path + '.py') + if fp.exists(): + return fp.path else: raise e.NetTestNotFound(path) class Deck(InputFile): - def __init__(self, deck_hash=None, + def __init__(self, deck_hash=None, deckFile=None, decks_directory=config.decks_directory): self.id = deck_hash @@ -107,7 +109,7 @@ class Deck(InputFile): @property def cached_file(self): return os.path.join(self.decksDirectory, self.deckHash) - + @property def cached_descriptor(self): return self.cached_file + '.desc' @@ -162,7 +164,7 @@ class Deck(InputFile): if self.bouncer: log.msg("Looking up test helpers...") yield self.lookupTestHelpers() - + @defer.inlineCallbacks def lookupTestHelpers(self): self.oonibclient.address = self.bouncer @@ -178,7 +180,7 @@ class Deck(InputFile): if th['test_class'].localOptions[th['option']]: continue required_test_helpers.append(th['name']) - + if not required_test_helpers and not requires_collector: defer.returnValue(None) @@ -212,7 +214,7 @@ class Deck(InputFile): if 'url' in i: log.debug("Downloading %s" % i['url']) self.oonibclient.address = i['address'] - + try: input_file = yield self.oonibclient.downloadInput(i['hash']) except: @@ -222,5 +224,5 @@ class Deck(InputFile): input_file.verify() except AssertionError: raise e.UnableToLoadDeckInput, cached_path - + i['test_class'].localOptions[i['key']] = input_file.cached_file diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 889b108..9bd1add 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -181,7 +181,7 @@ def runWithDirector(logging=True, start_tor=True): except Exception as e: log.err(e) sys.exit(5) - + d = director.start(start_tor=start_tor) def setup_nettest(_): diff --git a/ooni/reporter.py b/ooni/reporter.py index a512891..273242e 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -171,7 +171,7 @@ class YAMLReporter(OReporter): if not os.path.isdir(report_destination): raise InvalidDestination - + if not report_filename: report_filename = "report-" + \ test_details['test_name'] + "-" + \ @@ -301,7 +301,7 @@ class OONIBReporter(OReporter): from txsocksx.http import SOCKS5Agent from twisted.internet import reactor - + if self.collectorAddress.startswith('httpo://'): self.collectorAddress = \ self.collectorAddress.replace('httpo://', 'http://') @@ -369,7 +369,7 @@ class OONIBReporter(OReporter): log.err("Failed to parse collector response %s" % backend_response) log.exception(e) raise errors.OONIBReportCreationError - + if response.code == 406: # XXX make this more strict log.err("The specified input or nettests cannot be submitted to this collector.")

1 0

[translation/https_everywhere_completed] Update translations for https_everywhere_completed
by translation＠torproject.org 09 May '14

09 May '14

commit 18aac1b156dfb11c926e009c7a312d001d214085 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 9 17:15:19 2014 +0000 Update translations for https_everywhere_completed --- bg/https-everywhere.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/bg/https-everywhere.properties b/bg/https-everywhere.properties index b00a14a..5c20247 100644 --- a/bg/https-everywhere.properties +++ b/bg/https-everywhere.properties @@ -5,3 +5,4 @@ https-everywhere.menu.noRules = (Няма правила за тази стра https-everywhere.menu.unknownRules = (Непознати правила за тази страница) https-everywhere.toolbar.hint = HTTPS Everywhere е активен. Можете да го превключвате от сайт на сайт чрез натискане на иконката в адресната лента. https-everywhere.migration.notification0 = За да се приложи критическа поправка, това обновление анулира вашите настройки на HTTPS Everywhere правилата към стандартните им стойности. +https-everywhere.menu.ruleset-tests = Стартиране на тест за правилата на HTTPS Everywhere

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 09 May '14

09 May '14

commit c4a648493019b39dbcf85d8b1f59c67594f49432 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 9 17:15:14 2014 +0000 Update translations for https_everywhere --- bg/https-everywhere.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bg/https-everywhere.properties b/bg/https-everywhere.properties index ad5b7c4..5c20247 100644 --- a/bg/https-everywhere.properties +++ b/bg/https-everywhere.properties @@ -5,4 +5,4 @@ https-everywhere.menu.noRules = (Няма правила за тази стра https-everywhere.menu.unknownRules = (Непознати правила за тази страница) https-everywhere.toolbar.hint = HTTPS Everywhere е активен. Можете да го превключвате от сайт на сайт чрез натискане на иконката в адресната лента. https-everywhere.migration.notification0 = За да се приложи критическа поправка, това обновление анулира вашите настройки на HTTPS Everywhere правилата към стандартните им стойности. -https-everywhere.menu.ruleset-tests = Run HTTPS Everywhere Ruleset Tests +https-everywhere.menu.ruleset-tests = Стартиране на тест за правилата на HTTPS Everywhere

1 0