July 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Update lantern.py
by art＠torproject.org 27 Jul '15

27 Jul '15

commit c8c45cdb32f51593ba31bb48f43307b63f63b968 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jun 29 20:49:43 2015 +0200 Update lantern.py --- ooni/nettests/third_party/lantern.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 896e11e..c7173ed 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -80,11 +80,11 @@ class LanternTest(ProcessTest): def addResultToReport(result): self.report['body'] = result - self.report['bootstrapped'] = True + self.report['success'] = True def addFailureToReport(failure): self.report['failure'] = handleAllFailures(failure) - self.report['bootstrapped'] = False + self.report['success'] = False def doRequest(noreason): log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url)

1 0

[ooni-probe/master] Merge pull request #387 from anadahz/fix/feature/meek
by art＠torproject.org 27 Jul '15

27 Jul '15

commit a3b6e778fb7cf7bead9f8b43365cb49a8e8613a6 Merge: 90117e8 a987066 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jun 29 20:49:59 2015 +0200 Merge pull request #387 from anadahz/fix/feature/meek Fix/feature/meek ooni/nettests/blocking/meek_fronted_requests.py | 79 +++++++++++++++++++++++ 1 file changed, 79 insertions(+)

1 0

[ooni-probe/master] Merge pull request #388 from TheTorProject/feature/lantern
by art＠torproject.org 27 Jul '15

27 Jul '15

commit 90117e81023594bcf1bc453e1153fc6554088925 Merge: 7888084 c8c45cd Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jun 29 20:49:50 2015 +0200 Merge pull request #388 from TheTorProject/feature/lantern Feature/lantern ooni/nettests/third_party/lantern.py | 100 ++++++++++++++++++++++++++++++++++ ooni/templates/process.py | 5 +- 2 files changed, 104 insertions(+), 1 deletion(-)

1 0

[ooni-probe/master] PEP8 Corrections
by art＠torproject.org 27 Jul '15

27 Jul '15

commit a987066560c219ceee0748fd379302e0c1d2f156 Author: anadahz <kojgelo(a)inbox.com> Date: Fri Apr 17 01:45:55 2015 +0200 PEP8 Corrections --- ooni/nettests/blocking/meek_fronted_requests.py | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ooni/nettests/blocking/meek_fronted_requests.py b/ooni/nettests/blocking/meek_fronted_requests.py index 762e27e..3795008 100644 --- a/ooni/nettests/blocking/meek_fronted_requests.py +++ b/ooni/nettests/blocking/meek_fronted_requests.py @@ -7,12 +7,12 @@ from ooni.templates import httpt from ooni.utils import log class UsageOptions(usage.Options): - optParameters = [ ['ExpectedBody', 'B', + optParameters = [ ['expectedBody', 'B', 'I’m just a happy little web server.\n', 'Expected body content from GET response'], - ['DomainName', 'D', None, - 'Specify a single fronted DomainName to test.'], - ['HostHeader', 'H', None, + ['domainName', 'D', None, + 'Specify a single fronted domainName to test.'], + ['hostHeader', 'H', None, 'Specify "inside" Host Header to test.'] ] @@ -22,7 +22,7 @@ class meekTest(httpt.HTTPTest): Header of the "inside" meek-server. The meek-server handles a GET request and response with: "I’m just a happy little web server.\n". The input file should be formatted as (one per line): - "DomainName:HostHeader" + "domainName:hostHeader" www.google.com:meek-reflect.appspot.com ajax.aspnetcdn.com:az668014.vo.msecnd.net a0.awsstatic.com:d2zfqthxsdq309.cloudfront.net @@ -33,7 +33,7 @@ class meekTest(httpt.HTTPTest): usageOptions = UsageOptions inputFile = ['file', 'f', None, - "File containing the DomainName:HostHeader combinations to\ + "File containing the domainName:hostHeader combinations to\ be tested, one per line."] inputs = [('www.google.com', 'meek-reflect.appspot.com'), ('ajax.aspnetcdn.com', 'az668014.vo.msecnd.net'), @@ -49,31 +49,31 @@ class meekTest(httpt.HTTPTest): if self.input: if (isinstance(self.input, tuple) or isinstace(self.input, list)): - self.DomainName, self.header = self.input + self.domainName, self.header = self.input else: - self.DomainName, self.header = self.input.split(':') - elif (self.localOptions['DomainName'] and - self.localOptions['HostHeader']): - self.DomainName = self.localOptions['DomainName'] - self.header = self.localOptions['HostHeader'] + self.domainName, self.header = self.input.split(':') + elif (self.localOptions['domainName'] and + self.localOptions['hostHeader']): + self.domainName = self.localOptions['domainName'] + self.header = self.localOptions['hostHeader'] - self.ExpectedBody = self.localOptions['ExpectedBody'] - self.DomainName = 'https://' + self.DomainName + self.expectedBody = self.localOptions['expectedBody'] + self.domainName = 'https://' + self.domainName def test_meek_response(self): """ Detects if the fronted request is blocked. """ log.msg("Testing fronted domain:%s with Host Header:%s" - % (self.DomainName, self.header)) + % (self.domainName, self.header)) def process_body(body): - if self.ExpectedBody != body: + if self.expectedBody != body: self.report['success'] = False else: self.report['success'] = True headers = {} headers['Host'] = [self.header] - return self.doRequest(self.DomainName, method="GET", headers=headers, + return self.doRequest(self.domainName, method="GET", headers=headers, body_processor=process_body)

1 0

[ooni-probe/master] Add Tests for Lantern (https://getlantern.org)
by art＠torproject.org 27 Jul '15

27 Jul '15

commit c326ff127c97d8c139ee0c7b36cf3dc8ac2c8afa Author: aagbsn <aagbsn(a)extc.org> Date: Sat Mar 28 20:07:12 2015 +0000 Add Tests for Lantern (https://getlantern.org) LanternBootstrapProcessDirector monitors Lanterns output (stdout, stderr) and calls back when Lantern has bootstrapped. If a timeout is configured and Lantern has not bootstrapped before it fires, then the process director will errback instead. LanternTest has a single test test_lantern_circumvent that waits for 20 seconds for LanternBootstrapProcessDirector to call back when Lantern has bootstrapped, and then makes a single request to http://google.com The report contains the output of stdout and stderr, if Lantern bootstrapped successfully, the page body requested, or a failure and reason. --- ooni/nettests/third_party/lantern.py | 108 ++++++++++++++++++++++++++++++++++ ooni/templates/process.py | 1 + 2 files changed, 109 insertions(+) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py new file mode 100644 index 0000000..cb5b57e --- /dev/null +++ b/ooni/nettests/third_party/lantern.py @@ -0,0 +1,108 @@ +from twisted.internet import defer, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.web.client import ProxyAgent, readBody +from ooni.templates.process import ProcessTest, ProcessDirector +from ooni.utils import log +from ooni.errors import handleAllFailures, TaskTimedOut +import os.path +from os import getenv + + +class LanternBootstrapProcessDirector(ProcessDirector): + """ + This Process Director monitors Lantern during its + bootstrap and fires a callback if bootstrap is + successful or an errback if it fails to bootstrap + before timing out. + """ + + def __init__(self, timeout=None): + self.stderr = "" + self.stdout = "" + self.finished = None + self.timeout = timeout + self.stdin = None + self.timer = None + self.exit_reason = None + self.bootstrapped = defer.Deferred() + + def errReceived(self, data): + self.stderr += data + + def processEnded(self, reason): + log.debug("Ended %s" % reason) + + def cancelTimer(self): + if self.timeout and self.timer: + self.timer.cancel() + log.debug("Cancelled Timer") + + def outReceived(self, data): + self.stdout += data + # output received, see if we have bootstrapped + if not self.bootstrapped.called and "client (http) proxy at" in self.stdout: + log.debug("Bootstrap Detected") + self.cancelTimer() + self.bootstrapped.callback("bootstrapped") + + +class LanternTest(ProcessTest): + + """ + This class tests Lantern (https://getlantern.org) + + test_lantern_circumvent + Starts Lantern on Linux in --headless mode and + determine if it bootstraps successfully or not. + Then, make a HTTP request for http://google.com + and records the response body or failure string. + + XXX: fix the path issue + """ + + name = "Lantern Circumvention Tool Test" + author = "Aaron Gibson" + version = "0.0.1" + timeout = 20 + + def setUp(self): + self.processDirector = LanternBootstrapProcessDirector(timeout=self.timeout) + + def runLantern(self): + command = ["lantern_linux", "--headless"] + paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), command[0]) for x in getenv('PATH').split(':')]) + if paths: + command[0] = paths[0] + log.debug("Spawning Lantern") + reactor.spawnProcess(self.processDirector, command[0], command) + + def addOutputToReport(self, result): + self.report['stdout'] = self.processDirector.stdout + self.report['stderr'] = self.processDirector.stderr + return result + + def test_lantern_circumvent(self): + + proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) + agent = ProxyAgent(proxyEndpoint, reactor) + + def addResultToReport(result): + self.report['body'] = result + self.report['bootstrapped'] = True + + def addFailureToReport(failure): + self.report['failure'] = handleAllFailures(failure) + self.report['bootstrapped'] = False + + def doRequest(noreason): + log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for http://google.com") + request = agent.request("GET", "http://google.com") + request.addCallback(readBody) + request.addCallback(addResultToReport) + return request + + self.processDirector.bootstrapped.addCallback(doRequest) + self.processDirector.bootstrapped.addBoth(self.addOutputToReport) + self.processDirector.bootstrapped.addErrback(addFailureToReport) + self.runLantern() + return self.processDirector.bootstrapped diff --git a/ooni/templates/process.py b/ooni/templates/process.py index c73ea29..5e187ee 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -21,6 +21,7 @@ class ProcessDirector(protocol.ProcessProtocol): self.transport.loseConnection() def resetTimer(self): + log.debug("Resetting Timer") if self.timeout is not None: if self.timer is not None: self.timer.cancel()

1 0

[ooni-probe/master] Do not reset timer when output is received
by art＠torproject.org 27 Jul '15

27 Jul '15

commit 084a99bd2e0ba6734ea2ad6029ab688372ed7818 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Mar 30 14:28:36 2015 +0000 Do not reset timer when output is received If the timer is reset upon output received, eventually OONI's built-in timer will fire and consider the the test a failure. As demonstrated in the example code, the process is launched by a Measurement method. Long-running processes probably shouldn't be started inside Measurements, and it might make more sense to launch processes as part of the NetTest setUp method instead. --- ooni/templates/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 5e187ee..2d62de1 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -51,7 +51,6 @@ class ProcessDirector(protocol.ProcessProtocol): self.transport.closeStdin() def outReceived(self, data): - self.resetTimer() log.debug("STDOUT: %s" % data) self.stdout += data if self.shouldClose():

1 0

[ooni-probe/master] Refactor to use template methods
by art＠torproject.org 27 Jul '15

27 Jul '15

commit f3dedf882764b99dc3a5c17fb6640640f70cd576 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Apr 3 19:34:29 2015 +0000 Refactor to use template methods --- ooni/nettests/third_party/lantern.py | 36 +++++++++++----------------------- ooni/templates/process.py | 5 ++++- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index cb5b57e..a0aea78 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -8,7 +8,7 @@ import os.path from os import getenv -class LanternBootstrapProcessDirector(ProcessDirector): +class LanternProcessDirector(ProcessDirector): """ This Process Director monitors Lantern during its bootstrap and fires a callback if bootstrap is @@ -16,7 +16,8 @@ class LanternBootstrapProcessDirector(ProcessDirector): before timing out. """ - def __init__(self, timeout=None): + def __init__(self, d, timeout=None): + self.d = d self.stderr = "" self.stdout = "" self.finished = None @@ -26,17 +27,6 @@ class LanternBootstrapProcessDirector(ProcessDirector): self.exit_reason = None self.bootstrapped = defer.Deferred() - def errReceived(self, data): - self.stderr += data - - def processEnded(self, reason): - log.debug("Ended %s" % reason) - - def cancelTimer(self): - if self.timeout and self.timer: - self.timer.cancel() - log.debug("Cancelled Timer") - def outReceived(self, data): self.stdout += data # output received, see if we have bootstrapped @@ -57,7 +47,6 @@ class LanternTest(ProcessTest): Then, make a HTTP request for http://google.com and records the response body or failure string. - XXX: fix the path issue """ name = "Lantern Circumvention Tool Test" @@ -66,23 +55,20 @@ class LanternTest(ProcessTest): timeout = 20 def setUp(self): - self.processDirector = LanternBootstrapProcessDirector(timeout=self.timeout) + self.d = defer.Deferred() + self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) + self.d.addCallback(self.processEnded, "lantern_linux") def runLantern(self): command = ["lantern_linux", "--headless"] - paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), command[0]) for x in getenv('PATH').split(':')]) - if paths: + + paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), command[0]) for x in getenv('PATH').split(':')]) + if paths: command[0] = paths[0] log.debug("Spawning Lantern") reactor.spawnProcess(self.processDirector, command[0], command) - def addOutputToReport(self, result): - self.report['stdout'] = self.processDirector.stdout - self.report['stderr'] = self.processDirector.stderr - return result - def test_lantern_circumvent(self): - proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) agent = ProxyAgent(proxyEndpoint, reactor) @@ -99,10 +85,10 @@ class LanternTest(ProcessTest): request = agent.request("GET", "http://google.com") request.addCallback(readBody) request.addCallback(addResultToReport) + request.addCallback(self.processDirector.close) return request self.processDirector.bootstrapped.addCallback(doRequest) - self.processDirector.bootstrapped.addBoth(self.addOutputToReport) self.processDirector.bootstrapped.addErrback(addFailureToReport) self.runLantern() - return self.processDirector.bootstrapped + return self.d diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 2d62de1..3bbd8f6 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -16,12 +16,15 @@ class ProcessDirector(protocol.ProcessProtocol): self.timer = None self.exit_reason = None + def cancelTimer(self): + if self.timeout and self.timer: + self.timer.cancel() + def close(self, reason=None): self.reason = reason self.transport.loseConnection() def resetTimer(self): - log.debug("Resetting Timer") if self.timeout is not None: if self.timer is not None: self.timer.cancel()

1 0

[ooni-probe/master] Add meek fronted requests test
by art＠torproject.org 27 Jul '15

27 Jul '15

commit 54fe88a3b8151c1a41ecd597ccf6a17db32d9af7 Author: V <auser@0> Date: Sun Apr 5 18:17:55 2015 +0200 Add meek fronted requests test --- ooni/nettests/blocking/meek_fronted_requests.py | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/ooni/nettests/blocking/meek_fronted_requests.py b/ooni/nettests/blocking/meek_fronted_requests.py new file mode 100644 index 0000000..20f42c9 --- /dev/null +++ b/ooni/nettests/blocking/meek_fronted_requests.py @@ -0,0 +1,74 @@ +# -*- encoding: utf-8 -*- +# +# :licence: see LICENSE + +from twisted.python import usage +from ooni.templates import httpt +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [ ['ExpectedBody', 'B', + 'I’m just a happy little web server.\n', + 'Expected body content from GET response'], + ['DomainName', 'D', None, + 'Specify a single fronted DomainName to test.'], + ['HostHeader', 'H', None, + 'Specify "inside" Host Header to test.'] + ] + +class meekTest(httpt.HTTPTest): + """ + Performs a HTTP GET request to a list of fronted domains with the Host + Header of the "inside" meek-server. The meek-server handles a GET request + and response with: "I’m just a happy little web server.\n". + The input file should be formatted as (one per line): + "DomainName:HostHeader" + + Some default meek DomainName and HostHeader combinations: + www.google.com:meek-reflect.appspot.com + ajax.aspnetcdn.com:az668014.vo.msecnd.net + a0.awsstatic.com:d2zfqthxsdq309.cloudfront.net + """ + name = "meek fronted requests test" + version = "0.0.1" + + usageOptions = UsageOptions + inputFile = ['file', 'f', None, + "File containing the DomainName:HostHeader combinations to\ + be tested, one per line."] + + requiresRoot = False + requiresTor = False + + def setUp(self): + """ + Check for inputs. + """ + if self.input: + self.DomainName, self.header = self.input.split(':') + elif (self.localOptions['DomainName'] and + self.localOptions['HostHeader']): + self.DomainName = self.localOptions['DomainName'] + self.header = self.localOptions['HostHeader'] + else: + raise Exception("No input specified") + + self.ExpectedBody = self.localOptions['ExpectedBody'] + self.DomainName = 'https://' + self.DomainName + + def test_meek_response(self): + """ + Detects if the fronted request is blocked. + """ + log.msg("Testing fronted domain:%s with Host Header:%s" + % (self.DomainName, self.header)) + def process_body(body): + if self.ExpectedBody != body: + self.report['censored'] = True + else: + self.report['censored'] = False + + headers = {} + headers['Host'] = [self.header] + return self.doRequest(self.DomainName, method="GET", headers=headers, + body_processor=process_body)

1 0

[ooni-probe/master] Add command line URL option and description
by art＠torproject.org 27 Jul '15

27 Jul '15

commit 74514bc964a80ed71c4502205b2604e436cb2de5 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Apr 6 13:39:40 2015 +0000 Add command line URL option and description --- ooni/nettests/third_party/lantern.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index a0aea78..37fe840 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,5 +1,6 @@ from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.python import usage from twisted.web.client import ProxyAgent, readBody from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log @@ -7,6 +8,9 @@ from ooni.errors import handleAllFailures, TaskTimedOut import os.path from os import getenv +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test.'],] class LanternProcessDirector(ProcessDirector): """ @@ -50,14 +54,19 @@ class LanternTest(ProcessTest): """ name = "Lantern Circumvention Tool Test" + description = "Bootstraps Lantern and does a HTTP GET for the specified URL" author = "Aaron Gibson" version = "0.0.1" timeout = 20 + usageOptions = UsageOptions + requiredOptions = ['url'] def setUp(self): self.d = defer.Deferred() self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) self.d.addCallback(self.processEnded, "lantern_linux") + if self.localOptions['url']: + self.url = self.localOptions['url'] def runLantern(self): command = ["lantern_linux", "--headless"] @@ -81,8 +90,8 @@ class LanternTest(ProcessTest): self.report['bootstrapped'] = False def doRequest(noreason): - log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for http://google.com") - request = agent.request("GET", "http://google.com") + log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url) + request = agent.request("GET", self.url) request.addCallback(readBody) request.addCallback(addResultToReport) request.addCallback(self.processDirector.close)

1 0

[ooni-probe/master] set command
by art＠torproject.org 27 Jul '15

27 Jul '15

commit 0a9856c9d4fd8741caf71c74c80f755f8e607ca3 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Apr 6 14:17:04 2015 +0000 set command --- ooni/nettests/third_party/lantern.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 37fe840..896e11e 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -62,20 +62,17 @@ class LanternTest(ProcessTest): requiredOptions = ['url'] def setUp(self): + self.command = ["lantern_linux", "--headless"] self.d = defer.Deferred() self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) - self.d.addCallback(self.processEnded, "lantern_linux") + self.d.addCallback(self.processEnded, self.command) if self.localOptions['url']: self.url = self.localOptions['url'] def runLantern(self): - command = ["lantern_linux", "--headless"] - - paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), command[0]) for x in getenv('PATH').split(':')]) - if paths: - command[0] = paths[0] + paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), self.command[0]) for x in getenv('PATH').split(':')]) log.debug("Spawning Lantern") - reactor.spawnProcess(self.processDirector, command[0], command) + reactor.spawnProcess(self.processDirector, paths[0], self.command) def test_lantern_circumvent(self): proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', 8787)

1 0