- tor-commits - lists.torproject.org

[ooni-probe/master] Add test for Vanilla Tor
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8b7adcae3bc80d3a844b60aacde5b6d5f8a0a2ae Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 16:06:21 2016 +0100 Add test for Vanilla Tor --- ooni/nettests/blocking/vanilla_tor.py | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py new file mode 100644 index 0000000..257f21a --- /dev/null +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -0,0 +1,105 @@ +# -*- encoding: utf-8 -*- +import os +import tempfile +import shutil + +from twisted.python import usage +from twisted.internet import reactor, error + +import txtorcon + +from ooni.utils import log, onion, net +from ooni import nettest + + +class TorIsNotInstalled(Exception): + pass + + +class UsageOptions(usage.Options): + optParameters = [ + ['timeout', 't', 200, + 'Specify the timeout after which to consider ' + 'the Tor bootstrapping process to have failed'], ] + + +class VanillaTor(nettest.NetTestCase): + name = "Vanilla Tor" + description = "A test for checking if vanilla Tor connections work" + author = "Arturo Filastò" + version = "0.1.0" + + usageOptions = UsageOptions + + def requirements(self): + if not onion.find_tor_binary(): + raise TorIsNotInstalled( + "For instructions on installing Tor see: " + "https://www.torproject.org/download/download") + + def setUp(self): + self.tor_progress = 0 + self.timeout = int(self.localOptions['timeout']) + + fd, self.tor_logfile = tempfile.mkstemp() + os.close(fd) + self.tor_datadir = tempfile.mkdtemp() + + self.report['error'] = None + self.report['success'] = None + self.report['timeout'] = self.timeout + self.report['transport_name'] = 'vanilla' + self.report['tor_version'] = str(onion.tor_details['version']) + self.report['tor_progress'] = 0 + self.report['tor_progress_tag'] = None + self.report['tor_progress_summary'] = None + self.report['tor_log'] = None + + def test_full_tor_connection(self): + config = txtorcon.TorConfig() + config.ControlPort = net.randomFreePort() + config.SocksPort = net.randomFreePort() + config.DataDirectory = self.tor_datadir + log.msg( + "Connecting to tor %s" % + (onion.tor_details['version'])) + + config.log = ['notice stdout', 'notice file %s' % self.tor_logfile] + config.save() + + def updates(prog, tag, summary): + log.msg("Progress is at: %s%%" % (prog)) + self.report['tor_progress'] = int(prog) + self.report['tor_progress_tag'] = tag + self.report['tor_progress_summary'] = summary + + d = txtorcon.launch_tor(config, reactor, timeout=self.timeout, + progress_updates=updates) + + @d.addCallback + def setup_complete(proto): + try: + proto.transport.signalProcess('TERM') + except error.ProcessExitedAlready: + proto.transport.loseConnection() + log.msg("Successfully connected to Tor") + self.report['success'] = True + + @d.addErrback + def setup_failed(failure): + log.msg("Failed to connect to Tor") + self.report['success'] = False + self.report['error'] = 'timeout-reached' + return + + @d.addCallback + def write_log(_): + with open(self.tor_logfile) as f: + self.report['tor_log'] = f.read() + os.remove(self.tor_logfile) + try: + shutil.rmtree(self.tor_datadir) + except: + pass + + return d

1 0

[ooni-probe/master] When GeoIP lookup is disabled it's not possible to perform sanitisations
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 81d05bdd15bf68c271cac5b0cde257372f7b4892 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 14:48:06 2016 +0100 When GeoIP lookup is disabled it's not possible to perform sanitisations --- ooni/templates/httpt.py | 2 +- ooni/templates/process.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d0822f2..12995bb 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -135,7 +135,7 @@ class HTTPTest(NetTestCase): } if response: # Attempt to redact the IP address of the probe from the responses - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 42a4256..faf0a66 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -108,7 +108,7 @@ class ProcessTest(NetTestCase): self.report['commands'] = [] # Attempt to redact the IP address of the probe from the standard output - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]")

1 0

[ooni-probe/master] Merge pull request #465 from TheTorProject/fix/venv-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e2baf90930a5ce290021e29f8a35935969f24e1b Merge: 9bc3006 d76970b Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 15:13:41 2016 +0100 Merge pull request #465 from TheTorProject/fix/venv-doc Fix install doc on virtualenv setup README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Package virtualenvwrapper not needed
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d76970b926c5ae7a2607e596c715cb56f6621ea3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Mar 15 10:33:37 2016 -0300 Package virtualenvwrapper not needed Removing virtualenvwrapper packages requirement. As spotted by @hellais --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 58c8376..384f71b 100644 --- a/README.rst +++ b/README.rst @@ -346,7 +346,7 @@ Setting up development environment On Debian based systems a development environment can be setup as follows: (prerequisites include build essentials, python-dev, and tor; for tor see https://www.torproject.org/docs/debian.html.en) - sudo apt-get install python-pip python-virtualenv virtualenv virtualenvwrapper + sudo apt-get install python-pip python-virtualenv virtualenv sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev git clone https://github.com/TheTorProject/ooni-probe cd ooni-probe

1 0

[ooni-probe/master] Add support for stripping potential inclusion of probe IPs in reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ee2fa7b4093117bd618af03382809ade4e9832a0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 17:57:10 2016 +0100 Add support for stripping potential inclusion of probe IPs in reports --- ooni/templates/httpt.py | 3 +++ ooni/templates/process.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..d0822f2 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -134,6 +134,9 @@ class HTTPTest(NetTestCase): } } if response: + # Attempt to redact the IP address of the probe from the responses + if config.privacy.includeip is False: + response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 82457dc..42a4256 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -1,5 +1,6 @@ from twisted.internet import protocol, defer, reactor +from ooni.settings import config from ooni.nettest import NetTestCase from ooni.utils import log @@ -66,7 +67,7 @@ class ProcessDirector(protocol.ProcessProtocol): self.stderr += data if self.shouldClose(): self.close("condition_met") - self.handlRead(None, data) + self.handleRead(None, data) def inConnectionLost(self): @@ -105,6 +106,12 @@ class ProcessTest(NetTestCase): log.debug("Finished %s: %s" % (command, result)) if not isinstance(self.report.get('commands'), list): self.report['commands'] = [] + + # Attempt to redact the IP address of the probe from the standard output + if config.privacy.includeip is False: + result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") + result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]") + self.report['commands'].append({ 'command_name': ' '.join(command), 'command_stdout': result['stdout'],

1 0

[ooni-probe/master] Add support for setting the url and expected-body in tests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c17f564789b85bc9baa9dc1c64e77d8ff070aebd Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 15:06:59 2016 +0100 Add support for setting the url and expected-body in tests * When these are changed from their default values a flag is set to false --- docs/source/tests/lantern.rst | 6 ++++++ docs/source/tests/psiphon.rst | 7 +++++++ ooni/nettests/third_party/lantern.py | 30 ++++++++++++++++++++++++++---- ooni/nettests/third_party/psiphon.py | 29 +++++++++++++++++++---------- ooni/utils/net.py | 3 +++ 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 9f3af2f..9f3ee09 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -31,6 +31,11 @@ How to run the test `ooniprobe nettests/third_party/lantern.py` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe nettests/third_party/lantern.py --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + Sample report ============= @@ -55,6 +60,7 @@ test_version: 0.0.1 --- body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." bootstrapped: true +default_configuration: true input: null lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} ``` diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index 3934bcb..3499501 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -35,6 +35,12 @@ To test Psiphon when it is installed in a different path other than the user hom `ooniprobe third_party/psiphon -p <path to Psiphon repository>` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe third_party/psiphon --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + + How to install Psiphon =================== @@ -80,6 +86,7 @@ Sample report agent: agent input: null psiphon_installed: true + default_configuration: true requests: - request: body: null diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 187e210..da19566 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,11 +1,22 @@ import os +import distutils.spawn + from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.web.client import ProxyAgent, readBody +from twisted.python import usage + from ooni.templates.process import ProcessTest, ProcessDirector -from ooni.utils import log +from ooni.utils import log, net from ooni.errors import handleAllFailures -import distutils.spawn + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over lantern (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] + ] class LanternTest(ProcessTest): """ @@ -26,10 +37,21 @@ class LanternTest(ProcessTest): timeout = 120 def setUp(self): + self.report['body'] = None + self.report['failure'] = None + self.report['success'] = None + self.report['default_configuration'] = True + self.command = [distutils.spawn.find_executable("lantern"), "--headless"] self.bootstrapped = defer.Deferred() self.exited = False - self.url = 'http://www.google.com/humans.txt' + + self.url = self.localOptions['url'] + if self.url != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False def stop(self, reason=None): if not self.exited: @@ -49,7 +71,7 @@ class LanternTest(ProcessTest): def test_lantern_circumvent(self): def addResultToReport(result): self.report['body'] = result - if result.startswith('Google is built by a large'): + if result.startswith(self.localOptions['expected-body']): log.msg("Got the HTTP response body I expected!") self.report['success'] = True else: diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 575002b..2b18b18 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -6,17 +6,19 @@ from twisted.internet import defer, reactor from twisted.internet.error import ProcessExitedAlready from twisted.python import usage -from ooni.utils import log +from ooni.utils import log, net from ooni.templates import process, httpt class UsageOptions(usage.Options): - log.debug("UsageOptions") optParameters = [ - ['psiphonpath', 'p', None, 'Specify psiphon python client path.'] + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over psiphon (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] ] - class PsiphonTest(httpt.HTTPTest, process.ProcessTest): """ @@ -43,8 +45,19 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): def setUp(self): log.debug('PsiphonTest.setUp') + self.report['bootstrapped_success'] = None + self.report['request_success'] = None + self.report['psiphon_found'] = None + self.report['default_configuration'] = True + self.bootstrapped = defer.Deferred() - self.url = 'http://www.google.com/humans.txt' + self.url = self.localOptions['url'] + + if self.localOptions['url'] != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -83,10 +96,6 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') self.createCommand() - - self.report['bootstrapped_success'] = None - self.report['request_success'] = None - self.report['psiphon_found'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') self.report['psiphon_found'] = False @@ -121,7 +130,7 @@ connect(False) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - if res.body.startswith('Google is built by a large'): + if res.body.startswith(self.localOptions['expected-body']): self.report['request_success'] = True else: self.report['request_success'] = False diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 96d62a1..1f87101 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -46,6 +46,9 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} +# This is used as a default for checking if we get the expected result when +# fetching URLs over some proxy. +GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large') class StringProducer(object): implements(IBodyProducer)

1 0

[ooni-probe/master] Update user documentation for psiphon and lantern
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c6964f5fb8ddcc3cec5d3eeebb28a07bd9ab0c7b Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 17:53:39 2016 +0100 Update user documentation for psiphon and lantern --- docs/source/tests/lantern.rst | 17 +++++++++-------- docs/source/tests/psiphon.rst | 17 ++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 0903c3a..9f3af2f 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -17,22 +17,25 @@ Description =========== This test launches Lantern in --headless mode, and parses output to determine -if it has bootstrapped. After bootstrap, it fetches the URL supplied by the ---url option using Lanterns http proxy interface listening on 127.0.0.1.8787. +if it has bootstrapped. After bootstrap, it fetches a URL using Lanterns HTTP +proxy interface listening on 127.0.0.1.8787 and checks to see if the response +body matches the expected result. +As a URL for testing we use http://www.google.com/humans.txt and look for the +string "Google is built by a large" in the response body. The specific string used to determine bootstrap from Lantern output in version -"2.0.10" is "client (http) proxy at" from standard output. +"2.0.10" is "Successfully dialed via" from standard output. How to run the test =================== -`ooniprobe nettests/third_party/lantern.py -u http://<url>` +`ooniprobe nettests/third_party/lantern.py` Sample report ============= From running: -`ooniprobe nettests/third_party/lantern.py -u http://www.google.com` +`ooniprobe nettests/third_party/lantern.py` :: @@ -50,9 +53,7 @@ test_name: lantern_circumvention_tool_test test_version: 0.0.1 ... --- -body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ - >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has moved\n\ - <A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" +body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." bootstrapped: true input: null lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index b05f1f0..3934bcb 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -16,7 +16,12 @@ Details Description =========== -This test first check that the Psiphon path exists, then launches Psiphon and parses output to determine if it has bootstrapped. After bootstrap, it fetches google.com (or other URL specified by the --url argument) using Psiphons SOCKS proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy argument). +This test first check that the Psiphon path exists, then launches Psiphon and +parses output to determine if it has bootstrapped. After bootstrap, it fetches +`http://www.google.com/humans.txt` using Psiphons SOCKS +proxy listening on 127.0.0.1:1080 (or otherwise specified by the --socksproxy +argument). +It will then check to see if the response body contains the string: "Google is built by a large" The specific string used to determine bootstrap from Psiphon output in version "0.0.1" is "Press Ctrl-C to terminate." from standard output. @@ -26,10 +31,6 @@ How to run the test `ooniprobe third_party/psiphon` -To test Psiphon with a different URL run: - -`ooniprobe third_party/psiphon -u http://<url>/` - To test Psiphon when it is installed in a different path other than the user home: `ooniprobe third_party/psiphon -p <path to Psiphon repository>` @@ -49,7 +50,7 @@ To run Psiphon manually, it must be run inside of the proper directory: Sample report ============= -`ooniprobe third_party/psiphon -u http://google.com` +`ooniprobe third_party/psiphon` :: @@ -87,9 +88,7 @@ Sample report tor: {is_tor: false} url: http://google.com response: - body: "<HTML><HEAD><meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\"\ - >\n<TITLE>301 Moved</TITLE></HEAD><BODY>\n<H1>301 Moved</H1>\nThe document has\ - \ moved\n<A HREF=\"http://www.google.com/\">here</A>.\r\n</BODY></HTML>\r\n" + body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." code: 301 headers: - - Content-Length

1 0

[ooni-probe/master] Refactor lantern test to use the ProcessTest template
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c1ff7516d35b7f76fb6638ae2bce04db5c4b3864 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 17:55:16 2016 +0100 Refactor lantern test to use the ProcessTest template * Improve blocking detection --- ooni/nettests/third_party/lantern.py | 98 ++++++++++++++---------------------- 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 1f1b3b0..187e210 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,47 +1,13 @@ +import os from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint -from twisted.python import usage from twisted.web.client import ProxyAgent, readBody from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log from ooni.errors import handleAllFailures -import os.path -from os import getenv - -class UsageOptions(usage.Options): - optParameters = [ - ['url', 'u', None, 'Specify a single URL to test.'],] - -class LanternProcessDirector(ProcessDirector): - """ - This Process Director monitors Lantern during its - bootstrap and fires a callback if bootstrap is - successful or an errback if it fails to bootstrap - before timing out. - """ - - def __init__(self, d, timeout=None): - self.d = d - self.stderr = "" - self.stdout = "" - self.finished = None - self.timeout = timeout - self.stdin = None - self.timer = None - self.exit_reason = None - self.bootstrapped = defer.Deferred() - - def outReceived(self, data): - self.stdout += data - # output received, see if we have bootstrapped - if not self.bootstrapped.called and "Connected to proxy on localhost" in self.stdout: - log.debug("Bootstrap Detected") - self.cancelTimer() - self.bootstrapped.callback("bootstrapped") - +import distutils.spawn class LanternTest(ProcessTest): - """ This class tests Lantern (https://getlantern.org) @@ -54,47 +20,59 @@ class LanternTest(ProcessTest): """ name = "Lantern Circumvention Tool Test" - description = "Bootstraps Lantern and does a HTTP GET for the specified URL" + description = "Bootstraps Lantern, connects to a URL and verifies if it contains the expected input" author = "Aaron Gibson" - version = "0.0.1" - timeout = 20 - usageOptions = UsageOptions - requiredOptions = ['url'] + version = "0.1.0" + timeout = 120 def setUp(self): - self.command = ["lantern", "--headless"] - self.d = defer.Deferred() - self.processDirector = LanternProcessDirector(self.d, timeout=self.timeout) - self.d.addCallback(self.processEnded, self.command) - if self.localOptions['url']: - self.url = self.localOptions['url'] - - def runLantern(self): - paths = filter(os.path.exists,[os.path.join(os.path.expanduser(x), self.command[0]) for x in getenv('PATH').split(':')]) - log.debug("Spawning Lantern") - reactor.spawnProcess(self.processDirector, paths[0], self.command) + self.command = [distutils.spawn.find_executable("lantern"), "--headless"] + self.bootstrapped = defer.Deferred() + self.exited = False + self.url = 'http://www.google.com/humans.txt' + + def stop(self, reason=None): + if not self.exited: + self.processDirector.close() + self.processDirector.transport.signalProcess('TERM') + self.exited = True + + def handleRead(self, stdout=None, stderr=None): + """ + This is called with each chunk of data from stdout and stderr. + """ + if not self.bootstrapped.called and "Successfully dialed via" in self.processDirector.stdout: + log.msg("Lantern connection successful") + self.processDirector.cancelTimer() + self.bootstrapped.callback("bootstrapped") def test_lantern_circumvent(self): - proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) - agent = ProxyAgent(proxyEndpoint, reactor) - def addResultToReport(result): self.report['body'] = result - self.report['success'] = True + if result.startswith('Google is built by a large'): + log.msg("Got the HTTP response body I expected!") + self.report['success'] = True + else: + self.report['success'] = False def addFailureToReport(failure): + log.err("Failed to connect to lantern") + log.failure(failure) self.report['failure'] = handleAllFailures(failure) self.report['success'] = False def doRequest(noreason): - log.debug("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url) + proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 8787) + agent = ProxyAgent(proxyEndpoint, reactor) + log.msg("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url) request = agent.request("GET", self.url) request.addCallback(readBody) request.addCallback(addResultToReport) request.addCallback(self.processDirector.close) return request - self.processDirector.bootstrapped.addCallback(doRequest) - self.processDirector.bootstrapped.addErrback(addFailureToReport) - self.runLantern() + self.bootstrapped.addCallback(doRequest) + self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(self.stop) + self.d = self.run(self.command, env=os.environ, usePTY=1) return self.d

1 0

[ooni-probe/master] Set the measurement_start_time closer to the test method call to improve accuracy
by art＠torproject.org 29 Apr '16

29 Apr '16

commit cb718ade5beaf546bb87375f4a79bf16bd15956a Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 19:44:49 2016 +0100 Set the measurement_start_time closer to the test method call to improve accuracy --- ooni/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index b5a15e0..381902a 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -116,8 +116,6 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'measurement_start_time' not in self.testInstance.report.keys(): - self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() self.testInstance.setUp() @@ -141,6 +139,8 @@ class Measurement(TaskWithTimeout): pass def run(self): + if 'measurement_start_time' not in self.testInstance.report.keys(): + self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() return self.netTestMethod()

1 0

[ooni-probe/master] hotfix: remove unneeded calls for check_for_censorship in http_host test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 7405cacf858399ac7a18bc228cf8662c62a347cf Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 21:00:27 2016 +0100 hotfix: remove unneeded calls for check_for_censorship in http_host test --- ooni/nettests/manipulation/http_host.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index d0387cb..cab4d47 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -68,14 +68,12 @@ class HTTPHost(httpt.HTTPTest): if not body.startswith("{"): log.msg("This does not appear to be JSON") self.report['transparent_http_proxy'] = True - self.check_for_censorship(body) return try: content = json.loads(body) except: log.msg("The json does not parse, this is not what we expected") self.report['transparent_http_proxy'] = True - self.check_for_censorship(body) return # We base the determination of the presence of a transparent HTTP

1 0