April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix install doc on virtualenv setup
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8a8500ab97a2eba292f93f05d129b9f627e59c6e Author: anadahz <kojgelo(a)inbox.com> Date: Mon Mar 14 19:09:06 2016 -0300 Fix install doc on virtualenv setup --- README.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6234b62..58c8376 100644 --- a/README.rst +++ b/README.rst @@ -350,7 +350,12 @@ On Debian based systems a development environment can be setup as follows: (prer sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev git clone https://github.com/TheTorProject/ooni-probe cd ooni-probe - mkvirtualenv ooniprobe # . ~/.virtualenvs/ooniprobe/bin/activate to access later + virtualenv venv +`virtualenv venv` will create a folder in the current directory which will +contain the Python executable files, and a copy of the pip library which you can +use to install other packages. To begin using the virtual environment, it needs +to be activated: + source venv/bin/activate pip install -r requirements.txt pip install -r requirements-dev.txt python setup.py install

1 0

[ooni-probe/master] Merge pull request #462 from TheTorProject/fix/install-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e1c428dbb24b81edccd67bb38188ccfcd756f9a0 Merge: 3a4d9ef f73a26a Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 16:05:48 2016 +0100 Merge pull request #462 from TheTorProject/fix/install-doc Remove obsolete install method on Debian/Ubuntu README.rst | 32 -------------------------------- 1 file changed, 32 deletions(-)

1 0

[ooni-probe/master] Always use the same URL when testing psiphon
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 959b84003e579398a3ecb1b91dd0b8a91efd9db2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 18:52:12 2016 +0100 Always use the same URL when testing psiphon * Minor refactoring --- ooni/nettests/third_party/psiphon.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index bfdec70..7c1462b 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -13,9 +13,8 @@ from ooni.templates import process, httpt class UsageOptions(usage.Options): log.debug("UsageOptions") optParameters = [ - ['url', 'u', None, 'Specify a single URL to test.'], - ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], - ['socksproxy', 's', None, 'Specify psiphon socks proxy ip:port.'],] + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'] + ] class PsiphonTest(httpt.HTTPTest, process.ProcessTest): @@ -33,16 +32,11 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): description = "Bootstraps Psiphon and \ does a HTTP GET for the specified URL" author = "juga" - version = "0.0.1" - timeout = 20 + version = "0.1.0" + timeout = 120 usageOptions = UsageOptions def _setUp(self): - # it is necessary to do this in _setUp instead of setUp - # because it needs to happen before HTTPTest's _setUp. - # incidentally, setting this option in setUp results in HTTPTest - # *saying* it is using this proxy while not actually using it. - log.debug('PiphonTest._setUp: setting socksproxy') self.localOptions['socksproxy'] = '127.0.0.1:1080' super(PsiphonTest, self)._setUp() @@ -50,10 +44,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): log.debug('PsiphonTest.setUp') self.bootstrapped = defer.Deferred() - if self.localOptions['url']: - self.url = self.localOptions['url'] - else: - self.url = 'https://check.torproject.org' + self.url = 'http://www.google.com/humans.txt' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -128,7 +119,11 @@ connect(False) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - self.report['request_success'] = True + if res.body.startswith('Google is built by a large'): + self.report['request_success'] = True + else: + self.report['request_success'] = False + return res d.addCallback(addSuccessToReport) def addFailureToReport(res):

1 0

[ooni-probe/master] Merge pull request #464 from TheTorProject/fix/psiphon-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9bc3006d48f8d037958e1db7fc0011120f1f4e93 Merge: e1c428d 6e4af44 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 14:39:28 2016 +0100 Merge pull request #464 from TheTorProject/fix/psiphon-doc Fix invalid references to http_requests test docs/source/tests/psiphon.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Add test for Vanilla Tor
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8b7adcae3bc80d3a844b60aacde5b6d5f8a0a2ae Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 16:06:21 2016 +0100 Add test for Vanilla Tor --- ooni/nettests/blocking/vanilla_tor.py | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py new file mode 100644 index 0000000..257f21a --- /dev/null +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -0,0 +1,105 @@ +# -*- encoding: utf-8 -*- +import os +import tempfile +import shutil + +from twisted.python import usage +from twisted.internet import reactor, error + +import txtorcon + +from ooni.utils import log, onion, net +from ooni import nettest + + +class TorIsNotInstalled(Exception): + pass + + +class UsageOptions(usage.Options): + optParameters = [ + ['timeout', 't', 200, + 'Specify the timeout after which to consider ' + 'the Tor bootstrapping process to have failed'], ] + + +class VanillaTor(nettest.NetTestCase): + name = "Vanilla Tor" + description = "A test for checking if vanilla Tor connections work" + author = "Arturo Filastò" + version = "0.1.0" + + usageOptions = UsageOptions + + def requirements(self): + if not onion.find_tor_binary(): + raise TorIsNotInstalled( + "For instructions on installing Tor see: " + "https://www.torproject.org/download/download") + + def setUp(self): + self.tor_progress = 0 + self.timeout = int(self.localOptions['timeout']) + + fd, self.tor_logfile = tempfile.mkstemp() + os.close(fd) + self.tor_datadir = tempfile.mkdtemp() + + self.report['error'] = None + self.report['success'] = None + self.report['timeout'] = self.timeout + self.report['transport_name'] = 'vanilla' + self.report['tor_version'] = str(onion.tor_details['version']) + self.report['tor_progress'] = 0 + self.report['tor_progress_tag'] = None + self.report['tor_progress_summary'] = None + self.report['tor_log'] = None + + def test_full_tor_connection(self): + config = txtorcon.TorConfig() + config.ControlPort = net.randomFreePort() + config.SocksPort = net.randomFreePort() + config.DataDirectory = self.tor_datadir + log.msg( + "Connecting to tor %s" % + (onion.tor_details['version'])) + + config.log = ['notice stdout', 'notice file %s' % self.tor_logfile] + config.save() + + def updates(prog, tag, summary): + log.msg("Progress is at: %s%%" % (prog)) + self.report['tor_progress'] = int(prog) + self.report['tor_progress_tag'] = tag + self.report['tor_progress_summary'] = summary + + d = txtorcon.launch_tor(config, reactor, timeout=self.timeout, + progress_updates=updates) + + @d.addCallback + def setup_complete(proto): + try: + proto.transport.signalProcess('TERM') + except error.ProcessExitedAlready: + proto.transport.loseConnection() + log.msg("Successfully connected to Tor") + self.report['success'] = True + + @d.addErrback + def setup_failed(failure): + log.msg("Failed to connect to Tor") + self.report['success'] = False + self.report['error'] = 'timeout-reached' + return + + @d.addCallback + def write_log(_): + with open(self.tor_logfile) as f: + self.report['tor_log'] = f.read() + os.remove(self.tor_logfile) + try: + shutil.rmtree(self.tor_datadir) + except: + pass + + return d

1 0

[ooni-probe/master] When GeoIP lookup is disabled it's not possible to perform sanitisations
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 81d05bdd15bf68c271cac5b0cde257372f7b4892 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 14:48:06 2016 +0100 When GeoIP lookup is disabled it's not possible to perform sanitisations --- ooni/templates/httpt.py | 2 +- ooni/templates/process.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d0822f2..12995bb 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -135,7 +135,7 @@ class HTTPTest(NetTestCase): } if response: # Attempt to redact the IP address of the probe from the responses - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 42a4256..faf0a66 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -108,7 +108,7 @@ class ProcessTest(NetTestCase): self.report['commands'] = [] # Attempt to redact the IP address of the probe from the standard output - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]")

1 0

[ooni-probe/master] Merge pull request #465 from TheTorProject/fix/venv-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e2baf90930a5ce290021e29f8a35935969f24e1b Merge: 9bc3006 d76970b Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 15:13:41 2016 +0100 Merge pull request #465 from TheTorProject/fix/venv-doc Fix install doc on virtualenv setup README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Package virtualenvwrapper not needed
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d76970b926c5ae7a2607e596c715cb56f6621ea3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Mar 15 10:33:37 2016 -0300 Package virtualenvwrapper not needed Removing virtualenvwrapper packages requirement. As spotted by @hellais --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 58c8376..384f71b 100644 --- a/README.rst +++ b/README.rst @@ -346,7 +346,7 @@ Setting up development environment On Debian based systems a development environment can be setup as follows: (prerequisites include build essentials, python-dev, and tor; for tor see https://www.torproject.org/docs/debian.html.en) - sudo apt-get install python-pip python-virtualenv virtualenv virtualenvwrapper + sudo apt-get install python-pip python-virtualenv virtualenv sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev git clone https://github.com/TheTorProject/ooni-probe cd ooni-probe

1 0

[ooni-probe/master] Add support for stripping potential inclusion of probe IPs in reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ee2fa7b4093117bd618af03382809ade4e9832a0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 17:57:10 2016 +0100 Add support for stripping potential inclusion of probe IPs in reports --- ooni/templates/httpt.py | 3 +++ ooni/templates/process.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..d0822f2 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -134,6 +134,9 @@ class HTTPTest(NetTestCase): } } if response: + # Attempt to redact the IP address of the probe from the responses + if config.privacy.includeip is False: + response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 82457dc..42a4256 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -1,5 +1,6 @@ from twisted.internet import protocol, defer, reactor +from ooni.settings import config from ooni.nettest import NetTestCase from ooni.utils import log @@ -66,7 +67,7 @@ class ProcessDirector(protocol.ProcessProtocol): self.stderr += data if self.shouldClose(): self.close("condition_met") - self.handlRead(None, data) + self.handleRead(None, data) def inConnectionLost(self): @@ -105,6 +106,12 @@ class ProcessTest(NetTestCase): log.debug("Finished %s: %s" % (command, result)) if not isinstance(self.report.get('commands'), list): self.report['commands'] = [] + + # Attempt to redact the IP address of the probe from the standard output + if config.privacy.includeip is False: + result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") + result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]") + self.report['commands'].append({ 'command_name': ' '.join(command), 'command_stdout': result['stdout'],

1 0

[ooni-probe/master] Add support for setting the url and expected-body in tests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c17f564789b85bc9baa9dc1c64e77d8ff070aebd Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 15:06:59 2016 +0100 Add support for setting the url and expected-body in tests * When these are changed from their default values a flag is set to false --- docs/source/tests/lantern.rst | 6 ++++++ docs/source/tests/psiphon.rst | 7 +++++++ ooni/nettests/third_party/lantern.py | 30 ++++++++++++++++++++++++++---- ooni/nettests/third_party/psiphon.py | 29 +++++++++++++++++++---------- ooni/utils/net.py | 3 +++ 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/docs/source/tests/lantern.rst b/docs/source/tests/lantern.rst index 9f3af2f..9f3ee09 100644 --- a/docs/source/tests/lantern.rst +++ b/docs/source/tests/lantern.rst @@ -31,6 +31,11 @@ How to run the test `ooniprobe nettests/third_party/lantern.py` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe nettests/third_party/lantern.py --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + Sample report ============= @@ -55,6 +60,7 @@ test_version: 0.0.1 --- body: "Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see google.com/careers." bootstrapped: true +default_configuration: true input: null lantern --headless: {exit_reason: process_done, stderr: '', stdout: ''} ``` diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index 3934bcb..3499501 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -35,6 +35,12 @@ To test Psiphon when it is installed in a different path other than the user hom `ooniprobe third_party/psiphon -p <path to Psiphon repository>` +For advanced usages you may also configure a different URL and expected body +for the response with the `--url` and `--expected-body` command line options. + +`ooniprobe third_party/psiphon --url http://www.github.com/humans.txt --expected-body '/* TEAM */'` + + How to install Psiphon =================== @@ -80,6 +86,7 @@ Sample report agent: agent input: null psiphon_installed: true + default_configuration: true requests: - request: body: null diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index 187e210..da19566 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -1,11 +1,22 @@ import os +import distutils.spawn + from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.web.client import ProxyAgent, readBody +from twisted.python import usage + from ooni.templates.process import ProcessTest, ProcessDirector -from ooni.utils import log +from ooni.utils import log, net from ooni.errors import handleAllFailures -import distutils.spawn + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over lantern (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] + ] class LanternTest(ProcessTest): """ @@ -26,10 +37,21 @@ class LanternTest(ProcessTest): timeout = 120 def setUp(self): + self.report['body'] = None + self.report['failure'] = None + self.report['success'] = None + self.report['default_configuration'] = True + self.command = [distutils.spawn.find_executable("lantern"), "--headless"] self.bootstrapped = defer.Deferred() self.exited = False - self.url = 'http://www.google.com/humans.txt' + + self.url = self.localOptions['url'] + if self.url != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False def stop(self, reason=None): if not self.exited: @@ -49,7 +71,7 @@ class LanternTest(ProcessTest): def test_lantern_circumvent(self): def addResultToReport(result): self.report['body'] = result - if result.startswith('Google is built by a large'): + if result.startswith(self.localOptions['expected-body']): log.msg("Got the HTTP response body I expected!") self.report['success'] = True else: diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 575002b..2b18b18 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -6,17 +6,19 @@ from twisted.internet import defer, reactor from twisted.internet.error import ProcessExitedAlready from twisted.python import usage -from ooni.utils import log +from ooni.utils import log, net from ooni.templates import process, httpt class UsageOptions(usage.Options): - log.debug("UsageOptions") optParameters = [ - ['psiphonpath', 'p', None, 'Specify psiphon python client path.'] + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], + ['url', 'u', net.GOOGLE_HUMANS[0], + 'Specify the URL to fetch over psiphon (default: http://www.google.com/humans.txt)'], + ['expected-body', 'e', net.GOOGLE_HUMANS[1], + 'Specify the beginning of the expected body in the response (default: ' + net.GOOGLE_HUMANS[1] + ')'] ] - class PsiphonTest(httpt.HTTPTest, process.ProcessTest): """ @@ -43,8 +45,19 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): def setUp(self): log.debug('PsiphonTest.setUp') + self.report['bootstrapped_success'] = None + self.report['request_success'] = None + self.report['psiphon_found'] = None + self.report['default_configuration'] = True + self.bootstrapped = defer.Deferred() - self.url = 'http://www.google.com/humans.txt' + self.url = self.localOptions['url'] + + if self.localOptions['url'] != net.GOOGLE_HUMANS[0]: + self.report['default_configuration'] = False + + if self.localOptions['expected-body'] != net.GOOGLE_HUMANS[1]: + self.report['default_configuration'] = False if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -83,10 +96,6 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') self.createCommand() - - self.report['bootstrapped_success'] = None - self.report['request_success'] = None - self.report['psiphon_found'] = None if not os.path.exists(self.psiphonpath): log.err('psiphon path does not exists, is it installed?') self.report['psiphon_found'] = False @@ -121,7 +130,7 @@ connect(False) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - if res.body.startswith('Google is built by a large'): + if res.body.startswith(self.localOptions['expected-body']): self.report['request_success'] = True else: self.report['request_success'] = False diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 96d62a1..1f87101 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -46,6 +46,9 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} +# This is used as a default for checking if we get the expected result when +# fetching URLs over some proxy. +GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large') class StringProducer(object): implements(IBodyProducer)

1 0