April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix invalid references to http_requests test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 6e4af441525a6b96858eb5e517a12f133a49f2e1 Author: anadahz <kojgelo(a)inbox.com> Date: Mon Mar 14 20:40:24 2016 -0300 Fix invalid references to http_requests test --- docs/source/tests/psiphon.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/tests/psiphon.rst b/docs/source/tests/psiphon.rst index 5050b79..b05f1f0 100644 --- a/docs/source/tests/psiphon.rst +++ b/docs/source/tests/psiphon.rst @@ -32,7 +32,7 @@ To test Psiphon with a different URL run: To test Psiphon when it is installed in a different path other than the user home: -`ooniprobe blocking/http_requests -p <path to Psiphon repository>` +`ooniprobe third_party/psiphon -p <path to Psiphon repository>` How to install Psiphon =================== @@ -49,7 +49,7 @@ To run Psiphon manually, it must be run inside of the proper directory: Sample report ============= -`ooniprobe blocking/http_requests -f example_inputs/url_lists_file.txt` +`ooniprobe third_party/psiphon -u http://google.com` ::

1 0

[ooni-probe/master] Fix install doc on virtualenv setup
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8a8500ab97a2eba292f93f05d129b9f627e59c6e Author: anadahz <kojgelo(a)inbox.com> Date: Mon Mar 14 19:09:06 2016 -0300 Fix install doc on virtualenv setup --- README.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6234b62..58c8376 100644 --- a/README.rst +++ b/README.rst @@ -350,7 +350,12 @@ On Debian based systems a development environment can be setup as follows: (prer sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev git clone https://github.com/TheTorProject/ooni-probe cd ooni-probe - mkvirtualenv ooniprobe # . ~/.virtualenvs/ooniprobe/bin/activate to access later + virtualenv venv +`virtualenv venv` will create a folder in the current directory which will +contain the Python executable files, and a copy of the pip library which you can +use to install other packages. To begin using the virtual environment, it needs +to be activated: + source venv/bin/activate pip install -r requirements.txt pip install -r requirements-dev.txt python setup.py install

1 0

[ooni-probe/master] Merge pull request #462 from TheTorProject/fix/install-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e1c428dbb24b81edccd67bb38188ccfcd756f9a0 Merge: 3a4d9ef f73a26a Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 16:05:48 2016 +0100 Merge pull request #462 from TheTorProject/fix/install-doc Remove obsolete install method on Debian/Ubuntu README.rst | 32 -------------------------------- 1 file changed, 32 deletions(-)

1 0

[ooni-probe/master] Always use the same URL when testing psiphon
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 959b84003e579398a3ecb1b91dd0b8a91efd9db2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 18:52:12 2016 +0100 Always use the same URL when testing psiphon * Minor refactoring --- ooni/nettests/third_party/psiphon.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index bfdec70..7c1462b 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -13,9 +13,8 @@ from ooni.templates import process, httpt class UsageOptions(usage.Options): log.debug("UsageOptions") optParameters = [ - ['url', 'u', None, 'Specify a single URL to test.'], - ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], - ['socksproxy', 's', None, 'Specify psiphon socks proxy ip:port.'],] + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'] + ] class PsiphonTest(httpt.HTTPTest, process.ProcessTest): @@ -33,16 +32,11 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): description = "Bootstraps Psiphon and \ does a HTTP GET for the specified URL" author = "juga" - version = "0.0.1" - timeout = 20 + version = "0.1.0" + timeout = 120 usageOptions = UsageOptions def _setUp(self): - # it is necessary to do this in _setUp instead of setUp - # because it needs to happen before HTTPTest's _setUp. - # incidentally, setting this option in setUp results in HTTPTest - # *saying* it is using this proxy while not actually using it. - log.debug('PiphonTest._setUp: setting socksproxy') self.localOptions['socksproxy'] = '127.0.0.1:1080' super(PsiphonTest, self)._setUp() @@ -50,10 +44,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): log.debug('PsiphonTest.setUp') self.bootstrapped = defer.Deferred() - if self.localOptions['url']: - self.url = self.localOptions['url'] - else: - self.url = 'https://check.torproject.org' + self.url = 'http://www.google.com/humans.txt' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -128,7 +119,11 @@ connect(False) d = self.doRequest(self.url) def addSuccessToReport(res): log.debug("PsiphonTest.callDoRequest.addSuccessToReport") - self.report['request_success'] = True + if res.body.startswith('Google is built by a large'): + self.report['request_success'] = True + else: + self.report['request_success'] = False + return res d.addCallback(addSuccessToReport) def addFailureToReport(res):

1 0

[ooni-probe/master] Merge pull request #464 from TheTorProject/fix/psiphon-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9bc3006d48f8d037958e1db7fc0011120f1f4e93 Merge: e1c428d 6e4af44 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 14:39:28 2016 +0100 Merge pull request #464 from TheTorProject/fix/psiphon-doc Fix invalid references to http_requests test docs/source/tests/psiphon.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Add test for Vanilla Tor
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 8b7adcae3bc80d3a844b60aacde5b6d5f8a0a2ae Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 16:06:21 2016 +0100 Add test for Vanilla Tor --- ooni/nettests/blocking/vanilla_tor.py | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py new file mode 100644 index 0000000..257f21a --- /dev/null +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -0,0 +1,105 @@ +# -*- encoding: utf-8 -*- +import os +import tempfile +import shutil + +from twisted.python import usage +from twisted.internet import reactor, error + +import txtorcon + +from ooni.utils import log, onion, net +from ooni import nettest + + +class TorIsNotInstalled(Exception): + pass + + +class UsageOptions(usage.Options): + optParameters = [ + ['timeout', 't', 200, + 'Specify the timeout after which to consider ' + 'the Tor bootstrapping process to have failed'], ] + + +class VanillaTor(nettest.NetTestCase): + name = "Vanilla Tor" + description = "A test for checking if vanilla Tor connections work" + author = "Arturo Filastò" + version = "0.1.0" + + usageOptions = UsageOptions + + def requirements(self): + if not onion.find_tor_binary(): + raise TorIsNotInstalled( + "For instructions on installing Tor see: " + "https://www.torproject.org/download/download") + + def setUp(self): + self.tor_progress = 0 + self.timeout = int(self.localOptions['timeout']) + + fd, self.tor_logfile = tempfile.mkstemp() + os.close(fd) + self.tor_datadir = tempfile.mkdtemp() + + self.report['error'] = None + self.report['success'] = None + self.report['timeout'] = self.timeout + self.report['transport_name'] = 'vanilla' + self.report['tor_version'] = str(onion.tor_details['version']) + self.report['tor_progress'] = 0 + self.report['tor_progress_tag'] = None + self.report['tor_progress_summary'] = None + self.report['tor_log'] = None + + def test_full_tor_connection(self): + config = txtorcon.TorConfig() + config.ControlPort = net.randomFreePort() + config.SocksPort = net.randomFreePort() + config.DataDirectory = self.tor_datadir + log.msg( + "Connecting to tor %s" % + (onion.tor_details['version'])) + + config.log = ['notice stdout', 'notice file %s' % self.tor_logfile] + config.save() + + def updates(prog, tag, summary): + log.msg("Progress is at: %s%%" % (prog)) + self.report['tor_progress'] = int(prog) + self.report['tor_progress_tag'] = tag + self.report['tor_progress_summary'] = summary + + d = txtorcon.launch_tor(config, reactor, timeout=self.timeout, + progress_updates=updates) + + @d.addCallback + def setup_complete(proto): + try: + proto.transport.signalProcess('TERM') + except error.ProcessExitedAlready: + proto.transport.loseConnection() + log.msg("Successfully connected to Tor") + self.report['success'] = True + + @d.addErrback + def setup_failed(failure): + log.msg("Failed to connect to Tor") + self.report['success'] = False + self.report['error'] = 'timeout-reached' + return + + @d.addCallback + def write_log(_): + with open(self.tor_logfile) as f: + self.report['tor_log'] = f.read() + os.remove(self.tor_logfile) + try: + shutil.rmtree(self.tor_datadir) + except: + pass + + return d

1 0

[ooni-probe/master] When GeoIP lookup is disabled it's not possible to perform sanitisations
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 81d05bdd15bf68c271cac5b0cde257372f7b4892 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 14:48:06 2016 +0100 When GeoIP lookup is disabled it's not possible to perform sanitisations --- ooni/templates/httpt.py | 2 +- ooni/templates/process.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d0822f2..12995bb 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -135,7 +135,7 @@ class HTTPTest(NetTestCase): } if response: # Attempt to redact the IP address of the probe from the responses - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 42a4256..faf0a66 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -108,7 +108,7 @@ class ProcessTest(NetTestCase): self.report['commands'] = [] # Attempt to redact the IP address of the probe from the standard output - if config.privacy.includeip is False: + if config.privacy.includeip is False and config.probe_ip.address is not None: result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]")

1 0

[ooni-probe/master] Merge pull request #465 from TheTorProject/fix/venv-doc
by art＠torproject.org 29 Apr '16

29 Apr '16

commit e2baf90930a5ce290021e29f8a35935969f24e1b Merge: 9bc3006 d76970b Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 15 15:13:41 2016 +0100 Merge pull request #465 from TheTorProject/fix/venv-doc Fix install doc on virtualenv setup README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Package virtualenvwrapper not needed
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d76970b926c5ae7a2607e596c715cb56f6621ea3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Mar 15 10:33:37 2016 -0300 Package virtualenvwrapper not needed Removing virtualenvwrapper packages requirement. As spotted by @hellais --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 58c8376..384f71b 100644 --- a/README.rst +++ b/README.rst @@ -346,7 +346,7 @@ Setting up development environment On Debian based systems a development environment can be setup as follows: (prerequisites include build essentials, python-dev, and tor; for tor see https://www.torproject.org/docs/debian.html.en) - sudo apt-get install python-pip python-virtualenv virtualenv virtualenvwrapper + sudo apt-get install python-pip python-virtualenv virtualenv sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev git clone https://github.com/TheTorProject/ooni-probe cd ooni-probe

1 0

[ooni-probe/master] Add support for stripping potential inclusion of probe IPs in reports
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ee2fa7b4093117bd618af03382809ade4e9832a0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Mar 11 17:57:10 2016 +0100 Add support for stripping potential inclusion of probe IPs in reports --- ooni/templates/httpt.py | 3 +++ ooni/templates/process.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 3702e86..d0822f2 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -134,6 +134,9 @@ class HTTPTest(NetTestCase): } } if response: + # Attempt to redact the IP address of the probe from the responses + if config.privacy.includeip is False: + response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '', diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 82457dc..42a4256 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -1,5 +1,6 @@ from twisted.internet import protocol, defer, reactor +from ooni.settings import config from ooni.nettest import NetTestCase from ooni.utils import log @@ -66,7 +67,7 @@ class ProcessDirector(protocol.ProcessProtocol): self.stderr += data if self.shouldClose(): self.close("condition_met") - self.handlRead(None, data) + self.handleRead(None, data) def inConnectionLost(self): @@ -105,6 +106,12 @@ class ProcessTest(NetTestCase): log.debug("Finished %s: %s" % (command, result)) if not isinstance(self.report.get('commands'), list): self.report['commands'] = [] + + # Attempt to redact the IP address of the probe from the standard output + if config.privacy.includeip is False: + result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]") + result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]") + self.report['commands'].append({ 'command_name': ' '.join(command), 'command_stdout': result['stdout'],

1 0