May 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Patch twisted HTTPClientParser to accept HTTP response lines with a missing reason
by art＠torproject.org 30 May '16

30 May '16

commit c50f50cb0984cefabc82bd4f559062dbc893736c Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun May 8 19:09:46 2016 +0200 Patch twisted HTTPClientParser to accept HTTP response lines with a missing reason --- ooni/utils/trueheaders.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index a5b2fe3..5f2b399 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -115,6 +115,26 @@ class HTTPClientParser(_newclient.HTTPClientParser): headers.addRawHeader(name, value) + def statusReceived(self, status): + parts = status.split(b' ', 2) + if len(parts) != 3: + # Here we add the extra missing part. + parts.append("XXX") + + try: + statusCode = int(parts[1]) + except ValueError: + raise _newclient.ParseError(u"non-integer status code", status) + + self.response = _newclient.Response._construct( + self.parseVersion(parts[0]), + statusCode, + parts[2], + self.headers, + self.transport, + self.request) + + class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): def request(self, request): if self._state != 'QUIESCENT':

1 0

[ooni-probe/master] Take also into consideration if the HTTP headers match in detecting blockpages.
by art＠torproject.org 30 May '16

30 May '16

commit 958850898457e78e7091dd2aa5193e9956d26a8f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 9 15:56:35 2016 +0200 Take also into consideration if the HTTP headers match in detecting blockpages. This significantly brings down the false positive ratio. Though there are still some instances of sites being mis-reported as blocked. Examples of these are sites that have different web servers for sites in the various languages. --- ooni/nettests/blocking/web_connectivity.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 08cae57..eb05835 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -347,6 +347,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = 'inconsistent' tcp_connect = self.compare_tcp_experiments() + got_expected_web_page = (self.report['body_length_match'] or + self.report['headers_match']) if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -354,23 +356,30 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): # XXX we may want to have different codes for these two types of # blocking - elif (dns_consistent == True and tcp_connect == True and - self.report['body_length_match'] == False): - blocking = 'http' - elif (dns_consistent == True and tcp_connect == True and - experiment_http_failure is not None and - control_http_failure is None): + elif (dns_consistent == True and + tcp_connect == True and + got_expected_web_page == False): blocking = 'http' + elif (dns_consistent == True and + tcp_connect == True and + experiment_http_failure is not None and + control_http_failure is None): + if experiment_http_failure == 'dns_lookup_error': + blocking = 'dns' + else: + blocking = 'http' + elif (dns_consistent == False and - (experiment_http_failure is not None or - self.report['body_length_match'] == False)): + (got_expected_web_page == False or + experiment_http_failure is not None)): blocking = 'dns' # This happens when the DNS resolution is injected, but the domain # doesn't have a valid record anymore or it resolves to an address # that is only accessible from within the country/network of the probe. elif (dns_consistent == False and + got_expected_web_page == False and (self.control['dns']['failure'] is not None or control_http_failure != experiment_http_failure)): blocking = 'dns'

1 0

[ooni-probe/master] Use settings.Config.ooni_home as the data directory (to avoid broken paths when using sudo or setting environment vars)
by art＠torproject.org 30 May '16

30 May '16

commit edcb8379aa78d2e9fdeb039aa7e6fd82170fa2dd Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Apr 18 18:46:33 2016 +0200 Use settings.Config.ooni_home as the data directory (to avoid broken paths when using sudo or setting environment vars) --- ooni/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/settings.py b/ooni/settings.py index 6e0f9a5..ffbf68e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -58,7 +58,7 @@ class OConfig(object): @property def data_directory_candidates(self): dirs = [ - os.path.join(expanduser('~'+self.current_user), '.ooni'), + self.ooni_home, self.var_lib_path, self.usr_share_path, os.path.join(get_ooni_root(), '..', 'data'),

1 0

[ooni-probe/master] Refactor code for setting of tor port
by art＠torproject.org 30 May '16

30 May '16

commit 1e3954bb26b2ac94b968d3e60cbf7ef75bb4c9b9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 13 15:33:00 2016 +0200 Refactor code for setting of tor port --- ooni/director.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 82d0e85..e6f864e 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -345,11 +345,13 @@ class Director(object): log.msg("%d%%: %s" % (prog, summary)) tor_config = TorConfig() - if config.tor.control_port: - tor_config.ControlPort = config.tor.control_port + if config.tor.control_port is None: + config.tor.control_port = int(randomFreePort()) + if config.tor.socks_port is None: + config.tor.socks_port = int(randomFreePort()) - if config.tor.socks_port: - tor_config.SocksPort = config.tor.socks_port + tor_config.ControlPort = config.tor.control_port + tor_config.SocksPort = config.tor.socks_port if config.tor.data_dir: data_dir = os.path.expanduser(config.tor.data_dir) @@ -384,18 +386,6 @@ class Director(object): tor_config.User = pwd.getpwuid(os.geteuid()).pw_name tor_config.save() - - if not hasattr(tor_config, 'ControlPort'): - control_port = int(randomFreePort()) - tor_config.ControlPort = control_port - config.tor.control_port = control_port - - if not hasattr(tor_config, 'SocksPort'): - socks_port = int(randomFreePort()) - tor_config.SocksPort = socks_port - config.tor.socks_port = socks_port - - tor_config.save() log.debug("Setting control port as %s" % tor_config.ControlPort) log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)

1 0

[ooni-probe/master] Clean up the output of ooniprobe
by art＠torproject.org 30 May '16

30 May '16

commit a1cf2789966b9550e3a044a0d3df7393328a7dee Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 17 17:36:47 2016 +0200 Clean up the output of ooniprobe * Make OneShotFactory not be noisy * Don't log an error when we are just connecting to a backend to check if it's up. --- ooni/backend_client.py | 2 +- ooni/utils/net.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index ae376e4..3b551d4 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -143,7 +143,7 @@ class OONIBClient(object): except ValueError: raise e.get_error(None) if 'error' in response: - log.err("Got this backend error message %s" % response) + log.debug("Got this backend error message %s" % response) raise e.get_error(response['error']) return response diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 1f87101..ad5454e 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -3,21 +3,21 @@ import socket from random import randint from zope.interface import implements -from twisted.internet import protocol, defer +from twisted.internet import defer +from twisted.internet.protocol import Factory, Protocol from twisted.web.iweb import IBodyProducer from scapy.config import conf from ooni.errors import IfaceError -try: - from twisted.internet.endpoints import connectProtocol -except ImportError: - def connectProtocol(endpoint, protocol): - class OneShotFactory(protocol.Factory): - def buildProtocol(self, addr): - return protocol - return endpoint.connect(OneShotFactory()) +# This is our own connectProtocol to avoid noisy twisted cluttering our logs +def connectProtocol(endpoint, protocol): + class OneShotFactory(Factory): + noisy = False + def buildProtocol(self, addr): + return protocol + return endpoint.connect(OneShotFactory()) # if sys.platform.system() == 'Windows': # import _winreg as winreg @@ -68,7 +68,7 @@ class StringProducer(object): pass -class BodyReceiver(protocol.Protocol): +class BodyReceiver(Protocol): def __init__(self, finished, content_length=None, body_processor=None): self.finished = finished self.data = "" @@ -92,7 +92,7 @@ class BodyReceiver(protocol.Protocol): self.finished.errback(exc) -class Downloader(protocol.Protocol): +class Downloader(Protocol): def __init__(self, download_path, finished, content_length=None): self.finished = finished @@ -113,7 +113,7 @@ class Downloader(protocol.Protocol): self.finished.callback(None) -class ConnectAndCloseProtocol(protocol.Protocol): +class ConnectAndCloseProtocol(Protocol): def connectionMade(self): self.transport.loseConnection()

1 0

[ooni-probe/master] Change test for setupCollector() function
by art＠torproject.org 30 May '16

30 May '16

commit 8fd571f03319462babc1ca925cb147498d28f138 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed May 18 16:10:07 2016 +0200 Change test for setupCollector() function --- ooni/tests/test_oonicli.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 24877e8..4a58736 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -221,5 +221,11 @@ class TestOoniCli(ConfigTestCase): global_options = { 'collector': collector1 } - collector_address = setupCollector(global_options, collector2) - self.assertEqual(collector_address, collector1) + try: + collector_client = setupCollector(global_options, collector2) + self.assertEqual(collector_client.settings['address'], collector1) + self.assertEqual(collector_client.settings['type'], 'https') + except errors.CollectorUnsupported: + # Older versions of twisted will raise this. We could be more + # strict and do a check for older twisted versions in here. + pass

1 0

[ooni-probe/master] Merge pull request #511 from TheTorProject/fix/exit_ip
by art＠torproject.org 30 May '16

30 May '16

commit e2de248c9e1b778dabe345eb3d9a659ec6e88cc2 Merge: 07eef08 3ce772e Author: anadahz <anadahz(a)users.noreply.github.com> Date: Sat May 21 20:22:36 2016 +0000 Merge pull request #511 from TheTorProject/fix/exit_ip Fix regression bug that lead to exit_ip and exit_name not being included ooni/templates/httpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] Use .dev0 to indicate master branch to conform to PEP-440 (#507)
by art＠torproject.org 30 May '16

30 May '16

commit 2af0f7ebdc669926b21b3a1bb27712bc54d04ecb Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri May 20 17:55:28 2016 +0200 Use .dev0 to indicate master branch to conform to PEP-440 (#507) --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 15e4278..e02b430 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.5.0-master" +__version__ = "1.5.0.dev0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ __resources_version__ = "1.4.2"

1 0

[ooni-probe/master] Fix regression bug that lead to exit_ip and exit_name not being included
by art＠torproject.org 30 May '16

30 May '16

commit 3ce772e21ca395993d3ad2878cc394a768f5be79 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat May 21 19:33:42 2016 +0200 Fix regression bug that lead to exit_ip and exit_name not being included This closes #509 --- ooni/templates/httpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e608f8f..fe283a7 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -30,7 +30,7 @@ class StreamListener(StreamListenerMixin): def stream_succeeded(self, stream): host=self.request['url'].split('/')[2] try: - if stream.target_host == host and len(self.request['tor']) == 1: + if stream.target_host == host and self.request['tor']['exit_ip'] is None: self.request['tor']['exit_ip'] = stream.circuit.path[-1].ip self.request['tor']['exit_name'] = stream.circuit.path[-1].name config.tor_state.stream_listeners.remove(self)

1 0

[ooni-probe/master] Add support for Gzip Decoding in web connectivity test
by art＠torproject.org 30 May '16

30 May '16

commit f3cb68e06118ab7c20de9559404fae76715a5500 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 23 12:10:32 2016 +0200 Add support for Gzip Decoding in web connectivity test --- ooni/nettests/blocking/web_connectivity.py | 36 +++++++++++++++++++++--------- ooni/templates/httpt.py | 14 ++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 97cde60..f68b5aa 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -6,6 +6,7 @@ from urlparse import urlparse from ipaddr import IPv4Address, AddressValueError +from twisted.web.client import GzipDecoder from twisted.internet import reactor from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint @@ -72,6 +73,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): author = "Arturo Filastò" version = "0.1.0" + contentDecoders = [('gzip', GzipDecoder)] + usageOptions = UsageOptions inputFile = [ @@ -156,6 +159,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['dns_consistency'] = None self.report['body_length_match'] = None self.report['headers_match'] = None + self.report['status_code_match'] = None self.report['accessible'] = None self.report['blocking'] = None @@ -177,8 +181,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): 'ips': [] }, 'http_request': { - 'body_length': None, - 'failure': True, + 'body_length': -1, + 'failure': None, + 'status_code': -1, 'headers': {} } } @@ -270,6 +275,19 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): else: return False + def compare_http_experiments(self, experiment_http_response): + + self.report['body_length_match'] = \ + self.compare_body_lengths(experiment_http_response) + + self.report['headers_match'] = \ + self.compare_headers(experiment_http_response) + + self.report['status_code_match'] = ( + experiment_http_response.code == + self.control['http_request']['status_code'] + ) + def compare_dns_experiments(self, experiment_dns_answers): if self.control['dns']['failure'] is not None and \ self.control['dns']['failure'] == self.report['dns_experiment_failure']: @@ -319,7 +337,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def determine_blocking(self, experiment_http_response, experiment_dns_answers): blocking = False - control_http_failure = self.report['control']['http_request']['failure'] + control_http_failure = self.control['http_request']['failure'] if control_http_failure is not None: control_http_failure = control_http_failure.split(" ")[0] @@ -328,10 +346,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_failure = experiment_http_failure.split(" ")[0] if (experiment_http_failure is None and control_http_failure is None): - self.report['body_length_match'] = self.compare_body_lengths( - experiment_http_response) - self.report['headers_match'] = self.compare_headers( - experiment_http_response) + self.compare_http_experiments(experiment_http_response) dns_consistent = self.compare_dns_experiments(experiment_dns_answers) if dns_consistent is True: @@ -341,7 +356,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): tcp_connect = self.compare_tcp_experiments() got_expected_web_page = (self.report['body_length_match'] or - self.report['headers_match']) + self.report['headers_match']) and \ + self.report['status_code_match'] if (dns_consistent == True and tcp_connect == False and experiment_http_failure is not None): @@ -488,7 +504,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): for reason, urls in summary['blocked'].items(): log.msg("") - log.msg("URLS blocked due to {}".format(reason)) - log.msg("--------------------"+'-'*len(reason)) + log.msg("URLS possibly blocked due to {}".format(reason)) + log.msg("-----------------------------"+'-'*len(reason)) for url in urls: log.msg("* {}".format(url)) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 7cbfd1d..ca0306b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -5,8 +5,11 @@ import random from txtorcon.interface import StreamListenerMixin from twisted.web.client import readBody, PartialDownloadError +from twisted.web.client import ContentDecoderAgent + from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint + from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent from ooni.utils.trueheaders import FixedRedirectAgent @@ -96,6 +99,12 @@ class HTTPTest(NetTestCase): randomizeUA = False followRedirects = False + # You can specify a list of tuples in the format of (CONTENT_TYPE, + # DECODER) + # For example to support Gzip decoding you should specify + # contentDecoders = [('gzip', GzipDecoder)] + contentDecoders = [] + baseParameters = [['socksproxy', 's', None, 'Specify a socks proxy to use for requests (ip:port)']] @@ -138,6 +147,11 @@ class HTTPTest(NetTestCase): "(<= 10.1). I will not be able to follow redirects."\ "This may make the testing less precise.") + if len(self.contentDecoders) > 0: + self.control_agent = ContentDecoderAgent(self.control_agent, + self.contentDecoders) + self.agent = ContentDecoderAgent(self.agent, + self.contentDecoders) self.processInputs() log.debug("Finished test setup")

1 0