January 2017 - tor-commits - lists.torproject.org

[ooni-probe/master] Add support for ignoring redirects to IPs in private IP space
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 4f0c2af37093070e4cec0593a0f57568a4db231c Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 23 16:08:27 2016 +0000 Add support for ignoring redirects to IPs in private IP space --- ooni/common/ip_utils.py | 29 +++++++++++++++++++++++++++++ ooni/common/txextra.py | 18 +++++++++++++++++- ooni/nettests/blocking/web_connectivity.py | 30 +++++++++--------------------- ooni/templates/httpt.py | 26 +++++++++++++++++++++----- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/ooni/common/ip_utils.py b/ooni/common/ip_utils.py new file mode 100644 index 0000000..8745b7a --- /dev/null +++ b/ooni/common/ip_utils.py @@ -0,0 +1,29 @@ +from ipaddr import IPv4Address, IPv6Address +from ipaddr import AddressValueError + + +def is_public_ipv4_address(address): + return not is_private_ipv4_address(address) + + +def is_private_ipv4_address(address): + try: + ip_address = IPv4Address(address) + return any( + [ip_address.is_private, ip_address.is_loopback] + ) + except AddressValueError: + return False + +def is_private_address(address): + try: + ip_address = IPv4Address(address) + except AddressValueError: + try: + ip_address = IPv6Address(address) + except AddressValueError: + return False + + return any( + [ip_address.is_private, ip_address.is_loopback] + ) diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py index 7a84592..39b8996 100644 --- a/ooni/common/txextra.py +++ b/ooni/common/txextra.py @@ -19,6 +19,8 @@ from twisted.internet.defer import Deferred, fail, maybeDeferred, failure from twisted.python import log +from .ip_utils import is_private_address + class TrueHeaders(Headers): def __init__(self, rawHeaders=None): self._rawHeaders = dict() @@ -168,6 +170,10 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): This is a redirect agent with this patch manually applied: https://twistedmatrix.com/trac/ticket/8265 """ + def __init__(self, agent, redirectLimit=20, ignorePrivateRedirects=False): + self.ignorePrivateRedirects = ignorePrivateRedirects + BrowserLikeRedirectAgent.__init__(self, agent, redirectLimit) + def _handleRedirect(self, response, method, uri, headers, redirectCount): """ Handle a redirect response, checking the number of redirects already @@ -191,12 +197,22 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): response.request.absoluteURI, locationHeaders[0] ) + uri = client.URI.fromBytes(location) + if self.ignorePrivateRedirects and (is_private_address(uri.host) or + uri.host == "localhost"): + return response + deferred = self._agent.request(method, location, headers) def _chainResponse(newResponse): + if isinstance(newResponse, Failure): + # This is needed to write the response even in case of failure + newResponse.previousResponse = response + newResponse.requestLocation = location + return newResponse newResponse.setPreviousResponse(response) return newResponse - deferred.addCallback(_chainResponse) + deferred.addBoth(_chainResponse) return deferred.addCallback( self._handleResponse, method, uri, headers, redirectCount + 1) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 1d1b742..629ef1c 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -3,28 +3,24 @@ import csv from urlparse import urlparse -from ipaddr import IPv4Address, AddressValueError - -from twisted.web.client import GzipDecoder +from twisted.internet import defer from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.names import client - -from twisted.internet import defer from twisted.python import usage +from twisted.web.client import GzipDecoder from ooni import geoip -from ooni.utils import log - from ooni.backend_client import WebConnectivityClient - +from ooni.common.http_utils import REQUEST_HEADERS from ooni.common.http_utils import extractTitle -from ooni.utils.net import COMMON_SERVER_HEADERS -from ooni.templates import httpt, dnst +from ooni.common.ip_utils import is_public_ipv4_address +from ooni.common.tcp_utils import TCPConnectFactory from ooni.errors import failureToString +from ooni.templates import httpt, dnst +from ooni.utils import log +from ooni.utils.net import COMMON_SERVER_HEADERS -from ooni.common.tcp_utils import TCPConnectFactory -from ooni.common.http_utils import REQUEST_HEADERS class InvalidControlResponse(Exception): pass @@ -42,15 +38,6 @@ class UsageOptions(usage.Options): ] -def is_public_ipv4_address(address): - try: - ip_address = IPv4Address(address) - return not any( - [ip_address.is_private, ip_address.is_loopback] - ) - except AddressValueError: - return None - class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity @@ -79,6 +66,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): requiresRoot = False requiresTor = False followRedirects = True + ignorePrivateRedirects = True # These are the options to be shown on the GUI simpleOptions = [ diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d1aa1dc..e57463b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -66,6 +66,10 @@ class HTTPTest(NetTestCase): randomizeUA = False followRedirects = False + # When this is set to False we will follow redirects pointing to IPs in + # rfc1918 + ignorePrivateRedirects = False + # You can specify a list of tuples in the format of (CONTENT_TYPE, # DECODER) # For example to support Gzip decoding you should specify @@ -107,7 +111,10 @@ class HTTPTest(NetTestCase): if self.followRedirects: try: self.control_agent = FixedRedirectAgent(self.control_agent) - self.agent = FixedRedirectAgent(self.agent) + self.agent = FixedRedirectAgent( + self.agent, + ignorePrivateRedirects=self.ignorePrivateRedirects + ) self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted " @@ -129,7 +136,8 @@ class HTTPTest(NetTestCase): def processInputs(self): pass - def addToReport(self, request, response=None, response_body=None, failure_string=None): + def addToReport(self, request, response=None, response_body=None, + failure_string=None, previous_response=None): """ Adds to the report the specified request and response. @@ -176,9 +184,10 @@ class HTTPTest(NetTestCase): session['failure'] = failure_string self.report['requests'].append(session) - if response and response.previousResponse: - self.addToReport(request, response.previousResponse, + previous_response = response.previousResponse + if previous_response: + self.addToReport(request, previous_response, response_body=None, failure_string=None) @@ -367,7 +376,14 @@ class HTTPTest(NetTestCase): else: log.err("Error performing HTTP request: %s" % request['url']) failure_string = handleAllFailures(failure) - self.addToReport(request, failure_string=failure_string) + previous_response = None + if getattr(failure, "previousResponse", None): + previous_response = failure.previousResponse + if getattr(failure, "requestLocation", None): + request['url'] = failure.requestLocation + + self.addToReport(request, failure_string=failure_string, + previous_response=previous_response) return failure if use_tor:

1 0

[ooni-probe/master] Add issue template (#604)
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 4ee1a90275bdd9fd120823beb5eaa44fe6aa2563 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 22 15:49:56 2016 +0000 Add issue template (#604) --- .github/ISSUE_TEMPLATE.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..3123450 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,25 @@ +### Description + +[Description of the bug or feature] + +### Steps to Reproduce + +1. [First Step] +2. [Second Step] +3. [and so on...] + +**Expected behavior:** [What you expected to happen] + +**Actual behavior:** [What actually happened] + +### Versions + +**How did you install ooniprobe** + +Did you get it via pip, homebrew or debian packages? + +**What version of ooniprobe are you running?** + +You can get this information by running `ooniprobe --version` + +**What operating system are you running and what version is it?**

1 0

[ooni-probe/master] Update changelog
by art＠torproject.org 13 Jan '17

13 Jan '17

commit a62215219c46b1c2e7587ae4ec5776ad2d5621c6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 29 21:53:56 2016 +0200 Update changelog --- ChangeLog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 528ef62..b2b8195 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -17,6 +17,8 @@ Bug fixes: * Fix https://github.com/TheTorProject/ooni-probe/issues/650 +* Fix https://github.com/TheTorProject/ooni-probe/issues/651 + v2.0.0 (Fri, 14 Oct 2016) ------------------------------ codename: mezzanine

1 0

[ooni-probe/master] Only check for loopback addresses when ignoring redirects
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 1b5fc495111c2881c7e0e7575131a299dec3e3c7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 23 16:38:58 2016 +0000 Only check for loopback addresses when ignoring redirects * Also look for .local domains in redirects --- ooni/common/ip_utils.py | 23 +++++++++++++++++++---- ooni/common/txextra.py | 4 ++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ooni/common/ip_utils.py b/ooni/common/ip_utils.py index 8745b7a..afc5bf4 100644 --- a/ooni/common/ip_utils.py +++ b/ooni/common/ip_utils.py @@ -15,15 +15,30 @@ def is_private_ipv4_address(address): except AddressValueError: return False -def is_private_address(address): + +def is_private_address(address, only_loopback=False): + """ + Checks to see if an IP address is in private IP space and if the + hostname is either localhost or *.local. + + :param address: an IP address of a hostname + :param only_loopback: will only check if the IP address is either + 127.0.0.1/8 or ::1 in ipv6 + :return: True if the IP address or host is in private space + """ try: ip_address = IPv4Address(address) except AddressValueError: try: ip_address = IPv6Address(address) except AddressValueError: + if address == "localhost": + return True + elif address.endswith(".local"): + return True return False - return any( - [ip_address.is_private, ip_address.is_loopback] - ) + candidates = [ip_address.is_loopback] + if not only_loopback: + candidates.append(ip_address.is_private) + return any(candidates) diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py index 39b8996..e3507bf 100644 --- a/ooni/common/txextra.py +++ b/ooni/common/txextra.py @@ -198,8 +198,8 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): locationHeaders[0] ) uri = client.URI.fromBytes(location) - if self.ignorePrivateRedirects and (is_private_address(uri.host) or - uri.host == "localhost"): + if self.ignorePrivateRedirects and is_private_address(uri.host, + only_loopback=True): return response deferred = self._agent.request(method, location, headers)

1 0

[ooni-probe/master] Add support for redacting IP address of user in response headers
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 6e43f2145d8bffb5e735c37c17b1d80ced463959 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Nov 23 16:10:13 2016 +0000 Add support for redacting IP address of user in response headers --- ooni/templates/httpt.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index e57463b..b9308b7 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -163,19 +163,25 @@ class HTTPTest(NetTestCase): 'response': None } if response: + if (getattr(response, 'request', None) and + getattr(response.request, 'absoluteURI', None)): + session['request']['url'] = response.request.absoluteURI + if self.localOptions.get('withoutbody', 0) is 0: response_body = representBody(response_body) else: response_body = '' + + response_headers = _representHeaders(response.headers) # Attempt to redact the IP address of the probe from the responses if (config.privacy.includeip is False and probe_ip.address is not None and (isinstance(response_body, str) or isinstance(response_body, unicode))): response_body = response_body.replace(probe_ip.address, "[REDACTED]") - if (getattr(response, 'request', None) and - getattr(response.request, 'absoluteURI', None)): - session['request']['url'] = response.request.absoluteURI + for key, value in response_headers.items(): + response_headers[key] = value.replace(probe_ip.address, + "[REDACTED]") session['response'] = { - 'headers': _representHeaders(response.headers), + 'headers': response_headers, 'body': response_body, 'code': response.code }

1 0

[ooni-probe/master] Hotfix/comparison (#678)
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 356759b6ac6ad78f680eb0f5223bfc4fb25f3f0e Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 15 20:08:44 2016 +0100 Hotfix/comparison (#678) * In python "is" != "==" I think this has already bitten me in the past grr. * bump version to dev0 --- ooni/__init__.py | 2 +- ooni/utils/log.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index c2d5fdf..385054e 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.1" +__version__ = "2.0.2.dev0" __all__ = [ 'agent', diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 353b8b2..42eb9ed 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -173,9 +173,9 @@ class OONILogger(object): if config.advanced.debug: stdout_log_level = levels['DEBUG'] - if config.basic.rotate is 'daily': + if config.basic.rotate == 'daily': logfile = DailyLogFile(log_filename, log_folder) - elif config.basic.rotate is 'length': + elif config.basic.rotate == 'length': logfile = LogFile(log_filename, log_folder, rotateLength=int(human_size_to_bytes( config.basic.rotate_length

1 0

[ooni-probe/master] Implement a basic test that tests for reachability of the whatsapp servers
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 3e9c0337b4142a884797784cfa9a94be423a2a09 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Jul 31 14:55:16 2016 +0200 Implement a basic test that tests for reachability of the whatsapp servers --- ooni/nettest.py | 4 + ooni/nettests/blocking/web_connectivity.py | 2 +- ooni/nettests/blocking/whatsapp.py | 401 +++++++++++++++++++++++++++++ 3 files changed, 406 insertions(+), 1 deletion(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 150858c..5ff5485 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -268,6 +268,10 @@ class NetTestLoader(object): parameter.pop() self.usageOptions.optParameters.append(parameter) + if getattr(test_class.usageOptions, 'optFlags', None): + for parameter in test_class.usageOptions.optFlags: + self.usageOptions.optFlags.append(parameter) + if getattr(test_class, 'inputFile', None): self.usageOptions.optParameters.append(test_class.inputFile) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 629ef1c..1d5cf1f 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -38,7 +38,7 @@ class UsageOptions(usage.Options): ] -class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): + class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity """ diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py new file mode 100644 index 0000000..ac4ed1c --- /dev/null +++ b/ooni/nettests/blocking/whatsapp.py @@ -0,0 +1,401 @@ +# -*- encoding: utf-8 -*- + +import random + +import ipaddr + +from twisted.internet import defer, reactor +from twisted.python import usage +from twisted.internet.endpoints import TCP4ClientEndpoint + +from ooni.utils import log +from ooni.common.http_utils import extractTitle +from ooni.common.tcp_utils import TCPConnectFactory +from ooni.errors import failureToString + +from ooni.templates import httpt, dnst + +# These are taken from https://www.whatsapp.com/cidr.txt +WHATSAPP_IPV4 = """\ +31.13.64.51/32 +31.13.65.49/32 +31.13.66.49/32 +31.13.67.51/32 +31.13.68.52/32 +31.13.69.240/32 +31.13.70.49/32 +31.13.71.49/32 +31.13.72.52/32 +31.13.73.49/32 +31.13.74.49/32 +31.13.75.52/32 +31.13.76.81/32 +31.13.77.49/32 +31.13.78.53/32 +31.13.79.195/32 +31.13.80.53/32 +31.13.81.53/32 +31.13.82.51/32 +31.13.83.51/32 +31.13.84.51/32 +31.13.85.51/32 +31.13.86.51/32 +31.13.87.51/32 +31.13.88.49/32 +31.13.90.51/32 +31.13.91.51/32 +31.13.92.52/32 +31.13.93.51/32 +31.13.94.52/32 +31.13.95.63/32 +50.22.198.204/30 +50.22.210.32/30 +50.22.210.128/27 +50.22.225.64/27 +50.22.235.248/30 +50.22.240.160/27 +50.23.90.128/27 +50.97.57.128/27 +75.126.39.32/27 +108.168.174.0/27 +108.168.176.192/26 +108.168.177.0/27 +108.168.180.96/27 +108.168.254.65/32 +108.168.255.224/32 +108.168.255.227/32 +157.240.0.53/32 +157.240.3.53/32 +158.85.0.96/27 +158.85.5.192/27 +158.85.46.128/27 +158.85.48.224/27 +158.85.58.0/25 +158.85.61.192/27 +158.85.224.160/27 +158.85.233.32/27 +158.85.249.128/27 +158.85.254.64/27 +169.44.36.0/25 +169.44.57.64/27 +169.44.58.64/27 +169.44.80.0/26 +169.44.82.96/27 +169.44.82.128/27 +169.44.82.192/26 +169.44.83.0/26 +169.44.83.96/27 +169.44.83.128/27 +169.44.83.192/26 +169.44.84.0/24 +169.44.85.64/27 +169.45.71.32/27 +169.45.71.96/27 +169.45.87.128/26 +169.45.169.192/27 +169.45.182.96/27 +169.45.210.64/27 +169.45.214.224/27 +169.45.219.224/27 +169.45.237.192/27 +169.45.238.32/27 +169.45.248.96/27 +169.45.248.160/27 +169.53.29.128/27 +169.53.48.32/27 +169.53.71.224/27 +169.53.250.128/26 +169.53.252.64/27 +169.53.255.64/27 +169.54.2.160/27 +169.54.44.224/27 +169.54.51.32/27 +169.54.55.192/27 +169.54.193.160/27 +169.54.210.0/27 +169.54.222.128/27 +169.55.69.128/26 +169.55.74.32/27 +169.55.126.64/26 +169.55.210.96/27 +169.55.235.160/27 +173.192.162.32/27 +173.192.219.128/27 +173.192.222.160/27 +173.192.231.32/27 +173.193.205.0/27 +173.193.230.96/27 +173.193.230.128/27 +173.193.230.192/27 +173.193.239.0/27 +174.36.208.128/27 +174.36.210.32/27 +174.36.251.192/27 +174.37.199.192/27 +174.37.217.64/27 +174.37.231.64/27 +174.37.243.64/27 +174.37.251.0/27 +179.60.192.51/32 +179.60.193.51/32 +179.60.195.51/32 +184.173.136.64/27 +184.173.147.32/27 +184.173.161.64/32 +184.173.161.160/27 +184.173.173.116/32 +184.173.179.32/27 +185.60.216.53/32 +185.60.218.53/32 +192.155.212.192/27 +198.11.193.182/31 +198.11.251.32/27 +198.23.80.0/27 +208.43.115.192/27 +208.43.117.79/32 +208.43.122.128/27""" + +WHATSAPP_IPV6 = """\ +2607:f0d0:1b01:d4::/64 +2607:f0d0:1b02:14d::/64 +2607:f0d0:1b04:32::/64 +2607:f0d0:1b04:bb::/64 +2607:f0d0:1b04:bc::/64 +2607:f0d0:1b06::/64 +2607:f0d0:1b06:4::/64 +2607:f0d0:2102:229::/64 +2607:f0d0:2601:37::/64 +2607:f0d0:3003:1bc::/64 +2607:f0d0:3004:136::/64 +2607:f0d0:3004:174::/64 +2607:f0d0:3005:183::/64 +2607:f0d0:3005:1a3::/64 +2607:f0d0:3006:84::/64 +2607:f0d0:3006:af::/64 +2607:f0d0:3801:38::/64 +2607:f0d0:3802:48::/64 +2a03:2880:f200:c5:face:b00c::167/128 +2a03:2880:f200:1c5:face:b00c::167/128 +2a03:2880:f201:c5:face:b00c::167/128 +2a03:2880:f202:c4:face:b00c::167/128 +2a03:2880:f203:c5:face:b00c::167/128 +2a03:2880:f204:c5:face:b00c::167/128 +2a03:2880:f205:c5:face:b00c::167/128 +2a03:2880:f206:c5:face:b00c::167/128 +2a03:2880:f207:c5:face:b00c::167/128 +2a03:2880:f208:c5:face:b00c::167/128 +2a03:2880:f209:c5:face:b00c::167/128 +2a03:2880:f20a:c5:face:b00c::167/128 +2a03:2880:f20b:c5:face:b00c::167/128 +2a03:2880:f20c:c6:face:b00c::167/128 +2a03:2880:f20d:c5:face:b00c::167/128 +2a03:2880:f20e:c5:face:b00c::167/128 +2a03:2880:f20f:c6:face:b00c::167/128 +2a03:2880:f210:c5:face:b00c::167/128 +2a03:2880:f211:c5:face:b00c::167/128 +2a03:2880:f212:c5:face:b00c::167/128 +2a03:2880:f213:c5:face:b00c::167/128 +2a03:2880:f213:80c5:face:b00c::167/128 +2a03:2880:f214:c5:face:b00c::167/128 +2a03:2880:f215:c5:face:b00c::167/128 +2a03:2880:f216:c5:face:b00c::167/128 +2a03:2880:f217:c5:face:b00c::167/128 +2a03:2880:f218:c3:face:b00c::167/128 +2a03:2880:f219:c5:face:b00c::167/128 +2a03:2880:f21a:c5:face:b00c::167/128 +2a03:2880:f21b:c5:face:b00c::167/128 +2a03:2880:f21c:c5:face:b00c::167/128 +2a03:2880:f21c:80c5:face:b00c::167/128 +2a03:2880:f21f:c5:face:b00c::167/128 +2a03:2880:f221:c5:face:b00c::167/128 +2a03:2880:f222:c5:face:b00c::167/128 +2a03:2880:f223:c5:face:b00c::167/128 +2a03:2880:f225:c4:face:b00c::167/128 +2a03:2880:f226:c6:face:b00c::167/128""" + +class DidNotConnect(Exception): + pass + +class WhatsAppNetwork(object): + def __init__(self): + self.ipv4_networks = [] + for ip in WHATSAPP_IPV4.split("\n"): + try: + self.ipv4_networks.append(ipaddr.IPv4Network(ip)) + except Exception as exc: + log.err("IP is wrong") + log.msg(ip) + self.ipv6_networks = map(ipaddr.IPv6Network, + WHATSAPP_IPV6.split("\n")) + + def contains(self, ip_address): + ip = ipaddr.IPAddress(ip_address) + if isinstance(ip, ipaddr.IPv4Address): + networks = self.ipv4_networks + elif isinstance(ip, ipaddr.IPv6Address): + networks = self.ipv6_networks + else: + raise RuntimeError("Should never happen") + for network in networks: + if network.Contains(ip): + return True + return False + +class UsageOptions(usage.Options): + optFlags = [ + ['all-endpoints', 'e', 'Should we attempt to connect to all whatsapp' + ' endpoints?'], + ] + +class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): + name = "Whatsapp" + description = ("This test checks to see if the servers used by whatsapp " + "messenger are reachable") + author = "Arturo Filastò" + version = "0.1.0" + + requiresRoot = False + requiresTor = False + followRedirects = True + usageOptions = UsageOptions + + def setUp(self): + self.report['registratison_server_failure'] = None + self.report['registration_server_status'] = None + self.report['whatsapp_web_failure'] = None + self.report['whatsapp_web_status'] = None + + self.report['whatsapp_endpoints_status'] = None + self.report['whatsapp_endpoints_dns_inconsistent'] = [] + self.report['whatsapp_endpoints_blocked'] = [] + + self.report['tcp_connect'] = [] + + @defer.inlineCallbacks + def test_registration_server(self): + url = 'https://v.whatsapp.net/v2/register' + # Ensure I get back: + # {"status": "fail", "reason": "missing_param", "param": "code"} + + try: + response = yield self.doRequest(url, 'GET') + except Exception as exc: + failure_string = failureToString(defer.failure.Failure(exc)) + log.err("Failed to contact the registration server %s" % failure_string) + self.report['registratison_server_failure'] = failure_string + self.report['registration_server_status'] = 'blocked' + defer.returnValue(None) + + log.msg("Successfully connected to registration server!") + self.report['registration_server_status'] = 'ok' + + @defer.inlineCallbacks + def _test_whatsapp_web(self, url): + try: + response = yield self.doRequest(url, 'GET') + except Exception as exc: + failure_string = failureToString(defer.failure.Failure(exc)) + log.err("Failed to connect to whatsapp web %s" % failure_string) + self.report['whatsapp_web_failure'] = failure_string + self.report['whatsapp_web_status'] = 'blocked' + defer.returnValue(None) + + title = extractTitle(response.body).strip() + if title != "WhatsApp Web": + self.report['whatsapp_web_status'] = 'blocked' + + @defer.inlineCallbacks + def test_whatsapp_web(self): + yield self._test_whatsapp_web('https://web.whatsapp.com/') + yield self._test_whatsapp_web('http://web.whatsapp.com/') + if self.report['whatsapp_web_status'] != 'blocked': + self.report['whatsapp_web_status'] = 'ok' + + def _test_connect_to_port(self, address, port): + result = { + 'ip': address, + 'port': port, + 'status': { + 'success': None, + 'failure': None + } + } + point = TCP4ClientEndpoint(reactor, address, port, timeout=10) + d = point.connect(TCPConnectFactory()) + @d.addCallback + def cb(p): + result['status']['success'] = True + result['status']['failure'] = False + self.report['tcp_connect'].append(result) + + @d.addErrback + def eb(failure): + result['status']['success'] = False + result['status']['failure'] = failureToString(failure) + self.report['tcp_connect'].append(result) + + @defer.inlineCallbacks + def _test_connect(self, address): + possible_ports = [443, 5222] + + connected = False + for port in possible_ports: + try: + yield self._test_connect_to_port(address, port) + connected = False + except Exception as exc: + pass + + if connected == False: + raise DidNotConnect() + + @defer.inlineCallbacks + def _test_endpoint(self, hostname, whatsapp_network): + log.msg("Testing %s" % hostname) + addresses = yield self.performALookup(hostname) + consistent = True + for address in addresses[:]: + try: + is_in_whats_app_network = whatsapp_network.contains(address) + except ValueError: + # This happens when it's not an IP + addresses.remove(address) + continue + if not is_in_whats_app_network: + self.report['whatsapp_endpoints_status'] = 'blocked' + consistent = False + + if consistent == False: + log.msg("%s presents an inconsistent DNS response" % hostname) + self.report['whatsapp_endpoints_dns_inconsistent'].append(hostname) + defer.returnValue(None) + + dl = [] + for address in addresses: + dl.append(self._test_connect(address)) + results = yield defer.DeferredList(dl, consumeErrors=True) + + tcp_blocked = False + for success, result in results: + if success == False: + tcp_blocked = True + + if tcp_blocked == True: + log.msg("%s is blocked based on TCP") + self.report['whatsapp_endpoints_blocked'].append(hostname) + self.report['whatsapp_endpoints_status'] = 'blocked' + else: + self.report['whatsapp_endpoints_status'] = 'ok' + + + @defer.inlineCallbacks + def test_endpoints(self): + possible_endpoints = map(lambda x: "e%s.whatsapp.net" % x, range(1, 16)) + whatsapp_network = WhatsAppNetwork() + to_test_endpoints = [] + if self.localOptions['all-endpoints']: + to_test_endpoints += possible_endpoints + else: + to_test_endpoints += [random.choice(possible_endpoints)] + for endpoint in to_test_endpoints: + yield self._test_endpoint(endpoint, whatsapp_network)

1 0

[ooni-probe/master] Wait for tor to start on decks that use a tor based collector (#684)
by art＠torproject.org 13 Jan '17

13 Jan '17

commit b29e6b51aa37643aa06ced7e70164d101de87f32 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 29 19:10:55 2016 +0000 Wait for tor to start on decks that use a tor based collector (#684) --- ooni/deck/deck.py | 4 ++++ ooni/ui/web/server.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 59946ab..50887b5 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -150,6 +150,10 @@ class NGDeck(object): if self.metadata.get('no_collector', False): self.no_collector = True + if (self.no_collector is False and + self.bouncer.backend_type == "onion"): + self.requires_tor = True + @property def tasks(self): return self._tasks diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index bb4e45f..40c03e2 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -529,7 +529,6 @@ class WebUIAPI(object): @requires_true(attrs=['_is_initialized']) @defer.inlineCallbacks def api_measurement_summary(self, request, measurement_id): - log.warn("SUMMARY") try: measurement = get_measurement(measurement_id) except InsecurePath:

1 0

[ooni-probe/master] Fix some bugs in the whatsapp test
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 3373100a0cd7a6ba1f9000c34f11a924836dd2d6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Aug 3 19:53:21 2016 +0200 Fix some bugs in the whatsapp test thanks to @anadahz for finding them --- ooni/nettests/blocking/whatsapp.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py index ac4ed1c..5707cfa 100644 --- a/ooni/nettests/blocking/whatsapp.py +++ b/ooni/nettests/blocking/whatsapp.py @@ -334,6 +334,8 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): result['status']['failure'] = failureToString(failure) self.report['tcp_connect'].append(result) + return d + @defer.inlineCallbacks def _test_connect(self, address): possible_ports = [443, 5222] @@ -342,12 +344,12 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): for port in possible_ports: try: yield self._test_connect_to_port(address, port) - connected = False + connected = True except Exception as exc: pass if connected == False: - raise DidNotConnect() + raise DidNotConnect @defer.inlineCallbacks def _test_endpoint(self, hostname, whatsapp_network): @@ -381,10 +383,11 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): tcp_blocked = True if tcp_blocked == True: - log.msg("%s is blocked based on TCP") + log.msg("%s is blocked based on TCP" % hostname) self.report['whatsapp_endpoints_blocked'].append(hostname) self.report['whatsapp_endpoints_status'] = 'blocked' else: + log.msg("No blocking detected via TCP on %s" % hostname) self.report['whatsapp_endpoints_status'] = 'ok' @@ -393,7 +396,7 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): possible_endpoints = map(lambda x: "e%s.whatsapp.net" % x, range(1, 16)) whatsapp_network = WhatsAppNetwork() to_test_endpoints = [] - if self.localOptions['all-endpoints']: + if self.localOptions.get('all-endpoints', False): to_test_endpoints += possible_endpoints else: to_test_endpoints += [random.choice(possible_endpoints)]

1 0

[ooni-probe/master] Test all TCP endpoints even if DNS is inconsistent
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 65d79e6291a08d125f467e74bec3293aa07b1a8c Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 12 13:29:48 2016 +0200 Test all TCP endpoints even if DNS is inconsistent * Make the dns consistency check a bit more lax --- ooni/nettests/blocking/whatsapp.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py index 5707cfa..9b68f30 100644 --- a/ooni/nettests/blocking/whatsapp.py +++ b/ooni/nettests/blocking/whatsapp.py @@ -222,7 +222,7 @@ class WhatsAppNetwork(object): for ip in WHATSAPP_IPV4.split("\n"): try: self.ipv4_networks.append(ipaddr.IPv4Network(ip)) - except Exception as exc: + except Exception: log.err("IP is wrong") log.msg(ip) self.ipv6_networks = map(ipaddr.IPv6Network, @@ -252,7 +252,7 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): description = ("This test checks to see if the servers used by whatsapp " "messenger are reachable") author = "Arturo Filastò" - version = "0.1.0" + version = "0.2.0" requiresRoot = False requiresTor = False @@ -278,7 +278,7 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): # {"status": "fail", "reason": "missing_param", "param": "code"} try: - response = yield self.doRequest(url, 'GET') + yield self.doRequest(url, 'GET') except Exception as exc: failure_string = failureToString(defer.failure.Failure(exc)) log.err("Failed to contact the registration server %s" % failure_string) @@ -345,7 +345,7 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): try: yield self._test_connect_to_port(address, port) connected = True - except Exception as exc: + except Exception: pass if connected == False: @@ -355,22 +355,20 @@ class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): def _test_endpoint(self, hostname, whatsapp_network): log.msg("Testing %s" % hostname) addresses = yield self.performALookup(hostname) - consistent = True + ip_in_whats_app_network = False + for address in addresses[:]: try: - is_in_whats_app_network = whatsapp_network.contains(address) + ip_in_whats_app_network |= whatsapp_network.contains(address) except ValueError: # This happens when it's not an IP addresses.remove(address) continue - if not is_in_whats_app_network: - self.report['whatsapp_endpoints_status'] = 'blocked' - consistent = False - if consistent == False: + if ip_in_whats_app_network == False: log.msg("%s presents an inconsistent DNS response" % hostname) + self.report['whatsapp_endpoints_status'] = 'blocked' self.report['whatsapp_endpoints_dns_inconsistent'].append(hostname) - defer.returnValue(None) dl = [] for address in addresses:

1 0