[tor-commits] [ooni-probe/master] Implement a basic test that tests for reachability of the whatsapp servers

art at torproject.org art at torproject.org
Fri Jan 13 12:39:57 UTC 2017


commit 3e9c0337b4142a884797784cfa9a94be423a2a09
Author: Arturo Filastò <arturo at filasto.net>
Date:   Sun Jul 31 14:55:16 2016 +0200

    Implement a basic test that tests for reachability of the whatsapp servers
---
 ooni/nettest.py                            |   4 +
 ooni/nettests/blocking/web_connectivity.py |   2 +-
 ooni/nettests/blocking/whatsapp.py         | 401 +++++++++++++++++++++++++++++
 3 files changed, 406 insertions(+), 1 deletion(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 150858c..5ff5485 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -268,6 +268,10 @@ class NetTestLoader(object):
                     parameter.pop()
                 self.usageOptions.optParameters.append(parameter)
 
+        if getattr(test_class.usageOptions, 'optFlags', None):
+            for parameter in test_class.usageOptions.optFlags:
+                self.usageOptions.optFlags.append(parameter)
+
         if getattr(test_class, 'inputFile', None):
             self.usageOptions.optParameters.append(test_class.inputFile)
 
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 629ef1c..1d5cf1f 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -38,7 +38,7 @@ class UsageOptions(usage.Options):
     ]
 
 
-class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
+    class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     """
     Web connectivity
     """
diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py
new file mode 100644
index 0000000..ac4ed1c
--- /dev/null
+++ b/ooni/nettests/blocking/whatsapp.py
@@ -0,0 +1,401 @@
+# -*- encoding: utf-8 -*-
+
+import random
+
+import ipaddr
+
+from twisted.internet import defer, reactor
+from twisted.python import usage
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from ooni.utils import log
+from ooni.common.http_utils import extractTitle
+from ooni.common.tcp_utils import TCPConnectFactory
+from ooni.errors import failureToString
+
+from ooni.templates import httpt, dnst
+
+# These are taken from https://www.whatsapp.com/cidr.txt
+WHATSAPP_IPV4 = """\
+31.13.64.51/32
+31.13.65.49/32
+31.13.66.49/32
+31.13.67.51/32
+31.13.68.52/32
+31.13.69.240/32
+31.13.70.49/32
+31.13.71.49/32
+31.13.72.52/32
+31.13.73.49/32
+31.13.74.49/32
+31.13.75.52/32
+31.13.76.81/32
+31.13.77.49/32
+31.13.78.53/32
+31.13.79.195/32
+31.13.80.53/32
+31.13.81.53/32
+31.13.82.51/32
+31.13.83.51/32
+31.13.84.51/32
+31.13.85.51/32
+31.13.86.51/32
+31.13.87.51/32
+31.13.88.49/32
+31.13.90.51/32
+31.13.91.51/32
+31.13.92.52/32
+31.13.93.51/32
+31.13.94.52/32
+31.13.95.63/32
+50.22.198.204/30
+50.22.210.32/30
+50.22.210.128/27
+50.22.225.64/27
+50.22.235.248/30
+50.22.240.160/27
+50.23.90.128/27
+50.97.57.128/27
+75.126.39.32/27
+108.168.174.0/27
+108.168.176.192/26
+108.168.177.0/27
+108.168.180.96/27
+108.168.254.65/32
+108.168.255.224/32
+108.168.255.227/32
+157.240.0.53/32
+157.240.3.53/32
+158.85.0.96/27
+158.85.5.192/27
+158.85.46.128/27
+158.85.48.224/27
+158.85.58.0/25
+158.85.61.192/27
+158.85.224.160/27
+158.85.233.32/27
+158.85.249.128/27
+158.85.254.64/27
+169.44.36.0/25
+169.44.57.64/27
+169.44.58.64/27
+169.44.80.0/26
+169.44.82.96/27
+169.44.82.128/27
+169.44.82.192/26
+169.44.83.0/26
+169.44.83.96/27
+169.44.83.128/27
+169.44.83.192/26
+169.44.84.0/24
+169.44.85.64/27
+169.45.71.32/27
+169.45.71.96/27
+169.45.87.128/26
+169.45.169.192/27
+169.45.182.96/27
+169.45.210.64/27
+169.45.214.224/27
+169.45.219.224/27
+169.45.237.192/27
+169.45.238.32/27
+169.45.248.96/27
+169.45.248.160/27
+169.53.29.128/27
+169.53.48.32/27
+169.53.71.224/27
+169.53.250.128/26
+169.53.252.64/27
+169.53.255.64/27
+169.54.2.160/27
+169.54.44.224/27
+169.54.51.32/27
+169.54.55.192/27
+169.54.193.160/27
+169.54.210.0/27
+169.54.222.128/27
+169.55.69.128/26
+169.55.74.32/27
+169.55.126.64/26
+169.55.210.96/27
+169.55.235.160/27
+173.192.162.32/27
+173.192.219.128/27
+173.192.222.160/27
+173.192.231.32/27
+173.193.205.0/27
+173.193.230.96/27
+173.193.230.128/27
+173.193.230.192/27
+173.193.239.0/27
+174.36.208.128/27
+174.36.210.32/27
+174.36.251.192/27
+174.37.199.192/27
+174.37.217.64/27
+174.37.231.64/27
+174.37.243.64/27
+174.37.251.0/27
+179.60.192.51/32
+179.60.193.51/32
+179.60.195.51/32
+184.173.136.64/27
+184.173.147.32/27
+184.173.161.64/32
+184.173.161.160/27
+184.173.173.116/32
+184.173.179.32/27
+185.60.216.53/32
+185.60.218.53/32
+192.155.212.192/27
+198.11.193.182/31
+198.11.251.32/27
+198.23.80.0/27
+208.43.115.192/27
+208.43.117.79/32
+208.43.122.128/27"""
+
+WHATSAPP_IPV6 = """\
+2607:f0d0:1b01:d4::/64
+2607:f0d0:1b02:14d::/64
+2607:f0d0:1b04:32::/64
+2607:f0d0:1b04:bb::/64
+2607:f0d0:1b04:bc::/64
+2607:f0d0:1b06::/64
+2607:f0d0:1b06:4::/64
+2607:f0d0:2102:229::/64
+2607:f0d0:2601:37::/64
+2607:f0d0:3003:1bc::/64
+2607:f0d0:3004:136::/64
+2607:f0d0:3004:174::/64
+2607:f0d0:3005:183::/64
+2607:f0d0:3005:1a3::/64
+2607:f0d0:3006:84::/64
+2607:f0d0:3006:af::/64
+2607:f0d0:3801:38::/64
+2607:f0d0:3802:48::/64
+2a03:2880:f200:c5:face:b00c::167/128
+2a03:2880:f200:1c5:face:b00c::167/128
+2a03:2880:f201:c5:face:b00c::167/128
+2a03:2880:f202:c4:face:b00c::167/128
+2a03:2880:f203:c5:face:b00c::167/128
+2a03:2880:f204:c5:face:b00c::167/128
+2a03:2880:f205:c5:face:b00c::167/128
+2a03:2880:f206:c5:face:b00c::167/128
+2a03:2880:f207:c5:face:b00c::167/128
+2a03:2880:f208:c5:face:b00c::167/128
+2a03:2880:f209:c5:face:b00c::167/128
+2a03:2880:f20a:c5:face:b00c::167/128
+2a03:2880:f20b:c5:face:b00c::167/128
+2a03:2880:f20c:c6:face:b00c::167/128
+2a03:2880:f20d:c5:face:b00c::167/128
+2a03:2880:f20e:c5:face:b00c::167/128
+2a03:2880:f20f:c6:face:b00c::167/128
+2a03:2880:f210:c5:face:b00c::167/128
+2a03:2880:f211:c5:face:b00c::167/128
+2a03:2880:f212:c5:face:b00c::167/128
+2a03:2880:f213:c5:face:b00c::167/128
+2a03:2880:f213:80c5:face:b00c::167/128
+2a03:2880:f214:c5:face:b00c::167/128
+2a03:2880:f215:c5:face:b00c::167/128
+2a03:2880:f216:c5:face:b00c::167/128
+2a03:2880:f217:c5:face:b00c::167/128
+2a03:2880:f218:c3:face:b00c::167/128
+2a03:2880:f219:c5:face:b00c::167/128
+2a03:2880:f21a:c5:face:b00c::167/128
+2a03:2880:f21b:c5:face:b00c::167/128
+2a03:2880:f21c:c5:face:b00c::167/128
+2a03:2880:f21c:80c5:face:b00c::167/128
+2a03:2880:f21f:c5:face:b00c::167/128
+2a03:2880:f221:c5:face:b00c::167/128
+2a03:2880:f222:c5:face:b00c::167/128
+2a03:2880:f223:c5:face:b00c::167/128
+2a03:2880:f225:c4:face:b00c::167/128
+2a03:2880:f226:c6:face:b00c::167/128"""
+
+class DidNotConnect(Exception):
+    pass
+
+class WhatsAppNetwork(object):
+    def __init__(self):
+        self.ipv4_networks = []
+        for ip in WHATSAPP_IPV4.split("\n"):
+            try:
+                self.ipv4_networks.append(ipaddr.IPv4Network(ip))
+            except Exception as exc:
+                log.err("IP is wrong")
+                log.msg(ip)
+        self.ipv6_networks = map(ipaddr.IPv6Network,
+                                 WHATSAPP_IPV6.split("\n"))
+
+    def contains(self, ip_address):
+        ip = ipaddr.IPAddress(ip_address)
+        if isinstance(ip, ipaddr.IPv4Address):
+            networks = self.ipv4_networks
+        elif isinstance(ip, ipaddr.IPv6Address):
+            networks = self.ipv6_networks
+        else:
+            raise RuntimeError("Should never happen")
+        for network in networks:
+            if network.Contains(ip):
+                return True
+        return False
+
+class UsageOptions(usage.Options):
+    optFlags = [
+        ['all-endpoints', 'e', 'Should we attempt to connect to all whatsapp'
+                               ' endpoints?'],
+    ]
+
+class WhatsappTest(httpt.HTTPTest, dnst.DNSTest):
+    name = "Whatsapp"
+    description = ("This test checks to see if the servers used by whatsapp "
+                   "messenger are reachable")
+    author = "Arturo Filastò"
+    version = "0.1.0"
+
+    requiresRoot = False
+    requiresTor = False
+    followRedirects = True
+    usageOptions = UsageOptions
+
+    def setUp(self):
+        self.report['registratison_server_failure'] = None
+        self.report['registration_server_status'] = None
+        self.report['whatsapp_web_failure'] = None
+        self.report['whatsapp_web_status'] = None
+
+        self.report['whatsapp_endpoints_status'] = None
+        self.report['whatsapp_endpoints_dns_inconsistent'] = []
+        self.report['whatsapp_endpoints_blocked'] = []
+
+        self.report['tcp_connect'] = []
+
+    @defer.inlineCallbacks
+    def test_registration_server(self):
+        url = 'https://v.whatsapp.net/v2/register'
+        # Ensure I get back:
+        # {"status": "fail", "reason": "missing_param", "param": "code"}
+
+        try:
+            response = yield self.doRequest(url, 'GET')
+        except Exception as exc:
+            failure_string = failureToString(defer.failure.Failure(exc))
+            log.err("Failed to contact the registration server %s" % failure_string)
+            self.report['registratison_server_failure'] = failure_string
+            self.report['registration_server_status'] = 'blocked'
+            defer.returnValue(None)
+
+        log.msg("Successfully connected to registration server!")
+        self.report['registration_server_status'] = 'ok'
+
+    @defer.inlineCallbacks
+    def _test_whatsapp_web(self, url):
+        try:
+            response = yield self.doRequest(url, 'GET')
+        except Exception as exc:
+            failure_string = failureToString(defer.failure.Failure(exc))
+            log.err("Failed to connect to whatsapp web %s" % failure_string)
+            self.report['whatsapp_web_failure'] = failure_string
+            self.report['whatsapp_web_status'] = 'blocked'
+            defer.returnValue(None)
+
+        title = extractTitle(response.body).strip()
+        if title != "WhatsApp Web":
+            self.report['whatsapp_web_status'] = 'blocked'
+
+    @defer.inlineCallbacks
+    def test_whatsapp_web(self):
+        yield self._test_whatsapp_web('https://web.whatsapp.com/')
+        yield self._test_whatsapp_web('http://web.whatsapp.com/')
+        if self.report['whatsapp_web_status'] != 'blocked':
+            self.report['whatsapp_web_status'] = 'ok'
+
+    def _test_connect_to_port(self, address, port):
+        result = {
+            'ip': address,
+            'port': port,
+            'status': {
+                'success': None,
+                'failure': None
+            }
+        }
+        point = TCP4ClientEndpoint(reactor, address, port, timeout=10)
+        d = point.connect(TCPConnectFactory())
+        @d.addCallback
+        def cb(p):
+            result['status']['success'] = True
+            result['status']['failure'] = False
+            self.report['tcp_connect'].append(result)
+
+        @d.addErrback
+        def eb(failure):
+            result['status']['success'] = False
+            result['status']['failure'] = failureToString(failure)
+            self.report['tcp_connect'].append(result)
+
+    @defer.inlineCallbacks
+    def _test_connect(self, address):
+        possible_ports = [443, 5222]
+
+        connected = False
+        for port in possible_ports:
+            try:
+                yield self._test_connect_to_port(address, port)
+                connected = False
+            except Exception as exc:
+                pass
+
+        if connected == False:
+            raise DidNotConnect()
+
+    @defer.inlineCallbacks
+    def _test_endpoint(self, hostname, whatsapp_network):
+        log.msg("Testing %s" % hostname)
+        addresses = yield self.performALookup(hostname)
+        consistent = True
+        for address in addresses[:]:
+            try:
+                is_in_whats_app_network = whatsapp_network.contains(address)
+            except ValueError:
+                # This happens when it's not an IP
+                addresses.remove(address)
+                continue
+            if not is_in_whats_app_network:
+                self.report['whatsapp_endpoints_status'] = 'blocked'
+                consistent = False
+
+        if consistent == False:
+            log.msg("%s presents an inconsistent DNS response" % hostname)
+            self.report['whatsapp_endpoints_dns_inconsistent'].append(hostname)
+            defer.returnValue(None)
+
+        dl = []
+        for address in addresses:
+            dl.append(self._test_connect(address))
+        results = yield defer.DeferredList(dl, consumeErrors=True)
+
+        tcp_blocked = False
+        for success, result in results:
+            if success == False:
+                tcp_blocked = True
+
+        if tcp_blocked == True:
+            log.msg("%s is blocked based on TCP")
+            self.report['whatsapp_endpoints_blocked'].append(hostname)
+            self.report['whatsapp_endpoints_status'] = 'blocked'
+        else:
+            self.report['whatsapp_endpoints_status'] = 'ok'
+
+
+    @defer.inlineCallbacks
+    def test_endpoints(self):
+        possible_endpoints = map(lambda x: "e%s.whatsapp.net" % x, range(1, 16))
+        whatsapp_network = WhatsAppNetwork()
+        to_test_endpoints = []
+        if self.localOptions['all-endpoints']:
+            to_test_endpoints += possible_endpoints
+        else:
+            to_test_endpoints += [random.choice(possible_endpoints)]
+        for endpoint in to_test_endpoints:
+            yield self._test_endpoint(endpoint, whatsapp_network)





More information about the tor-commits mailing list