[tor-commits] [bridgedb/develop] Implement moat distributor and server.

commit 3667a63027d6d02c87ceae03a22771ee03a56e2a Author: Isis Lovecruft isis@torproject.org Date: Thu Sep 14 17:13:24 2017 +0000

Implement moat distributor and server.

* ADD a new distributor, moat, which speaks to Tor Launcher over a JSON API interface. * ADD new `bridgedb.distributors` package. * ADD new `bridgedb.distributors.common` package for code which is shared by multiple distributors. * FIXES #22871: https://bugs.torproject.org/22871 --- .coveragerc | 1 + README.rst | 10 +- bridgedb.conf | 104 ++- bridgedb/captcha.py | 1 - bridgedb/configure.py | 12 +- bridgedb/distributors/__init__.py | 3 - bridgedb/distributors/common/__init__.py | 2 + bridgedb/distributors/common/http.py | 80 +++ bridgedb/distributors/email/__init__.py | 18 +- bridgedb/distributors/email/distributor.py | 3 + bridgedb/distributors/https/__init__.py | 6 +- bridgedb/distributors/https/distributor.py | 11 + bridgedb/distributors/https/server.py | 63 +- bridgedb/distributors/moat/__init__.py | 4 + bridgedb/distributors/moat/distributor.py | 64 ++ bridgedb/distributors/moat/request.py | 133 ++++ bridgedb/distributors/moat/server.py | 776 ++++++++++++++++++++ bridgedb/main.py | 59 +- bridgedb/test/https_helpers.py | 5 +- bridgedb/test/moat_helpers.py | 110 +++ bridgedb/test/test_distributors_common_http.py | 100 +++ bridgedb/test/test_distributors_moat_request.py | 52 ++ bridgedb/test/test_distributors_moat_server.py | 918 ++++++++++++++++++++++++ bridgedb/test/test_https_server.py | 70 -- bridgedb/test/test_main.py | 65 +- scripts/setup-tests | 7 + setup.py | 2 + 27 files changed, 2470 insertions(+), 209 deletions(-)

diff --git a/.coveragerc b/.coveragerc index 737310b..32d0c83 100644 --- a/.coveragerc +++ b/.coveragerc @@ -24,6 +24,7 @@ exclude_lines = if options[.verbosity.] def __repr__ if __name__ == .__main__.: + except Exception as impossible: # Ignore source code which cannot be found: ignore_errors = True # Exit with status code 2 if under this percentage is covered: diff --git a/README.rst b/README.rst index a2d3b27..3dbe647 100644 --- a/README.rst +++ b/README.rst @@ -444,7 +444,7 @@ The client MUST send a ``POST /meek/moat/fetch`` containing the following JSON:: { 'data': { 'version': '0.1.0', - 'type': 'moat client supported transports', + 'type': 'client-transports', 'supported': [ 'TRANSPORT', 'TRANSPORT', ... ], } } @@ -471,7 +471,7 @@ server will respond with the list of transports which is *does* support:: { 'data': { 'version': '0.1.0', - 'type': 'moat server supported transports', + 'type': 'moat-transports', 'supported': [ 'TRANSPORT', 'TRANSPORT', ... ], } } @@ -483,7 +483,7 @@ following JSON containing a CAPTCHA challenge:: { 'data': { 'id': 1, - 'type': 'moat farfetchd challenge', + 'type': 'moat-challenge', 'version': '0.1.0', 'transport': TRANSPORT, 'image': CAPTCHA, @@ -513,7 +513,7 @@ To propose a solution to a CAPTCHA, the client MUST send a request for ``POST { 'data': { 'id': 2, - 'type': 'moat farfetchd solution', + 'type': 'moat-solution', 'version': '0.1.0', 'transport': TRANSPORT, 'challenge': CHALLENGE, @@ -547,7 +547,7 @@ server responds ``200 OK`` with the following JSON:: { 'data': { 'id': 3, - 'type': 'moat bridges', + 'type': 'moat-bridges', 'version': '0.1.0', 'bridges': [ 'BRIDGE_LINE', ... ], 'qrcode': QRCODE, diff --git a/bridgedb.conf b/bridgedb.conf index 25a9cbf..d0366cf 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -150,12 +150,6 @@ STATUS_FILE = "networkstatus-bridges" # IGNORE_NETWORKSTATUS = True

-# Certificate file and private key for the HTTPS Distributor. To create a -# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files -# in your current directory. -HTTPS_CERT_FILE="cert" -HTTPS_KEY_FILE="privkey.pem" - #---------------- # Output Files \ Where to store created data #------------------------------------------------------------------------------ @@ -294,15 +288,104 @@ SUPPORTED_TRANSPORTS = { DEFAULT_TRANSPORT = 'obfs4'

#------------------------------- +# Moat Distribution Options \ +#------------------------------------------------------------------------------ +# +# These options configure the behaviour of a web interface which speaks JSON API +# to a remote application in order to present said application with the +# necessary information for creating a user interface for bridge distribution +# mechanism, similar to the web interface of BridgeDB's HTTPS Distributor. If +# MOAT_DIST is enabled, make sure that the MOAT_CERT_FILE and MOAT_KEY_FILE +# options point to the correct location of your SSL certificate and key! +# ------------------------------------------------------------------------------ + +# (boolean) True to enable distribution via Moat; False otherwise. +MOAT_DIST = True + +# (boolean) True to only allow Moat distribution via a Meek tunnel. False to +# only allow Moat distribution via untunneled HTTP(S). +MOAT_DIST_VIA_MEEK_ONLY = True + +# Certificate file and private key for the Moar Distributor. To create a +# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files +# in your current directory. +MOAT_TLS_CERT_FILE="moat-tls.crt" +MOAT_TLS_KEY_FILE="moat-tls.pem" + +# (string) The Fully-Qualified Domain Name (FQDN) of the server that the Moat +# and/or HTTPS distributor(s) is/are publicly reachable at. +if MOAT_DIST_VIA_MEEK_ONLY: + MOAT_SERVER_PUBLIC_ROOT = '/meek/moat' +else: + MOAT_SERVER_PUBLIC_ROOT = '/moat' + +# How many bridges do we give back in an answer (either HTTP or HTTPS)? +MOAT_BRIDGES_PER_ANSWER = 3 + +# (list) An ordered list of the preferred transports which moat should +# distribute, in order from most preferable to least preferable. +MOAT_TRANSPORT_PREFERENCE_LIST = ["obfs4", "vanilla"] + +# (string or None) The IP address where we listen for HTTPS connections. If +# ``None``, listen on the default interface. +MOAT_HTTPS_IP = '127.0.0.1' + +# (integer or None) The port to listen on for incoming HTTPS connections. +MOAT_HTTPS_PORT = 6791 + +# (string or None) The IP address to listen on for unencrypted HTTP +# connections. Set to ``None`` to disable unencrypted connections to the web +# interface. +MOAT_HTTP_IP = None + +# (integer or None) The port to listen on for incoming HTTP connections. +MOAT_HTTP_PORT = None + +# If true, there is a trusted proxy relaying incoming messages to us: take +# the *last* entry from its X-Forwarded-For header as the client's IP. +MOAT_USE_IP_FROM_FORWARDED_HEADER = True + +# How many clusters do we group IPs in when distributing bridges based on IP? +# Note that if PROXY_LIST_FILES is set (below), what we actually do here +# is use one higher than the number here, and the extra cluster is used +# for answering requests made by IP addresses in the PROXY_LIST_FILES file. +MOAT_N_IP_CLUSTERS = 4 + +# (string or None) The period at which the available bridges rotates to a +# separate set of bridges. This setting can be used in the form +# +# "COUNT PERIOD" where +# COUNT is an integer +# PERIOD is one of "second", "minute", "hour", "day", +# "week", or "month" (or any plural form). +# +# For example, setting HTTPS_ROTATION_PERIOD = "3 days" will result in the set +# of bridges which are available through the web interface (either HTTP or +# HTTPS) getting rotated once every three days. Setting this to None disables +# rotation entirely. +MOAT_ROTATION_PERIOD = "3 hours" + +# The location of the files which store the HMAC secret key and RSA keypair +# (for checking captcha responses): +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key' +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key' + +#------------------------------- # HTTP(S) Distribution Options \ #------------------------------------------------------------------------------ # # These options configure the behaviour of the web interface bridge -# distribution mechanism. If HTTPS_DIST is enabled, make sure that the above +# distribution mechanism. If HTTPS_DIST is enabled, make sure that the # HTTPS_CERT_FILE and HTTPS_KEY_FILE options point to the correct location of # your SSL certificate and key! #------------------------------------------------------------------------------

+# Certificate file and private key for the HTTPS Distributor. To create a +# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files +# in your current directory. +HTTPS_CERT_FILE="cert" +HTTPS_KEY_FILE="privkey.pem" + # (string) The Fully-Qualified Domain Name (FQDN) of the server that the HTTP # and/or HTTPS distributor(s) is/are publicly reachable at. SERVER_PUBLIC_FQDN = 'bridges.torproject.org' @@ -585,14 +668,17 @@ EMAIL_GPG_PASSPHRASE_FILE = None # Once a bridge is assigned to either of the first two groups, it stays there # persistently. The bridges are allocated to these groups in a proportion of # -# ``HTTPS_SHARE`` : ``EMAIL_SHARE`` : ``RESERVED_SHARE`` +# ``MOAT_SHARE`` : ``HTTPS_SHARE`` : ``EMAIL_SHARE`` : ``RESERVED_SHARE`` # ------------------------------------------------------------------------------

+# The proportion of bridges to allocate to Moat distribution. +MOAT_SHARE = 20 + # The proportion of bridges to allocate to HTTP distribution. HTTPS_SHARE = 10

# The proportion of bridges to allocate to Email distribution. -EMAIL_SHARE = 5 +EMAIL_SHARE = 2

# An integer specifying the proportion of bridges which should remain # unallocated, for backup usage and manual distribution. diff --git a/bridgedb/captcha.py b/bridgedb/captcha.py index ad89aea..b66972c 100644 --- a/bridgedb/captcha.py +++ b/bridgedb/captcha.py @@ -290,7 +290,6 @@ class GimpCaptcha(Captcha): if hmacIsValid: try: answerBlob = secretKey.decrypt(encBlob) - timestamp = answerBlob[:12].lstrip('0') then = cls.sched.nextIntervalStarts(int(timestamp)) now = int(time.time()) diff --git a/bridgedb/configure.py b/bridgedb/configure.py index 1a5e949..0bc4dd1 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -110,6 +110,7 @@ def loadConfig(configFile=None, configCls=None):

for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", + "MOAT_CERT_FILE", "MOAT_KEY_FILE", "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", @@ -120,11 +121,18 @@ def loadConfig(configFile=None, configCls=None): else: setattr(config, attr, os.path.abspath(os.path.expanduser(setting)))

- for attr in ["HTTPS_ROTATION_PERIOD", "EMAIL_ROTATION_PERIOD"]: + for attr in ["MOAT_ROTATION_PERIOD", + "HTTPS_ROTATION_PERIOD", + "EMAIL_ROTATION_PERIOD"]: setting = getattr(config, attr, None) # Default to None setattr(config, attr, setting)

- for attr in ["IGNORE_NETWORKSTATUS", "CSP_ENABLED", "CSP_REPORT_ONLY", + for attr in ["IGNORE_NETWORKSTATUS", + "MOAT_CSP_ENABLED", + "MOAT_CSP_REPORT_ONLY", + "MOAT_CSP_INCLUDE_SELF", + "CSP_ENABLED", + "CSP_REPORT_ONLY", "CSP_INCLUDE_SELF"]: setting = getattr(config, attr, True) # Default to True setattr(config, attr, setting) diff --git a/bridgedb/distributors/__init__.py b/bridgedb/distributors/__init__.py index 951ef44..cc5f7a0 100644 --- a/bridgedb/distributors/__init__.py +++ b/bridgedb/distributors/__init__.py @@ -1,5 +1,2 @@ """Methods for distributing bridges."""

-import email -import https -#import moat diff --git a/bridgedb/distributors/common/__init__.py b/bridgedb/distributors/common/__init__.py new file mode 100644 index 0000000..fac0768 --- /dev/null +++ b/bridgedb/distributors/common/__init__.py @@ -0,0 +1,2 @@ +"""Common resources and utilities for bridge distribution systems.""" + diff --git a/bridgedb/distributors/common/http.py b/bridgedb/distributors/common/http.py new file mode 100644 index 0000000..86c26b8 --- /dev/null +++ b/bridgedb/distributors/common/http.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_distributors_common_http -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see included AUTHORS file +# :copyright: (c) 2017, The Tor Project, Inc. +# (c) 2017, Isis Lovecruft +# :license: see LICENSE for licensing information + +""" +.. py:module:: bridgedb.distributors.common.http + :synopsis: Common utilities for HTTP-based distributors. + +bridgedb.distributors.common.http +================================== + +Common utilities for HTTP-based distributors. +""" + +import logging +import os + +from bridgedb.parse.addr import isIPAddress + + +#: The fully-qualified domain name for any and all web servers we run. +SERVER_PUBLIC_FQDN = None + + +def setFQDN(fqdn, https=True): + """Set the global :data:`SERVER_PUBLIC_FQDN` variable. + + :param str fqdn: The public, fully-qualified domain name of the HTTP + server that will serve this resource. + :param bool https: If ``True``, then ``'https://%27%60%60 will be prepended to + the FQDN. This is primarily used to create a + ``Content-Security-Policy`` header that will only allow resources to + be sourced via HTTPS, otherwise, if ``False``, it allow resources to + be sourced via any transport protocol. + """ + if https: + fqdn = 'https://' + fqdn + + logging.info("Setting HTTP server public FQDN to %r" % fqdn) + + global SERVER_PUBLIC_FQDN + SERVER_PUBLIC_FQDN = fqdn + +def getFQDN(): + """Get the setting for the HTTP server's public FQDN from the global + :data:`SERVER_PUBLIC_FQDN variable. + + :rtype: str or None + """ + return SERVER_PUBLIC_FQDN + +def getClientIP(request, useForwardedHeader=False): + """Get the client's IP address from the ``'X-Forwarded-For:'`` + header, or from the :api:`request <twisted.web.server.Request>`. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. + :param bool useForwardedHeader: If ``True``, attempt to get the client's + IP address from the ``'X-Forwarded-For:'`` header. + :rtype: ``None`` or :any:`str` + :returns: The client's IP address, if it was obtainable. + """ + ip = None + + if useForwardedHeader: + header = request.getHeader("X-Forwarded-For") + if header: + ip = header.split(",")[-1].strip() + if not isIPAddress(ip): + logging.warn("Got weird X-Forwarded-For value %r" % header) + ip = None + else: + ip = request.getClientIP() + + return ip diff --git a/bridgedb/distributors/email/__init__.py b/bridgedb/distributors/email/__init__.py index 8f9ed03..5c0ef50 100644 --- a/bridgedb/distributors/email/__init__.py +++ b/bridgedb/distributors/email/__init__.py @@ -1,4 +1,20 @@ -"""Servers for BridgeDB's email bridge distributor.""" +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# please also see AUTHORS file +# :copyright: (c) 2012-2017 Isis Lovecruft +# (c) 2007-2017, The Tor Project, Inc. +# (c) 2007-2017, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +'''Package containing modules for BridgeDB's email bridge distributor. + +.. py:module:: bridgedb.distributors.email + :synopsis: Package containing modules for BridgeDB's email bridge + distributor. +'''

import autoresponder import distributor diff --git a/bridgedb/distributors/email/distributor.py b/bridgedb/distributors/email/distributor.py index 06a0c50..b76c26c 100644 --- a/bridgedb/distributors/email/distributor.py +++ b/bridgedb/distributors/email/distributor.py @@ -228,3 +228,6 @@ class EmailDistributor(Distributor): # Since prepopulateRings is called every half hour when the bridge # descriptors are re-parsed, we should clean the database then. self.cleanDatabase() + + logging.info("Bridges allotted for %s distribution: %d" + % (self.name, len(self.hashring))) diff --git a/bridgedb/distributors/https/__init__.py b/bridgedb/distributors/https/__init__.py index 6c51a5f..7da8173 100644 --- a/bridgedb/distributors/https/__init__.py +++ b/bridgedb/distributors/https/__init__.py @@ -1,5 +1,5 @@ """Servers for BridgeDB's HTTPS bridge distributor."""

-import distributor -import request -import server +#import distributor +#import request +#import server diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py index 791229b..5ff9d83 100644 --- a/bridgedb/distributors/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -262,6 +262,17 @@ class HTTPSDistributor(Distributor): self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges)

+ logging.info("Bridges allotted for %s distribution: %d" + % (self.name, len(self.hashring))) + + logging.info("\tNum bridges:\tFilter set:") + for (ringname, (filterFn, subring)) in self.hashring.filterRings.items(): + filterSet = ' '.join(self.hashring.extractFilterNames(ringname)) + logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) + + logging.info("Total subrings for %s: %d" + % (self.name, len(self.hashring.filterRings))) + def insert(self, bridge): """Assign a bridge to this distributor.""" self.hashring.insert(bridge) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index b756659..352a838 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -50,6 +50,9 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha +from bridgedb.distributors.common.http import setFQDN +from bridgedb.distributors.common.http import getFQDN +from bridgedb.distributors.common.http import getClientIP from bridgedb.distributors.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress @@ -60,8 +63,9 @@ from bridgedb.schedule import ScheduledInterval from bridgedb.util import replaceControlChars

+#: The path to the HTTPS distributor's web templates. (Should be the +#: "templates" directory in the same directory as this file.) TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') -rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku')

# Setting `filesystem_checks` to False is recommended for production servers, # due to potential speed increases. This means that the atimes of the Mako @@ -76,61 +80,9 @@ lookup = TemplateLookup(directories=[TEMPLATE_DIR], collection_size=500) logging.debug("Set template root to %s" % TEMPLATE_DIR)

-#: This server's public, fully-qualified domain name. -SERVER_PUBLIC_FQDN = None - - -def setFQDN(fqdn, https=True): - """Set the global :data:`SERVER_PUBLIC_FQDN` variable. - - :param str fqdn: The public, fully-qualified domain name of the HTTP - server that will serve this resource. - :param bool https: If ``True``, then ``'https://%27%60%60 will be prepended to - the FQDN. This is primarily used to create a - ``Content-Security-Policy`` header that will only allow resources to - be sourced via HTTPS, otherwise, if ``False``, it allow resources to - be sourced via any transport protocol. - """ - if https: - fqdn = 'https://' + fqdn - - logging.info("Setting HTTP server public FQDN to %r" % fqdn) - - global SERVER_PUBLIC_FQDN - SERVER_PUBLIC_FQDN = fqdn - -def getFQDN(): - """Get the setting for the HTTP server's public FQDN from the global - :data:`SERVER_PUBLIC_FQDN variable. - - :rtype: str or None - """ - return SERVER_PUBLIC_FQDN - -def getClientIP(request, useForwardedHeader=False): - """Get the client's IP address from the ``'X-Forwarded-For:'`` - header, or from the :api:`request <twisted.web.server.Request>`. - - :type request: :api:`twisted.web.http.Request` - :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. - :param bool useForwardedHeader: If ``True``, attempt to get the client's - IP address from the ``'X-Forwarded-For:'`` header. - :rtype: ``None`` or :any:`str` - :returns: The client's IP address, if it was obtainable. - """ - ip = None - - if useForwardedHeader: - header = request.getHeader("X-Forwarded-For") - if header: - ip = header.split(",")[-1].strip() - if not isIPAddress(ip): - logging.warn("Got weird X-Forwarded-For value %r" % header) - ip = None - else: - ip = request.getClientIP() +#: Localisations which BridgeDB supports which should be rendered right-to-left. +rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku')

- return ip

def replaceErrorPage(request, error, template_name=None, html=True): """Create a general error page for displaying in place of tracebacks. @@ -334,6 +286,7 @@ class ErrorResource(CSPResource):

render_POST = render_GET

+ resource404 = ErrorResource('error-404.html', code=404) resource500 = ErrorResource('error-500.html', code=500) maintenance = ErrorResource('error-503.html', code=503) diff --git a/bridgedb/distributors/moat/__init__.py b/bridgedb/distributors/moat/__init__.py new file mode 100644 index 0000000..221663b --- /dev/null +++ b/bridgedb/distributors/moat/__init__.py @@ -0,0 +1,4 @@ +"""Servers for BridgeDB's Moat bridge distributor.""" + +#import distributor +#import server diff --git a/bridgedb/distributors/moat/distributor.py b/bridgedb/distributors/moat/distributor.py new file mode 100644 index 0000000..a8f0e1d --- /dev/null +++ b/bridgedb/distributors/moat/distributor.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_moat_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# Matthew Finkel 0x017DD169EA793BE2 sysrqb@torproject.org +# :copyright: (c) 2013-2017, Isis Lovecruft +# (c) 2013-2017, Matthew Finkel +# (c) 2007-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +""" +bridgedb.distributors.moat.distributor +========================== + +A Distributor that hands out bridges through a web interface. + +.. inheritance-diagram:: MoatDistributor + :parts: 1 +""" + +from bridgedb.distributors.https.distributor import HTTPSDistributor + +class MoatDistributor(HTTPSDistributor): + """A bridge distributor for Moat, a system which uses a JSON API to + provide a remote application with data necessary to the creation of a + user interface for distributing bridges. + + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. + :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. + """ + + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ + super(MoatDistributor, self).__init__(totalSubrings, key, proxies, + answerParameters) diff --git a/bridgedb/distributors/moat/request.py b/bridgedb/distributors/moat/request.py new file mode 100644 index 0000000..71ae3cd --- /dev/null +++ b/bridgedb/distributors/moat/request.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8; test-case-name: bridgedb.test.test_distributors_moat_request; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft isis@torproject.org 0xA3ADB67A2CDB8B35 +# :copyright: (c) 2007-2017, The Tor Project, Inc. +# (c) 2013-2017, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +""" +.. py:module:: bridgedb.distributors.moat.request + :synopsis: Classes for parsing and storing information about requests for + bridges which are sent to the moat distributor. + +bridgedb.distributors.moat.request +================================== + +Classes for parsing and storing information about requests for bridges +which are sent to the moat distributor. + +.. inheritance-diagram:: MoatBridgeRequest + +:: + + bridgedb.distributors.moat.request + | + |_ MoatBridgeRequest - A request for bridges which was received through + the moat distributor. + +.. +""" + +from __future__ import print_function + +import ipaddr +import logging +import re + +from bridgedb import bridgerequest +from bridgedb import geo +from bridgedb.parse import addr + + +#: A regular expression for matching the Pluggable Transport methodname in +#: HTTP GET request parameters. +TRANSPORT_REGEXP = "[_a-zA-Z][_a-zA-Z0-9]*" +TRANSPORT_PATTERN = re.compile(TRANSPORT_REGEXP) + +UNBLOCKED_REGEXP = "[a-zA-Z]{2}" +UNBLOCKED_PATTERN = re.compile(UNBLOCKED_REGEXP) + + +class MoatBridgeRequest(bridgerequest.BridgeRequestBase): + """We received a request for bridges through the moat distributor.""" + + def __init__(self, addClientCountryCode=False): + """Process a new bridge request received through the + :class:`~bridgedb.distributors.moat.distributor.MoatDistributor`. + + :param bool addClientCountryCode: If ``True``, then calling + :meth:`withoutBlockInCountry` will attempt to add the client's own + country code, geolocated from their IP, to the ``notBlockedIn`` + countries list. + """ + super(MoatBridgeRequest, self).__init__() + self.addClientCountryCode = addClientCountryCode + + def withIPversion(self): + """Determine if the request **parameters** were for bridges with IPv6 + addresses or not. + + .. note:: If the client's forwarded IP address was IPv6, then we assume + the client wanted IPv6 bridges. + """ + if addr.isIPAddress(self.client): + if self.client.version == 6: + logging.info("Moat request for bridges with IPv6 addresses.") + self.withIPv6() + + def withoutBlockInCountry(self, data): + """Determine which countries the bridges for this **request** should + not be blocked in. + + If :data:`addClientCountryCode` is ``True``, the the client's own + geolocated country code will be added to the to the + :data:`notBlockedIn` list. + + :param dict data: The decoded data from the JSON API request. + """ + countryCodes = data.get("unblocked", list()) + + for countryCode in countryCodes: + try: + country = UNBLOCKED_PATTERN.match(countryCode).group() + except (TypeError, AttributeError): + pass + else: + if country: + self.notBlockedIn.append(country.lower()) + logging.info("Moat request for bridges not blocked in: %r" + % country) + + if self.addClientCountryCode: + # Look up the country code of the input IP, and request bridges + # not blocked in that country. + if addr.isIPAddress(self.client): + country = geo.getCountryCode(ipaddr.IPAddress(self.client)) + if country: + self.notBlockedIn.append(country.lower()) + logging.info( + ("Moat client's bridges also shouldn't be blocked " + "in their GeoIP country code: %s") % country) + + def withPluggableTransportType(self, data): + """This request included a specific Pluggable Transport identifier. + + Add any Pluggable Transport methodname found in the JSON API + request field named "transport". + + :param dict data: The decoded data from the JSON API request. + """ + methodname = type('')(data.get("transport", "")) + + try: + transport = TRANSPORT_PATTERN.match(methodname).group() + except (TypeError, AttributeError): + pass + else: + if transport: + self.transports.append(transport) + logging.info("Moat request for transport type: %r" % transport) diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py new file mode 100644 index 0000000..daf62fa --- /dev/null +++ b/bridgedb/distributors/moat/server.py @@ -0,0 +1,776 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_distributors_moat_server -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see included AUTHORS file +# :copyright: (c) 2017, The Tor Project, Inc. +# (c) 2017, Isis Lovecruft +# :license: see LICENSE for licensing information + +""" +.. py:module:: bridgedb.distributors.moat.server + :synopsis: Server which implements JSON API to interface with Tor Browser + clients through a meek tunnel. + +bridgedb.distributors.moat.server +================================= + +Server which implements JSON API to interface with Tor Browser clients through a +meek tunnel. + +.. inheritance-diagram:: JsonAPIResource JsonAPIErrorResource CustomErrorHandlingResource JsonAPIDataResource CaptchaResource CaptchaCheckResource CaptchaFetchResource + :parts: 1 +""" + +from __future__ import print_function + +import base64 +import json +import logging +import time + +from functools import partial + +from ipaddr import IPAddress + +from twisted.internet import reactor +from twisted.internet.error import CannotListenError +from twisted.web import resource +from twisted.web.server import Site + +from bridgedb import captcha +from bridgedb import crypto +from bridgedb.distributors.common.http import setFQDN +from bridgedb.distributors.common.http import getFQDN +from bridgedb.distributors.common.http import getClientIP +from bridgedb.distributors.moat.request import MoatBridgeRequest +from bridgedb.qrcodes import generateQR +from bridgedb.schedule import Unscheduled +from bridgedb.schedule import ScheduledInterval +from bridgedb.util import replaceControlChars + + +#: The current version of the moat JSON API that we speak +MOAT_API_VERSION = '0.1.0' + +#: The root path to resources for the moat server +SERVER_PUBLIC_ROOT = None + +#: An ordered list of the preferred transports which moat should +#: distribute, in order from most preferable to least preferable. +TRANSPORT_PREFERENCE_LIST = None + +#: All of the pluggable transports BridgeDB currently supports. +SUPPORTED_TRANSPORTS = None + + +def getFQDNAndRoot(): + """Get the server's public FQDN plus the root directory for the web server. + """ + root = getRoot() + fqdn = getFQDN() + + if not root.startswith('/') and not fqdn.endswith('/'): + return '/'.join([fqdn, root]) + else: + return ''.join([fqdn, root]) + +def setRoot(root): + """Set the global :data:`SERVER_PUBLIC_ROOT` variable. + + :param str root: The path to the root directory for the web server. + """ + logging.info("Setting Moat server public root to %r" % root) + + global SERVER_PUBLIC_ROOT + SERVER_PUBLIC_ROOT = root + +def getRoot(): + """Get the setting for the HTTP server's public FQDN from the global + :data:`SERVER_PUBLIC_FQDN variable. + + :rtype: str or None + """ + return SERVER_PUBLIC_ROOT + +def setPreferredTransports(preferences): + """Set the global ``TRANSPORT_PREFERENCE_LIST``.""" + global TRANSPORT_PREFERENCE_LIST + TRANSPORT_PREFERENCE_LIST = preferences + +def getPreferredTransports(): + """Get the global ``TRANSPORT_PREFERENCE_LIST``. + + :rtype: list + :returns: A list of preferences for which pluggable transports to distribute + to moat clients. + """ + return TRANSPORT_PREFERENCE_LIST + +def setSupportedTransports(transports): + """Set the global ``SUPPORTED_TRANSPORTS``. + + :param dist transports: The ``SUPPORTED_TRANSPORTS`` dict from a + bridgedb.conf file. + """ + supported = [k for (k, v) in transports.items() if v] + + if not "vanilla" in supported: + supported.append("vanilla") + + global SUPPORTED_TRANSPORTS + SUPPORTED_TRANSPORTS = supported + +def getSupportedTransports(): + """Get the global ``SUPPORTED_TRANSPORTS``. + + :rtype: list + :returns: A list all pluggable transports we support. + """ + return SUPPORTED_TRANSPORTS + + +class JsonAPIResource(resource.Resource): + """A resource which conforms to the `JSON API spec http://jsonapi.org/`__. + """ + + def __init__(self, useForwardedHeader=True): + resource.Resource.__init__(self) + self.useForwardedHeader = useForwardedHeader + + def getClientIP(self, request): + """Get the client's IP address from the ``'X-Forwarded-For:'`` + header, or from the :api:`request <twisted.web.server.Request>`. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a + :api:`twisted.web.resource.Resource`. + :rtype: ``None`` or :any:`str` + :returns: The client's IP address, if it was obtainable. + """ + return getClientIP(request, self.useForwardedHeader) + + def formatDataForResponse(self, data, request): + """Format a dictionary of ``data`` into JSON and add necessary response + headers. + + This method will set the appropriate response headers: + * `Content-Type: application/vnd.api+json` + * `Server: moat/VERSION` + + :type data: dict + :param data: Some data to respond with. This will be formatted as JSON. + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. + :returns: The encoded data. + """ + request.responseHeaders.addRawHeader(b"Content-Type", b"application/vnd.api+json") + request.responseHeaders.addRawHeader(b"Server", b"moat/%s" % MOAT_API_VERSION) + + if data: + rendered = json.dumps(data) + else: + rendered = b"" + + return rendered + + +class JsonAPIErrorResource(JsonAPIResource): + """A JSON API resource which explains that some error has occured.""" + + isLeaf = True + + def __init__(self, id=0, type="", code=200, status="OK", detail=""): + """Create a :api:`twisted.web.resource.Resource` for a JSON API errors + object. + """ + resource.Resource.__init__(self) + self.id = id + self.type = type + self.code = code + self.status = status + self.detail = detail + + def render_GET(self, request): + # status codes and messages are at the JSON API layer, not HTTP layer: + data = { + 'errors': [{ + 'id': self.id, + 'type': self.type, + 'version': MOAT_API_VERSION, + 'code': self.code, + 'status': self.status, + 'detail': self.detail, + }] + } + return self.formatDataForResponse(data, request) + + render_POST = render_GET + + +resource403 = JsonAPIErrorResource(code=403, status="Forbidden") +resource406 = JsonAPIErrorResource(code=406, status="Not Acceptable") +resource415 = JsonAPIErrorResource(code=415, status="Unsupported Media Type") +resource419 = JsonAPIErrorResource(code=419, status="No You're A Teapot") +resource501 = JsonAPIErrorResource(code=501, status="Not Implemented") + + +class CustomErrorHandlingResource(resource.Resource): + """A :api:`twisted.web.resource.Resource` which wraps the + :api:`twisted.web.resource.Resource.getChild` method in order to use + custom error handling pages. + """ + def getChild(self, path, request): + logging.debug("[501] %s %s" % (request.method, request.uri)) + + response = resource501 + response.detail = "moat version %s does not implement %s %s" % \ + (MOAT_API_VERSION, request.method, request.uri) + return response + + +class JsonAPIDataResource(JsonAPIResource): + """A resource which returns some JSON API data.""" + + def __init__(self, useForwardedHeader=True): + JsonAPIResource.__init__(self, useForwardedHeader) + + def checkRequestHeaders(self, request): + """The JSON API specification requires servers to respond with certain HTTP + status codes and message if the client's request headers are inappropriate in + any of the following ways: + + * Servers MUST respond with a 415 Unsupported Media Type status code if + a request specifies the header Content-Type: application/vnd.api+json + with any media type parameters. + + * Servers MUST respond with a 406 Not Acceptable status code if a + request’s Accept header contains the JSON API media type and all + instances of that media type are modified with media type parameters. + """ + supports_json_api = False + accept_json_api_header = False + accept_header_is_ok = False + + if request.requestHeaders.hasHeader("Content-Type"): + headers = request.requestHeaders.getRawHeaders("Content-Type") + # The "pragma: no cover"s are because, no matter what I do, I cannot + # for the life of me trick twisted's test infrastructure to not send + # some variant of these headers. ¯_(ツ)_/¯ + for contentType in headers: # pragma: no cover + # The request must have the Content-Type set to 'application/vnd.api+json': + if contentType == 'application/vnd.api+json': + supports_json_api = True + # The request must not specify a Content-Type with media parameters: + if ';' in contentType: + supports_json_api = False + + if not supports_json_api: + return resource415 + + # If the request has an Accept header which contains + # 'application/vnd.api+json' then at least one instance of that type + # must have no parameters: + if request.requestHeaders.hasHeader("Accept"): # pragma: no cover + headers = request.requestHeaders.getRawHeaders("Accept") + for accept in headers: + if accept.startswith('application/vnd.api+json'): + accept_json_api_header = True + if ';' not in accept: + accept_header_is_ok = True + + if accept_json_api_header and not accept_header_is_ok: # pragma: no cover + return resource406 + + +class CaptchaResource(JsonAPIDataResource): + """A CAPTCHA.""" + + def __init__(self, hmacKey=None, publicKey=None, secretKey=None, + useForwardedHeader=True): + JsonAPIDataResource.__init__(self, useForwardedHeader) + self.hmacKey = hmacKey + self.publicKey = publicKey + self.secretKey = secretKey + + +class CaptchaFetchResource(CaptchaResource): + """A resource to retrieve a CAPTCHA challenge.""" + + isLeaf = True + + def __init__(self, hmacKey=None, publicKey=None, secretKey=None, + captchaDir="captchas", useForwardedHeader=True): + """DOCDOC + + :param bytes hmacKey: The master HMAC key, used for validating CAPTCHA + challenge strings in :meth:`captcha.GimpCaptcha.check`. The file + where this key is stored can be set via the + ``GIMP_CAPTCHA_HMAC_KEYFILE`` option in the config file. + are stored. See the ``GIMP_CAPTCHA_DIR`` config setting. + :param str secretkey: A PKCS#1 OAEP-padded, private RSA key, used for + verifying the client's solution to the CAPTCHA. See + :func:`bridgedb.crypto.getRSAKey` and the + ``GIMP_CAPTCHA_RSA_KEYFILE`` config setting. + :param str publickey: A PKCS#1 OAEP-padded, public RSA key, used for + creating the ``captcha_challenge_field`` string to give to a + client. + :param str captchaDir: The directory where the cached CAPTCHA images + :param bool useForwardedHeader: If ``True``, obtain the client's IP + address from the ``X-Forwarded-For`` HTTP header. + """ + CaptchaResource.__init__(self, hmacKey, publicKey, secretKey, + useForwardedHeader) + self.captchaDir = captchaDir + self.supportedTransports = getSupportedTransports() + + def getCaptchaImage(self, request): + """Get a random CAPTCHA image from our **captchaDir**. + + Creates a :class:`~bridgedb.captcha.GimpCaptcha`, and calls its + :meth:`~bridgedb.captcha.GimpCaptcha.get` method to return a random + CAPTCHA and challenge string. + + :type request: :api:`twisted.web.http.Request` + :param request: A client's initial request for some other resource + which is protected by this one (i.e. protected by a CAPTCHA). + :returns: A 2-tuple of ``(image, challenge)``, where:: + - ``image`` is a string holding a binary, JPEG-encoded image. + - ``challenge`` is a unique string associated with the request. + """ + # Create a new HMAC key, specific to requests from this client: + clientIP = self.getClientIP(request) + clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) + capt = captcha.GimpCaptcha(self.publicKey, self.secretKey, + clientHMACKey, self.captchaDir) + try: + capt.get() + except captcha.GimpCaptchaError as error: + logging.debug(error) + except Exception as impossible: + logging.error("Unhandled error while retrieving Gimp captcha!") + logging.error(impossible) + + return (capt.image, capt.challenge) + + def getPreferredTransports(self, supportedTransports): + """Choose which transport a client should request, based on their list + of ``supportedTransports``. + + :param list supportedTransports: A list of transports the client + reported that they support (as returned from + :meth:`~bridgedb.distributors.moat.server.CaptchaFetchResource.extractSupportedTransports`). + :rtype: str or list + :returns: A string specifying the chosen transport, provided there is an + overlap between which transports BridgeDB and the client support. + Otherwise, if there is no overlap, returns a list of all the + transports which BridgeDB *does* support. + """ + preferenceOrder = getPreferredTransports() + preferred = None + + for pt in preferenceOrder: + if pt in supportedTransports: + preferred = pt + + # If we couldn't pick the best one that we both support, return the + # whole list of what we're able to distribute: + if not preferred: + preferred = getSupportedTransports() + + return preferred + + def extractSupportedTransports(self, request): + """Extract the transports a client supports from their POST request. + + :param str request: A JSON blob containing the following + fields: + * "version": The moat protocol version. + * "type": "client-transports". + * "supported": ['TRANSPORT', … ] + where: + * TRANSPORT is a string identifying a transport, e.g. "obfs3" or + "obfs4". Currently supported transport identifiers are: + "vanilla", "fte", "obfs3", "obfs4", "scramblesuit". + :rtype: list + :returns: The list of transports the client supports. + """ + supported = [] + + try: + encoded_data = request.content.read() + data = json.loads(encoded_data)["data"][0] + + if data["type"] != "client-transports": + raise ValueError( + "Bad JSON API object type: expected %s got %s" % + ('client-transports', data["type"])) + elif data["version"] != MOAT_API_VERSION: + raise ValueError( + "Client requested protocol version %s, but we're using %s" % + (data["version"], MOAT_API_VERSION)) + elif not data["supported"]: + raise ValueError( + "Client didn't provide any supported transports") + else: + supported = data["supported"] + except KeyError as err: + logging.debug(("Error processing client POST request: Client JSON " + "API data missing '%s' field") % (err)) + except ValueError as err: + logging.warn("Error processing client POST request: %s" % err) + except Exception as impossible: + logging.error("Unhandled error while extracting moat client transports!") + logging.error(impossible) + + return supported + + def render_POST(self, request): + """Retrieve a captcha from the moat API server and serve it to the client. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object for a CAPTCHA. + :rtype: str + :returns: A JSON blob containing the following fields: + * "version": The moat protocol version. + * "image": A base64-encoded CAPTCHA JPEG image. + * "challenge": A base64-encoded, encrypted challenge. The client + will need to hold on to the and pass it back later, along with + their challenge response. + * "error": An ASCII error message. + Any of the above JSON fields may be "null". + """ + error = self.checkRequestHeaders(request) + + if error: # pragma: no cover + return error.render(request) + + supported = self.extractSupportedTransports(request) + preferred = self.getPreferredTransports(supported) + image, challenge = self.getCaptchaImage(request) + + data = { + 'data': [{ + 'id': 1, + 'type': 'moat-challenge', + 'version': MOAT_API_VERSION, + 'transport': preferred, + 'image': image, + 'challenge': challenge, # The challenge is already base64-encoded + }] + } + + try: + data["data"][0]["image"] = base64.b64encode(image) + except Exception as impossible: + logging.error("Could not construct or encode captcha!") + logging.error(impossible) + + return self.formatDataForResponse(data, request) + + +class CaptchaCheckResource(CaptchaResource): + """A resource to verify a CAPTCHA solution and distribute bridges.""" + + isLeaf = True + + def __init__(self, distributor, schedule, N=1, + hmacKey=None, publicKey=None, secretKey=None, + useForwardedHeader=True): + """Create a new resource for checking CAPTCHA solutions and returning + bridges to a client. + + :type distributor: :class:`MoatDistributor` + :param distributor: The mechanism to retrieve bridges for this + distributor. + :type schedule: :class:`~bridgedb.schedule.ScheduledInterval` + :param schedule: The time period used to tweak the bridge selection + procedure. + :param int N: The number of bridges to hand out per query. + :param bool useForwardedHeader: Whether or not we should use the the + X-Forwarded-For header instead of the source IP address. + """ + CaptchaResource.__init__(self, hmacKey, publicKey, secretKey, + useForwardedHeader) + self.distributor = distributor + self.schedule = schedule + self.nBridgesToGive = N + self.useForwardedHeader = useForwardedHeader + + def getBridgeLines(self, ip, data): + """Get bridge lines for a client's HTTP request. + + :param str ip: The client's IP address. + :param dict data: The decoded JSON API data from the client's request. + :rtype: list or None + :returns: A list of bridge lines. + """ + bridgeLines = None + interval = self.schedule.intervalStart(time.time()) + + logging.debug("Replying to JSON API request from %s." % ip) + + if ip and data: + bridgeRequest = MoatBridgeRequest() + bridgeRequest.client = IPAddress(ip) + bridgeRequest.isValid(True) + bridgeRequest.withIPversion() + bridgeRequest.withPluggableTransportType(data) + bridgeRequest.withoutBlockInCountry(data) + bridgeRequest.generateFilters() + + bridges = self.distributor.getBridges(bridgeRequest, interval) + bridgeLines = [replaceControlChars(bridge.getBridgeLine(bridgeRequest)) + for bridge in bridges] + + return bridgeLines + + def extractClientSolution(self, data): + """Extract the client's CAPTCHA solution from a POST request. + + This is used after receiving a POST request from a client (which + should contain their solution to the CAPTCHA), to extract the solution + and challenge strings. + + :param dict data: The decoded JSON API data from the client's request. + :returns: A redirect for a request for a new CAPTCHA if there was a + problem. Otherwise, returns a 2-tuple of strings, the first is the + client's CAPTCHA solution from the text input area, and the second + is the challenge string. + """ + qrcode = False + transport = None + challenge, solution = None, None + + try: + if data["type"] != "moat-solution": + raise ValueError( + "Bad JSON API object type: expected %s got %s" % + ("moat-solution", data["type"])) + elif data["id"] != 2: + raise ValueError( + "Bad JSON API data id: expected 2 got %s" % + (data["id"])) + elif data["version"] != MOAT_API_VERSION: + raise ValueError( + "Client requested protocol version %s, but we're using %s" % + (data["version"], MOAT_API_VERSION)) + elif data["transport"] not in getSupportedTransports(): + raise ValueError( + "Transport '%s' is not currently supported" % + data["transport"]) + else: + qrcode = data["qrcode"] + transport = type('')(data["transport"]) + challenge = type('')(data["challenge"]) + solution = type('')(data["solution"]) + except KeyError as err: + logging.warn(("Error processing client POST request: " + "Client JSON API data missing '%s' field.") % err) + except ValueError as err: + logging.warn("Error processing client POST request: %s" % err.message) + except Exception as impossible: + logging.error(impossible) + + return (qrcode, transport, challenge, solution) + + def checkSolution(self, challenge, solution, clientIP): + """Process a solved CAPTCHA via + :meth:`bridgedb.captcha.GimpCaptcha.check`. + + :param str challenge: A base64-encoded, encrypted challenge. + :param str solution: The client's solution to the captcha + :param str clientIP: The client's IP address. + :rtupe: bool + :returns: True, if the CAPTCHA solution was valid; False otherwise. + """ + valid = False + clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) + + try: + valid = captcha.GimpCaptcha.check(challenge, solution, + self.secretKey, clientHMACKey) + except Exception as impossible: + logging.error(impossible) + raise impossible + finally: + logging.debug("%sorrect captcha from %r: %r." % + ("C" if valid else "Inc", clientIP, solution)) + + return valid + + def failureResponse(self, id, request): + """Respond with status code "419 No You're A Teapot".""" + error_response = resource419 + error_response.type = 'moat-bridges' + + if id == 4: + error_response.id = 4 + error_response.detail = "The CAPTCHA solution was incorrect." + elif id == 5: + error_response.id = 5 + error_response.detail = "The CAPTCHA challenge timed out." + + return error_response.render(request) + + def render_POST(self, request): + """Process a client's CAPTCHA solution. + + If the client's CAPTCHA solution is valid (according to + :meth:`checkSolution`), process and serve their original + request. Otherwise, redirect them back to a new CAPTCHA page. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object, including POST arguments which + should include two key/value pairs: one key being + ``'captcha_challenge_field'``, and the other, + ``'captcha_response_field'``. These POST arguments should be + obtained from :meth:`render_GET`. + :rtype: str + :returns: A rendered HTML page containing a ReCaptcha challenge image + for the client to solve. + """ + valid = False + error = self.checkRequestHeaders(request) + + if error: # pragma: no cover + return error.render(request) + + data = { + "data": [{ + "id": 3, + "type": 'moat-bridges', + "version": MOAT_API_VERSION, + "bridges": None, + "qrcode": None, + }] + } + + try: + encoded_client_data = request.content.read() + client_data = json.loads(encoded_client_data)["data"][0] + clientIP = self.getClientIP(request) + + (include_qrcode, transport, + challenge, solution) = self.extractClientSolution(client_data) + + valid = self.checkSolution(challenge, solution, clientIP) + except captcha.CaptchaExpired: + logging.debug("The challenge had timed out") + return self.failureResponse(5, request) + except Exception as impossible: + logging.warn("Unhandled exception while processing a POST /fetch request!") + logging.error(impossible) + return self.failureResponse(4, request) + + if valid: + qrcode = None + bridgeLines = self.getBridgeLines(clientIP, client_data) + + if include_qrcode: + qrjpeg = generateQR(bridgeLines) + if qrjpeg: + qrcode = 'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + + data["data"][0]["qrcode"] = qrcode + data["data"][0]["bridges"] = bridgeLines + + return self.formatDataForResponse(data, request) + else: + return self.failureResponse(4, request) + + +def addMoatServer(config, distributor): + """Set up a web server for moat bridge distribution. + + :type config: :class:`bridgedb.persistent.Conf` + :param config: A configuration object from + :mod:`bridgedb.main`. Currently, we use these options:: + GIMP_CAPTCHA_DIR + SERVER_PUBLIC_FQDN + SUPPORTED_TRANSPORTS + MOAT_DIST + MOAT_DIST_VIA_MEEK_ONLY + MOAT_TLS_CERT_FILE + MOAT_TLS_KEY_FILE + MOAT_SERVER_PUBLIC_ROOT + MOAT_HTTPS_IP + MOAT_HTTPS_PORT + MOAT_HTTP_IP + MOAT_HTTP_PORT + MOAT_BRIDGES_PER_ANSWER + MOAT_TRANSPORT_PREFERENCE_LIST + MOAT_USE_IP_FROM_FORWARDED_HEADER + MOAT_ROTATION_PERIOD + MOAT_GIMP_CAPTCHA_HMAC_KEYFILE + MOAT_GIMP_CAPTCHA_RSA_KEYFILE + :type distributor: :class:`bridgedb.distributors.moat.distributor.MoatDistributor` + :param distributor: A bridge distributor. + :raises SystemExit: if the servers cannot be started. + :rtype: :api:`twisted.web.server.Site` + :returns: A webserver. + """ + captcha = None + fwdHeaders = config.MOAT_USE_IP_FROM_FORWARDED_HEADER + numBridges = config.MOAT_BRIDGES_PER_ANSWER + + logging.info("Starting moat servers...") + + setFQDN(config.SERVER_PUBLIC_FQDN) + setRoot(config.MOAT_SERVER_PUBLIC_ROOT) + setSupportedTransports(config.SUPPORTED_TRANSPORTS) + setPreferredTransports(config.MOAT_TRANSPORT_PREFERENCE_LIST) + + # Get the master HMAC secret key for CAPTCHA challenges, and then + # create a new HMAC key from it for use on the server. + captchaKey = crypto.getKey(config.MOAT_GIMP_CAPTCHA_HMAC_KEYFILE) + hmacKey = crypto.getHMAC(captchaKey, "Moat-Captcha-Key") + # Load or create our encryption keys: + secretKey, publicKey = crypto.getRSAKey(config.MOAT_GIMP_CAPTCHA_RSA_KEYFILE) + sched = Unscheduled() + + if config.MOAT_ROTATION_PERIOD: + count, period = config.MOAT_ROTATION_PERIOD.split() + sched = ScheduledInterval(count, period) + + sitePublicDir = getRoot() + + meek = CustomErrorHandlingResource() + moat = CustomErrorHandlingResource() + fetch = CaptchaFetchResource(hmacKey, publicKey, secretKey, + config.GIMP_CAPTCHA_DIR, fwdHeaders) + check = CaptchaCheckResource(distributor, sched, numBridges, + hmacKey, publicKey, secretKey, fwdHeaders) + + moat.putChild("fetch", fetch) + moat.putChild("check", check) + meek.putChild("moat", moat) + + root = CustomErrorHandlingResource() + root.putChild("meek", meek) + + site = Site(root) + site.displayTracebacks = False + + if config.MOAT_HTTP_PORT: # pragma: no cover + ip = config.MOAT_HTTP_IP or "" + port = config.MOAT_HTTP_PORT or 80 + try: + reactor.listenTCP(port, site, interface=ip) + except CannotListenError as error: + raise SystemExit(error) + logging.info("Started Moat HTTP server on %s:%d" % (str(ip), int(port))) + + if config.MOAT_HTTPS_PORT: # pragma: no cover + ip = config.MOAT_HTTPS_IP or "" + port = config.MOAT_HTTPS_PORT or 443 + try: + from twisted.internet.ssl import DefaultOpenSSLContextFactory + factory = DefaultOpenSSLContextFactory(config.MOAT_TLS_KEY_FILE, + config.MOAT_TLS_CERT_FILE) + reactor.listenSSL(port, site, factory, interface=ip) + except CannotListenError as error: + raise SystemExit(error) + logging.info("Started Moat TLS server on %s:%d" % (str(ip), int(port))) + + return site diff --git a/bridgedb/main.py b/bridgedb/main.py index 1601cf1..905af2d 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -33,6 +33,7 @@ from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig from bridgedb.distributors.email.distributor import EmailDistributor from bridgedb.distributors.https.distributor import HTTPSDistributor +from bridgedb.distributors.moat.distributor import MoatDistributor from bridgedb.parse import descriptors from bridgedb.parse.blacklist import parseBridgeBlacklistFile

@@ -220,9 +221,10 @@ def createBridgeRings(cfg, proxyList, key): known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an + :returns: A :class:`~bridgedb.Bridges.BridgeSplitter` hashring, an :class:`~bridgedb.distributors.https.distributor.HTTPSDistributor` or None, and an - :class:`~bridgedb.distributors.email.distributor.EmailDistributor` or None. + :class:`~bridgedb.distributors.email.distributor.EmailDistributor` or None, and an + :class:`~bridgedb.distributors.moat.distributor.MoatDistributor` or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -233,7 +235,18 @@ def createBridgeRings(cfg, proxyList, key): ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS, needFlags=cfg.FORCE_FLAGS)

- emailDistributor = ipDistributor = None + emailDistributor = ipDistributor = moatDistributor = None + + # As appropriate, create a Moat distributor. + if cfg.MOAT_DIST and cfg.MOAT_SHARE: + logging.debug("Setting up Moat Distributor...") + moatDistributor = MoatDistributor( + cfg.MOAT_N_IP_CLUSTERS, + crypto.getHMAC(key, "Moat-Dist-Key"), + proxyList, + answerParameters=ringParams) + hashring.addRing(moatDistributor.hashring, "moat", cfg.MOAT_SHARE) + # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") @@ -265,7 +278,7 @@ def createBridgeRings(cfg, proxyList, key): for pseudoRing in cfg.FILE_BUCKETS.keys(): hashring.addPseudoRing(pseudoRing)

- return hashring, emailDistributor, ipDistributor + return hashring, emailDistributor, ipDistributor, moatDistributor

def run(options, reactor=reactor): """This is BridgeDB's main entry point and main runtime loop. @@ -323,12 +336,14 @@ def run(options, reactor=reactor):

from bridgedb.distributors.email.server import addServer as addSMTPServer from bridgedb.distributors.https.server import addWebServer + from bridgedb.distributors.moat.server import addMoatServer

# Load the master key, or create a new one. key = crypto.getKey(config.MASTER_KEY_FILE) proxies = proxy.ProxySet() emailDistributor = None ipDistributor = None + moatDistributor = None

# Save our state state.proxies = proxies @@ -388,7 +403,8 @@ def run(options, reactor=reactor): logging.info("Reparsing bridge descriptors...") (hashring, emailDistributorTmp, - ipDistributorTmp) = createBridgeRings(cfg, state.proxies, key) + ipDistributorTmp, + moatDistributorTmp) = createBridgeRings(cfg, state.proxies, key) logging.info("Bridges loaded: %d" % len(hashring))

# Initialize our DB. @@ -398,33 +414,19 @@ def run(options, reactor=reactor):

if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings - logging.info("Bridges allotted for %s distribution: %d" - % (emailDistributorTmp.name, - len(emailDistributorTmp.hashring))) else: logging.warn("No email distributor created!")

if ipDistributorTmp is not None: ipDistributorTmp.prepopulateRings() # create default rings - - logging.info("Bridges allotted for %s distribution: %d" - % (ipDistributorTmp.name, - len(ipDistributorTmp.hashring))) - logging.info("\tNum bridges:\tFilter set:") - - nSubrings = 0 - ipSubrings = ipDistributorTmp.hashring.filterRings - for (ringname, (filterFn, subring)) in ipSubrings.items(): - nSubrings += 1 - filterSet = ' '.join( - ipDistributorTmp.hashring.extractFilterNames(ringname)) - logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) - - logging.info("Total subrings for %s: %d" - % (ipDistributorTmp.name, nSubrings)) else: logging.warn("No HTTP(S) distributor created!")

+ if moatDistributorTmp is not None: + moatDistributorTmp.prepopulateRings() + else: + logging.warn("No Moat distributor created!") + # Dump bridge pool assignments to disk. try: logging.debug("Dumping pool assignments to file: '%s'" @@ -443,6 +445,9 @@ def run(options, reactor=reactor): if inThread: # XXX shutdown the distributors if they were previously running # and should now be disabled + if moatDistributorTmp: + reactor.callFromThread(replaceBridgeRings, + moatDistributor, moatDistributorTmp) if ipDistributorTmp: reactor.callFromThread(replaceBridgeRings, ipDistributor, ipDistributorTmp) @@ -452,7 +457,7 @@ def run(options, reactor=reactor): else: # We're still starting up. Return these distributors so # they are configured in the outer-namespace - return emailDistributorTmp, ipDistributorTmp + return emailDistributorTmp, ipDistributorTmp, moatDistributorTmp

global _reloadFn _reloadFn = reload @@ -461,9 +466,11 @@ def run(options, reactor=reactor):

if reactor: # pragma: no cover # And actually load it to start parsing. Get back our distributors. - emailDistributor, ipDistributor = reload(False) + emailDistributor, ipDistributor, moatDistributor = reload(False)

# Configure all servers: + if config.MOAT_DIST and config.MOAT_SHARE: + addMoatServer(config, moatDistributor) if config.HTTPS_DIST and config.HTTPS_SHARE: addWebServer(config, ipDistributor) if config.EMAIL_DIST and config.EMAIL_SHARE: diff --git a/bridgedb/test/https_helpers.py b/bridgedb/test/https_helpers.py index d54e544..fad841e 100644 --- a/bridgedb/test/https_helpers.py +++ b/bridgedb/test/https_helpers.py @@ -347,9 +347,8 @@ class DummyRequest(RequestHelperDummyRequest): self.content = io.StringIO()

self.headers = {} # Needed for Twisted>14.0.2 - #self.outgoingHeaders = {} - #self.responseHeaders = Headers() - #self.requestHeaders = Headers() + self.responseHeaders = Headers() + self.requestHeaders = Headers()

def writeContent(self, data): """Add some **data** to the faked body of this request. diff --git a/bridgedb/test/moat_helpers.py b/bridgedb/test/moat_helpers.py new file mode 100644 index 0000000..c7f7718 --- /dev/null +++ b/bridgedb/test/moat_helpers.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# :copyright: (c) 2013-2017, Isis Lovecruft +# (c) 2007-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information + + +"""Helpers for testing the HTTPS Distributor and its servers.""" + + +import io + +from bridgedb.persistent import Conf + +from . import util + + +GIMP_CAPTCHA_DIR = 'captchas' +SERVER_PUBLIC_FQDN = 'bridges.torproject.org' +SUPPORTED_TRANSPORTS = { + 'obfs2': False, + 'obfs3': True, + 'obfs4': True, + 'scramblesuit': True, + 'fte': True, +} +MOAT_DIST = True +MOAT_DIST_VIA_MEEK_ONLY = True +MOAT_TLS_CERT_FILE="moat-tls.crt" +MOAT_TLS_KEY_FILE="moat-tls.pem" +if MOAT_DIST_VIA_MEEK_ONLY: + MOAT_SERVER_PUBLIC_ROOT = '/meek/moat' +else: + MOAT_SERVER_PUBLIC_ROOT = '/moat' +MOAT_BRIDGES_PER_ANSWER = 3 +MOAT_TRANSPORT_PREFERENCE_LIST = ["obfs4", "vanilla"] +MOAT_HTTPS_IP = '127.0.0.1' +MOAT_HTTPS_PORT = None +MOAT_HTTP_IP = None +MOAT_HTTP_PORT = None +MOAT_USE_IP_FROM_FORWARDED_HEADER = True +MOAT_N_IP_CLUSTERS = 4 +MOAT_ROTATION_PERIOD = "3 hours" +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key' +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key' + +TEST_CONFIG_FILE = io.StringIO(unicode("""\ +GIMP_CAPTCHA_DIR = %r +SERVER_PUBLIC_FQDN = %r +SUPPORTED_TRANSPORTS = %r +MOAT_DIST = %r +MOAT_DIST_VIA_MEEK_ONLY = %r +MOAT_TLS_CERT_FILE = %r +MOAT_TLS_KEY_FILE = %r +MOAT_SERVER_PUBLIC_ROOT = %r +MOAT_HTTPS_IP = %r +MOAT_HTTPS_PORT = %r +MOAT_HTTP_IP = %r +MOAT_HTTP_PORT = %r +MOAT_BRIDGES_PER_ANSWER = %r +MOAT_TRANSPORT_PREFERENCE_LIST = %r +MOAT_USE_IP_FROM_FORWARDED_HEADER = %r +MOAT_N_IP_CLUSTERS = %r +MOAT_ROTATION_PERIOD = %r +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = %r +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = %r +""" % (GIMP_CAPTCHA_DIR, + SERVER_PUBLIC_FQDN, + SUPPORTED_TRANSPORTS, + MOAT_DIST, + MOAT_DIST_VIA_MEEK_ONLY, + MOAT_TLS_CERT_FILE, + MOAT_TLS_KEY_FILE, + MOAT_SERVER_PUBLIC_ROOT, + MOAT_HTTPS_IP, + MOAT_HTTPS_PORT, + MOAT_HTTP_IP, + MOAT_HTTP_PORT, + MOAT_BRIDGES_PER_ANSWER, + MOAT_TRANSPORT_PREFERENCE_LIST, + MOAT_USE_IP_FROM_FORWARDED_HEADER, + MOAT_N_IP_CLUSTERS, + MOAT_ROTATION_PERIOD, + MOAT_GIMP_CAPTCHA_HMAC_KEYFILE, + MOAT_GIMP_CAPTCHA_RSA_KEYFILE))) + +def _createConfig(configFile=TEST_CONFIG_FILE): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(configFile.read(), '<string>', 'exec') + exec compiled in configuration + config = Conf(**configuration) + return config + + +class DummyMoatDistributor(object): + """A mocked :class:`bridgedb.distributors.moat.distributor.MoatDistributor` + which is used to test + :class:`bridgedb.distributors.moat.server.CaptchaFetchResource`. + """ + _bridge_class = util.DummyBridge + _bridgesPerResponseMin = 3 + + def getBridges(self, bridgeRequest=None, epoch=None): + """Needed because it's called in + :meth:`BridgesResource.getBridgeRequestAnswer`.""" + return [self._bridge_class() for _ in range(self._bridgesPerResponseMin)] diff --git a/bridgedb/test/test_distributors_common_http.py b/bridgedb/test/test_distributors_common_http.py new file mode 100644 index 0000000..60759e4 --- /dev/null +++ b/bridgedb/test/test_distributors_common_http.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.common.http`.""" + +from __future__ import print_function + +import logging +import os + +from twisted.trial import unittest +from twisted.web.test import requesthelper + +from bridgedb.distributors.common import http as server + +from bridgedb.test.https_helpers import DummyRequest + + +# For additional logger output for debugging, comment out the following: +logging.disable(50) +# and then uncomment the following line: +#server.logging.getLogger().setLevel(10) + + +class SetFQDNTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.https.server.setFQDN` and + :func:`bridgedb.distributors.https.server.setFQDN`. + """ + + def setUp(self): + self.originalFQDN = server.SERVER_PUBLIC_FQDN + + def tearDown(self): + server.SERVER_PUBLIC_FQDN = self.originalFQDN + + def test_setFQDN_https(self): + """Calling ``server.setFQDN([…], https=True)`` should prepend + ``"https://%22%60%60 to the module :data:`server.SERVER_PUBLIC_FQDN` + variable. + """ + server.setFQDN('example.com', https=True) + self.assertEqual(server.SERVER_PUBLIC_FQDN, "https://example.com") + + def test_setFQDN_http(self): + """Calling ``server.setFQDN([…], https=False)`` should not prepend + anything at all to the module :data:`server.SERVER_PUBLIC_FQDN` + variable. + """ + server.setFQDN('example.com', https=False) + self.assertEqual(server.SERVER_PUBLIC_FQDN, "example.com") + + +class GetClientIPTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.https.server.getClientIP`.""" + + def createRequestWithIPs(self): + """Set the IP address returned from ``request.getClientIP()`` to + '3.3.3.3', and the IP address reported in the 'X-Forwarded-For' header + to '2.2.2.2'. + """ + request = DummyRequest(['']) + request.headers.update({'x-forwarded-for': '2.2.2.2'}) + # See :api:`twisted.test.requesthelper.DummyRequest.getClientIP` + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.method = b'GET' + return request + + def test_getClientIP_XForwardedFor(self): + """getClientIP() should return the IP address from the + 'X-Forwarded-For' header when ``useForwardedHeader=True``. + """ + request = self.createRequestWithIPs() + clientIP = server.getClientIP(request, useForwardedHeader=True) + self.assertEqual(clientIP, '2.2.2.2') + + def test_getClientIP_XForwardedFor_bad_ip(self): + """getClientIP() should return None if the IP address from the + 'X-Forwarded-For' header is bad/invalid and + ``useForwardedHeader=True``. + """ + request = self.createRequestWithIPs() + request.headers.update({'x-forwarded-for': 'pineapple'}) + clientIP = server.getClientIP(request, useForwardedHeader=True) + self.assertEqual(clientIP, None) + + def test_getClientIP_fromRequest(self): + """getClientIP() should return the IP address from the request instance + when ``useForwardedHeader=False``. + """ + request = self.createRequestWithIPs() + clientIP = server.getClientIP(request) + self.assertEqual(clientIP, '3.3.3.3') diff --git a/bridgedb/test/test_distributors_moat_request.py b/bridgedb/test/test_distributors_moat_request.py new file mode 100644 index 0000000..ed7f493 --- /dev/null +++ b/bridgedb/test/test_distributors_moat_request.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.moat.request`.""" + +from __future__ import print_function + +from twisted.trial import unittest + +from bridgedb.distributors.moat import request + + +class MoatBridgeRequest(unittest.TestCase): + """Unittests for :class:`bridgedb.distributors.moat.request.MoatBridgeRequest`.""" + + def setUp(self): + self.bridgeRequest = request.MoatBridgeRequest() + + def test_withoutBlockInCountry(self): + data = {'unblocked': ['us', 'ir', 'sy']} + + self.bridgeRequest.withoutBlockInCountry(data) + self.bridgeRequest.generateFilters() + + self.assertItemsEqual(['byTransportNotBlockedIn(None,us,4)', + 'byTransportNotBlockedIn(None,ir,4)', + 'byTransportNotBlockedIn(None,sy,4)'], + [x.__name__ for x in self.bridgeRequest.filters]) + + def test_withoutBlockInCountry_not_a_valid_country_code(self): + data = {'unblocked': ['3']} + self.bridgeRequest.withoutBlockInCountry(data) + + def test_withoutBlockInCountry_unicode(self): + data = {'unblocked': ['föö']} + self.bridgeRequest.withoutBlockInCountry(data) + + def test_withoutBlockInCountry_not_a_valid_transport(self): + data = {'unblocked': ['3']} + self.bridgeRequest.withPluggableTransportType(data) + + def test_withPluggableTransportType_unicode(self): + data = {'transport': 'bifröst'} + self.bridgeRequest.withPluggableTransportType(data) diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py new file mode 100644 index 0000000..2145541 --- /dev/null +++ b/bridgedb/test/test_distributors_moat_server.py @@ -0,0 +1,918 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.moat.server`.""" + +from __future__ import print_function + +import base64 +import io +import json +import logging +import os +import shutil +import tempfile + +from twisted.trial import unittest +from twisted.web.resource import Resource +from twisted.web.test import requesthelper + +from bridgedb import crypto +from bridgedb.distributors.moat import server +from bridgedb.schedule import ScheduledInterval + +from bridgedb.test.moat_helpers import _createConfig +from bridgedb.test.moat_helpers import DummyMoatDistributor +from bridgedb.test.https_helpers import DummyRequest +from bridgedb.test.util import DummyBridge +from bridgedb.test.util import DummyMaliciousBridge + + +# For additional logger output for debugging, comment out the following: +logging.disable(50) +# and then uncomment the following line: +#server.logging.getLogger().setLevel(10) + + +# These keys are hardcoded so that we can test both expired and +# unexpired CAPTCHAs which were/are produced with them. +CAPTCHA_KEY = base64.b64decode("tFh0hkskhDBulvBtYkYy7qlQhyZh2MKsIOfaAmmvinQ=") +HMAC_KEY = base64.b64decode("vJPta7PflEb/qalt5Klgn9wyfgs=") + +SECRET_KEY = crypto.PKCS1_OAEP.new(crypto.RSA.importKey("""\ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAtz551eYMzSE8a56TGFRt4+bhZOgVCShBjaXv0LfuFtF4KXhv +cMXG8Zo9jw0M8HCk9YLhb40bLaeSxemcoaHiuC/zxgL8ECQ7GMVO6vi409UWRnl7 +VpCqyFPXg+V89TM67IsMejT8CDG4g62DYgSQ0Fpl5AsFd3RN2mQ38qdrmoBTXxQt +cmMXkEgDN20ZoVRFbWa+KurX9pv74wtNqUY2uUMxyGwXIrBfi6O3cyyjUAoSXsJT +iuKmgcuzE9+fDKzMftEf+k8OSs/DQWYARefQBCTndzha9ICwxFM5L3CWsPQxKtS4 +mkILRGLZSHYeswynwH98Swimgyv2FSqOZN9eYQIDAQABAoIBAQC0S1JQ9RKvWf46 +3UFZdOjSjb5DLF5WLjehiR0WPYKTDPKvywHK8a221c2vzGVoxUxpC6eHvEx7dR9i +f2JPXhrWose1kgY0U5GZ47isVKB2PHi4SpriJ2EBzgyEh+2UzB0z0/Qo4a0A2vrz +BGv6qwdZGTibUYTFbbeUI3sw0y16SxHPeGaYtfyLcx5/Nwd92V9NZ1pyaivVt4Uu +XgHnYxG7Y1uurSv6sdPouUN+H1o9msBm6EMRXjDrCAyVr0gmH2KQmnWedwSiY+eh +n5hP8jkOEQntxirMM5AR8N/KSXJdt9nccRPbamInn4eK1wlOzJJzwLpu8hhwJ+fc +6dWfBP9hAoGBAMxiSDtYqai6hTongIyKasycm14F0Xzcd5dVBwW1ANLdWbC32ti9 +n7i6TD7Fe2vfmiNpylMLbYLt5zODHT3eb3l1cWWpvkrMmxkvfj3to9sJfCg2p+jt +WzxugmSu9qO03C9nhSsFlD2Li754nPVCa5gk/1Gnxt9+RRI2Qzb7r1GFAoGBAOWF +eHJAO2SmZBc9doRqUgQXcZin/60f9PpxPqHtT8NzBItSxPSlkGYspDJaxPZfl5Y1 +7MZMNN6vBTitKXO3JXYZwCD+Y8s+trd/qm/vbco2SinWlgRlZyqcyBCvGKfBMUg/ +Nr8mRPyZ18yBEj5OtYadJcRCERk3ERWSt/ndLwItAoGAC/20KS8xfQG8cUYCB7zT +OT/y6ZhDyySQK6PEbrRI4RY1feW7hD3T0h2z/XbOn+yVeYBqa2bfPPBCQUZu/8M+ +HQ0j4wgLbw4EB30+1dlMZLxwuVdDkKnkUW5WXhvZwo8I4AsdyAFiyh2WzEz9QHJu +J5X8GMlUJKae3MusM9yeU5UCgYEAw8unYzd+My9qZRTunKkiTBE/u61dA/AmCNtA +Rdxu1dmxf7TdBaKTW0Yr0DT0nwQPCXn5AXSTCYAeoSm/GdKb53KyHrNEqGZYcpM6 +7wA+FWlYvPYsxZVHe+eBGBJ2ouzAwNQEPO5FnYMTv4Y/7N0yJ6K5TAHcGjmKnm+p ++EICTwUCgYEArF7zfcaxRXQtNKKEIYR9Q+zL/+fQEF2lxre61UNxpS0CmDrrAZwu +oa7cfLxocTUpYp7atUINzwVQgVd6AWta3v/PoXhkxo/CRd7pheyeH/ypFa7vAkzO +zZAlI9uLR/7XevId2W7b8U+8AWtxi3RLXSId9QzGRvjkoThBAKfLM30= +-----END RSA PRIVATE KEY-----""")) + +PUBLIC_KEY = crypto.PKCS1_OAEP.new(crypto.RSA.importKey("""\ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtz551eYMzSE8a56TGFRt +4+bhZOgVCShBjaXv0LfuFtF4KXhvcMXG8Zo9jw0M8HCk9YLhb40bLaeSxemcoaHi +uC/zxgL8ECQ7GMVO6vi409UWRnl7VpCqyFPXg+V89TM67IsMejT8CDG4g62DYgSQ +0Fpl5AsFd3RN2mQ38qdrmoBTXxQtcmMXkEgDN20ZoVRFbWa+KurX9pv74wtNqUY2 +uUMxyGwXIrBfi6O3cyyjUAoSXsJTiuKmgcuzE9+fDKzMftEf+k8OSs/DQWYARefQ +BCTndzha9ICwxFM5L3CWsPQxKtS4mkILRGLZSHYeswynwH98Swimgyv2FSqOZN9e +YQIDAQAB +-----END PUBLIC KEY-----""")) + + +class MiscellaneousTests(unittest.TestCase): + """Tests for helper functions in :mod:`bridgedb.distributors.moat.server`.""" + + def setUp(self): + self.config = _createConfig() + + def test_setRoot(self): + """If we call `setRoot()` with `root="/meek/moat"` then the public + root directory of the server should be "/meek/moat". + """ + server.setRoot("/meek/moat") + self.assertEqual(server.getRoot(), "/meek/moat") + + def test_getFQDNAndRoot(self): + """If the FQDN is set to "bridges.torproject.org" and we call `setRoot()` + with `root="/meek/moat"` then the public root directory of the server + should be "/meek/moat". + """ + server.setFQDN("bridges.torproject.org", https=True) + server.setRoot("/meek/moat") + + self.assertEqual(server.getFQDNAndRoot(), + "https://bridges.torproject.org/meek/moat") + + def test_getFQDNAndRoot_no_slash(self): + """If the FQDN is set to "bridges.torproject.org" and we call `setRoot()` + with `root="meek/moat"` then the public root directory of the server + should be "bridges.torproject.org/meek/moat". + """ + server.setFQDN("bridges.torproject.org", https=True) + server.setRoot("meek/moat") # missing the "/" prefix + + self.assertEqual(server.getFQDNAndRoot(), + "https://bridges.torproject.org/meek/moat") + + def test_setPreferredTransports(self): + """Setting the pluggable transport preference list to ["dinosaur"] + should set it thusly. + """ + prefs = ["dinosaur"] + + server.setPreferredTransports(prefs) + + self.assertEqual(server.getPreferredTransports(), prefs) + + def test_setSupportedPreferences(self): + """Taking the ``SUPPORTED_TRANSPORTS`` config option (a dict) and + passing it to ``setSupportedTransports()`` should convert it into a list + for the items in the dict whose value was ``True``. + """ + server.setSupportedTransports(self.config.SUPPORTED_TRANSPORTS) + + self.assertItemsEqual(server.getSupportedTransports(), + ["obfs4", "obfs3", "scramblesuit", "fte", "vanilla"]) + + +class JsonAPIResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIResource`.""" + + def setUp(self): + self.pagename = b'' + self.resource = server.JsonAPIResource() + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + def test_getClientIP(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + self.resource.getClientIP(request) + + def test_formatDataForResponse(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + data = {'data': { 'version': 'wow', + 'dinosaurs': 'cool', + 'triceratops': 'awesome', + 'velociraptors': 'terrifying', }} + + rendered = self.resource.formatDataForResponse(data, request) + + self.assertTrue(rendered) + self.assertTrue(request.responseHeaders.hasHeader('content-type')) + self.assertTrue(request.responseHeaders.hasHeader('server')) + self.assertEqual(request.responseHeaders.getRawHeaders('content-type'), + ['application/vnd.api+json']) + + def test_formatDataForResponse_no_data(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + rendered = self.resource.formatDataForResponse(None, request) + + self.assertEqual(rendered, b'') + + +class JsonAPIErrorResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIErrorResource`.""" + + def setUp(self): + self.pagename = b'' + self.root = Resource() + + def use_resource(self, resource): + self.resource = resource + self.root.putChild(self.pagename, self.resource) + + def do_render_for_method(self, method): + request = DummyRequest([self.pagename]) + request.method = method + + rendered = self.resource.render(request) + + self.assertTrue(rendered) + self.assertTrue(request.responseHeaders.hasHeader('content-type')) + self.assertTrue(request.responseHeaders.hasHeader('server')) + self.assertEqual(request.responseHeaders.getRawHeaders('content-type'), + ['application/vnd.api+json']) + + decoded = json.loads(rendered) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + + self.assertEqual(len(errors), 1) + + error = errors[0] + + return error + + def test_render_GET(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'GET') + + def test_render_POST(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'POST') + + def test_resource200_render_GET(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['id'], 0) + self.assertEqual(error['type'], '') + self.assertEqual(error['code'], 200) + self.assertEqual(error['status'], 'OK') + self.assertEqual(error['detail'], '') + + def test_resource403_render_GET(self): + self.use_resource(server.resource403) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 403) + self.assertEqual(error['status'], 'Forbidden') + + def test_resource406_render_GET(self): + self.use_resource(server.resource406) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 406) + self.assertEqual(error['status'], 'Not Acceptable') + + def test_resource415_render_GET(self): + self.use_resource(server.resource415) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 415) + self.assertEqual(error['status'], 'Unsupported Media Type') + + def test_resource419_render_GET(self): + self.use_resource(server.resource419) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 419) + self.assertEqual(error['status'], "No You're A Teapot") + + def test_resource501_render_GET(self): + self.use_resource(server.resource501) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 501) + self.assertEqual(error['status'], 'Not Implemented') + + +class CustomErrorHandlingResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CustomErrorHandlingResource`.""" + + def setUp(self): + self.pagename = b'' + self.resource = server.CustomErrorHandlingResource() + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + def test_getChild(self): + request = DummyRequest(['foo']) + request.method = b'GET' + response_resource = self.resource.getChild('/foo', request) + + self.assertTrue(response_resource) + self.assertIsInstance(response_resource, server.JsonAPIErrorResource) + + response = response_resource.render(request) + detail = json.loads(response)['errors'][0]['detail'] + + self.assertIn('does not implement GET http://dummy/', detail) + + +class JsonAPIDataResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIDataResource`.""" + + def setUp(self): + self.resource = server.JsonAPIDataResource() + + def test_checkRequestHeaders_no_headers(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + + def test_checkRequestHeaders_different_content_type(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + request.requestHeaders.addRawHeader('Content-Type', 'application/html') + + def test_checkRequestHeaders_with_media_type(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json;mp3') + + +class CaptchaFetchResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CaptchaFetchResource`.""" + + def setUp(self): + self.topDir = os.getcwd().rstrip('_trial_temp') + self.captchaDir = os.path.join(self.topDir, 'captchas') + self.captchaKey = CAPTCHA_KEY + self.hmacKey = HMAC_KEY + self.secretKey, self.publicKey = SECRET_KEY, PUBLIC_KEY + self.resource = server.CaptchaFetchResource(self.hmacKey, + self.publicKey, + self.secretKey, + self.captchaDir) + self.pagename = b'fetch' + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + self.make_captcha_directory() + + def make_captcha_directory(self): + if not os.path.isdir(self.captchaDir): + os.mkdir(self.captchaDir) + + def create_POST_with_data(self, data): + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + request.writeContent(data) + + return request + + def create_valid_POST(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def create_valid_POST_with_unsupported_transports(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4', 'dinosaur', 'karlthefog'], + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def test_init(self): + self.assertTrue(self.resource) + + def test_checkRequestHeaders_missing_content_type(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.removeHeader('Content-Type') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_missing_accept(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.removeHeader('Accept') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_content_type_with_media_parameters(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json;mp3') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_accept_with_media_parameters(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json;mp3') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_getCaptchaImage(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + image, challenge = self.resource.getCaptchaImage(request) + + self.assertIsNotNone(image) + self.assertIsNotNone(challenge) + + def test_getCaptchaImage_empty_captcha_dir(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + captchaDirOrig = self.resource.captchaDir + captchaDirNew = tempfile.mkdtemp() + self.resource.captchaDir = captchaDirNew + image, challenge = self.resource.getCaptchaImage(request) + self.resource.captchaDir = captchaDirOrig + shutil.rmtree(captchaDirNew) + + self.assertIsNone(image) + self.assertIsNone(challenge) + + def test_extractSupportedTransports_missing_type(self): + data = { + 'data': [{ + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_missing_version(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_missing_supported(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_wrong_type(self): + data = { + 'data': [{ + 'type': 'totoro', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_wrong_version(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': '0.0.1', # this version never existed + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_none_supported(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': [], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_preferred_transport(self): + request = self.create_valid_POST() + supported = self.resource.extractSupportedTransports(request) + + self.assertEqual(supported, ['obfs4']) + + def test_extractSupportedTransports_preferred_and_unknown_transports(self): + request = self.create_valid_POST_with_unsupported_transports() + supported = self.resource.extractSupportedTransports(request) + + self.assertEqual(supported, ['obfs4', 'dinosaur', 'karlthefog']) + + def test_getPreferredTransports_preferred_transport(self): + preferred = self.resource.getPreferredTransports(['obfs4']) + + self.assertEqual(preferred, 'obfs4') + + def test_getPreferredTransports_unknown_transport(self): + preferred = self.resource.getPreferredTransports(['dinosaur']) + + self.assertItemsEqual(preferred, + ['obfs4', 'obfs3', 'fte', 'scramblesuit', 'vanilla']) + + def assert_data_is_ok(self, decoded): + self.assertIsNone(decoded.get('errors')) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + + self.assertEqual(len(datas), 1) + + data = datas[0] + + self.assertEqual(data["type"], "moat-challenge") + self.assertEqual(data["version"], server.MOAT_API_VERSION) + self.assertIsNotNone(data["challenge"]) + self.assertIsNotNone(data["image"]) + self.assertIsNotNone(data["transport"]) + + def test_render_POST(self): + request = self.create_valid_POST() + response = self.resource.render(request) + + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assert_data_is_ok(decoded) + + +class CaptchaCheckResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CaptchaCheckResource`.""" + + def setUp(self): + self.topDir = os.getcwd().rstrip('_trial_temp') + self.captchaDir = os.path.join(self.topDir, 'captchas') + self.captchaKey = CAPTCHA_KEY + self.hmacKey = HMAC_KEY + self.secretKey, self.publicKey = SECRET_KEY, PUBLIC_KEY + self.distributor = DummyMoatDistributor() + self.schedule = ScheduledInterval("10", "minutes") + self.resource = server.CaptchaCheckResource(self.distributor, + self.schedule, 3, + self.hmacKey, + self.publicKey, + self.secretKey, + useForwardedHeader=False) + self.pagename = b'check' + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + self.solution = 'Tvx74PMy' + self.expiredChallenge = ( + "Vu-adMmSRsgr9PmPpGAznhrBQlys3zMkczIG2YQ7AngWqWnVn2y-LdAl8iHkrqkNhn" + "iyre02ZlUf5KD_KDqh_Km3dIoksOMW3eUuargLLnhIUldJ4PvSXPb7pwGev_FDY4gF" + "QDcmkrhFZm6RPzFWRgJjyY-2v6HRrmAMAGjGXSXnAc-8tDvVFSpo5Cce-saZou5W4G" + "TjzVcyG0WkXELA2nX8rozIDIr3mUyB1vb3f53KbW5b_oCEVC_LCSoxqjnS6ZSQpNzK" + "iz_PdOD2GIGPeclwiHAWM1pOS4cQVsTQR_z4ojZbpLiSp35n4Qbb11YOoreovZzlbS" + "7W38rAsTirkdeugcNq82AxKP3phEkyRcw--CzV") + + def create_POST_with_data(self, data): + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + + request.writeContent(data) + + return request + + def create_valid_POST_with_challenge(self, challenge): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': challenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def create_valid_POST_make_new_challenge(self): + request = DummyRequest([self.pagename]) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + resource = server.CaptchaFetchResource(self.hmacKey, self.publicKey, + self.secretKey, self.captchaDir, + useForwardedHeader=False) + image, challenge = resource.getCaptchaImage(request) + + request = self.create_valid_POST_with_challenge(challenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + return request + + def test_withoutBlockIn(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + 'unblocked': ['us', 'ir', 'sy'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + + self.resource.render(request) + + def test_extractClientSolution(self): + request = self.create_valid_POST_make_new_challenge() + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + qrcode, transport, challenge, solution = self.resource.extractClientSolution(content) + + self.assertFalse(qrcode) + self.assertIsNotNone(transport) + self.assertIsNotNone(challenge) + self.assertIsNotNone(solution) + + def test_extractClientSolution_missing_id(self): + data = { + 'data': [{ + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_id(self): + data = { + 'data': [{ + 'id': 69, # nice + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_weird_transport(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'dinosaur', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_version(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': '0.0.1', # this version never existed + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_type(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'boat-revolution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_failureResponse_5(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + response = self.resource.failureResponse(5, request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + self.assertEqual(len(errors), 1) + + error = errors[0] + self.assertEqual(error['status'], "No You're A Teapot") + self.assertEqual(error['code'], 419) + self.assertEqual(error['detail'], "The CAPTCHA challenge timed out.") + self.assertEqual(error['id'], 5) + + def test_checkSolution(self): + request = self.create_valid_POST_make_new_challenge() + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + clientIP = self.resource.getClientIP(request) + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + qrcode, transport, challenge, solution = self.resource.extractClientSolution(content) + result = self.resource.checkSolution(challenge, solution, clientIP) + + self.assertTrue(result) + + def test_render_POST_expired(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + self.assertEqual(len(errors), 1) + + error = errors[0] + self.assertEqual(error['status'], "No You're A Teapot") + self.assertEqual(error['code'], 419) + self.assertEqual(error['detail'], "The CAPTCHA solution was incorrect.") + self.assertEqual(error['version'], server.MOAT_API_VERSION) + self.assertEqual(error['type'], "moat-bridges") + self.assertEqual(error['id'], 4) + + def test_getBridgeLines(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + + bridgelines = self.resource.getBridgeLines('3.3.3.3', content) + + self.assertTrue(bridgelines) + + def test_getBridgeLines_no_data(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + + bridgelines = self.resource.getBridgeLines('3.3.3.3', None) + + self.assertIsNone(bridgelines) + + def test_render_POST_unexpired(self): + request = self.create_valid_POST_make_new_challenge() + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + self.assertEqual(len(datas), 1) + + data = datas[0] + self.assertIsNone(data['qrcode']) + self.assertIsNotNone(data['bridges']) + self.assertEqual(data['version'], server.MOAT_API_VERSION) + self.assertEqual(data['type'], 'moat-bridges') + self.assertEqual(data['id'], 3) + + def test_render_POST_unexpired_with_qrcode(self): + request = DummyRequest([self.pagename]) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + resource = server.CaptchaFetchResource(self.hmacKey, self.publicKey, + self.secretKey, self.captchaDir, + useForwardedHeader=False) + image, challenge = resource.getCaptchaImage(request) + + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': challenge, + 'solution': self.solution, + 'qrcode': True, + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + self.assertEqual(len(datas), 1) + + data = datas[0] + self.assertIsNotNone(data['qrcode']) + self.assertIsNotNone(data['bridges']) + self.assertEqual(data['version'], server.MOAT_API_VERSION) + self.assertEqual(data['type'], 'moat-bridges') + self.assertEqual(data['id'], 3) + + +class AddMoatServerTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.moat.server.addMoatServer()`.""" + + def setUp(self): + self.config = _createConfig() + self.distributor = DummyMoatDistributor() + + def test_addMoatServer(self): + server.addMoatServer(self.config, self.distributor) diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 2c41359..dbd177f 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -43,76 +43,6 @@ logging.disable(50) #server.logging.getLogger().setLevel(10)

-class SetFQDNTests(unittest.TestCase): - """Tests for :func:`bridgedb.distributors.https.server.setFQDN` and - :func:`bridgedb.distributors.https.server.setFQDN`. - """ - - def setUp(self): - self.originalFQDN = server.SERVER_PUBLIC_FQDN - - def tearDown(self): - server.SERVER_PUBLIC_FQDN = self.originalFQDN - - def test_setFQDN_https(self): - """Calling ``server.setFQDN([…], https=True)`` should prepend - ``"https://%22%60%60 to the module :data:`server.SERVER_PUBLIC_FQDN` - variable. - """ - server.setFQDN('example.com', https=True) - self.assertEqual(server.SERVER_PUBLIC_FQDN, "https://example.com") - - def test_setFQDN_http(self): - """Calling ``server.setFQDN([…], https=False)`` should not prepend - anything at all to the module :data:`server.SERVER_PUBLIC_FQDN` - variable. - """ - server.setFQDN('example.com', https=False) - self.assertEqual(server.SERVER_PUBLIC_FQDN, "example.com") - - -class GetClientIPTests(unittest.TestCase): - """Tests for :func:`bridgedb.distributors.https.server.getClientIP`.""" - - def createRequestWithIPs(self): - """Set the IP address returned from ``request.getClientIP()`` to - '3.3.3.3', and the IP address reported in the 'X-Forwarded-For' header - to '2.2.2.2'. - """ - request = DummyRequest(['']) - request.headers.update({'x-forwarded-for': '2.2.2.2'}) - # See :api:`twisted.test.requesthelper.DummyRequest.getClientIP` - request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) - request.method = b'GET' - return request - - def test_getClientIP_XForwardedFor(self): - """getClientIP() should return the IP address from the - 'X-Forwarded-For' header when ``useForwardedHeader=True``. - """ - request = self.createRequestWithIPs() - clientIP = server.getClientIP(request, useForwardedHeader=True) - self.assertEqual(clientIP, '2.2.2.2') - - def test_getClientIP_XForwardedFor_bad_ip(self): - """getClientIP() should return None if the IP address from the - 'X-Forwarded-For' header is bad/invalid and - ``useForwardedHeader=True``. - """ - request = self.createRequestWithIPs() - request.headers.update({'x-forwarded-for': 'pineapple'}) - clientIP = server.getClientIP(request, useForwardedHeader=True) - self.assertEqual(clientIP, None) - - def test_getClientIP_fromRequest(self): - """getClientIP() should return the IP address from the request instance - when ``useForwardedHeader=False``. - """ - request = self.createRequestWithIPs() - clientIP = server.getClientIP(request) - self.assertEqual(clientIP, '3.3.3.3') - - class ReplaceErrorPageTests(unittest.TestCase): """Tests for :func:`bridgedb.distributors.https.server.replaceErrorPage`."""

diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py index ab34055..2a124b1 100644 --- a/bridgedb/test/test_main.py +++ b/bridgedb/test/test_main.py @@ -302,12 +302,12 @@ class BridgedbTests(unittest.TestCase): hashring. """ proxyList = None - (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, - proxyList, - self.key) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + self.config, proxyList, self.key) + # Should have an HTTPSDistributor ring, an EmailDistributor ring, - # and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + # a MoatDistributor right, and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4)

def test_main_createBridgeRings_with_proxyList(self): """main.createBridgeRings() should add three hashrings to the @@ -316,12 +316,12 @@ class BridgedbTests(unittest.TestCase): exitRelays = ['1.1.1.1', '2.2.2.2', '3.3.3.3'] proxyList = main.proxy.ProxySet() proxyList.addExitRelays(exitRelays) - (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, - proxyList, - self.key) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + self.config, proxyList, self.key) + # Should have an HTTPSDistributor ring, an EmailDistributor ring, - # and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + # a MoatDistributor ring, and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4) self.assertGreater(len(httpsDist.proxies), 0) self.assertItemsEqual(exitRelays, httpsDist.proxies)

@@ -332,11 +332,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.HTTPS_DIST = False - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an EmailDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an EmailDistributor ring, a MoatDistributor ring, and an + # UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('https', hashring.rings) self.assertNotIn(httpsDist, hashring.ringsByName.values())

@@ -347,11 +348,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.EMAIL_DIST = False - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, a MoatDistributor ring, and an + # UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('email', hashring.rings) self.assertNotIn(emailDist, hashring.ringsByName.values())

@@ -362,11 +364,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.RESERVED_SHARE = 0 - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, and an EmailDistributor ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, an EmailDistributor ring, and a + # MoatDistributor ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('unallocated', hashring.rings)

def test_main_createBridgeRings_two_file_buckets(self): @@ -380,12 +383,12 @@ class BridgedbTests(unittest.TestCase): 'bridges-for-support-desk': 10, 'bridges-for-ooni-tests': 10, } - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, an EmailDistributor, and an - # UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, an EmailDistributor, a + # MoatDistributor and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4)

# Should have two pseudoRings: self.assertEqual(len(hashring.pseudoRings), 2) diff --git a/scripts/setup-tests b/scripts/setup-tests index 8e42762..18298f4 100755 --- a/scripts/setup-tests +++ b/scripts/setup-tests @@ -25,8 +25,15 @@ sed -r -i -e "s/(EMAIL_SMTP_PORT = )([1-9]{2,5})/\12525/" run/bridgedb.conf sed -r -i -e "s/(HTTP_UNENCRYPTED_BIND_IP = )(None)/\1'127.0.0.1'/" run/bridgedb.conf sed -r -i -e "s/(HTTP_UNENCRYPTED_PORT = )(None)/\16788/" run/bridgedb.conf sed -r -i -e "s/(SERVER_PUBLIC_FQDN = )(.*)/\1'127.0.0.1:6788'/" run/bridgedb.conf +# Enable plaintext HTTP distribution for the moat server as well +sed -r -i -e "s/(MOAT_HTTP_IP = )(None)/\1'127.0.0.1'/" run/bridgedb.conf +sed -r -i -e "s/(MOAT_HTTP_PORT = )(None)/\16790/" run/bridgedb.conf +# Create descriptors leekspin -n 100 cp -t run/from-authority networkstatus-bridges cached-extrainfo* bridge-descriptors cp -t run/from-bifroest networkstatus-bridges cached-extrainfo* bridge-descriptors +# Create TLS certificates ./scripts/make-ssl-cert cp -t run privkey.pem cert +cp privkey.pem run/moat-tls.pem +cp cert run/moat-tls.crt diff --git a/setup.py b/setup.py index a133424..1139a65 100644 --- a/setup.py +++ b/setup.py @@ -376,8 +376,10 @@ setuptools.setup( package_dir={'bridgedb': 'bridgedb'}, packages=['bridgedb', 'bridgedb.distributors', + 'bridgedb.distributors.common', 'bridgedb.distributors.email', 'bridgedb.distributors.https', + 'bridgedb.distributors.moat', 'bridgedb.parse', 'bridgedb.test', ],