November 2017 - tor-commits - lists.torproject.org

[bridgedb/master] Implement moat distributor and server.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 3667a63027d6d02c87ceae03a22771ee03a56e2a Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 14 17:13:24 2017 +0000 Implement moat distributor and server. * ADD a new distributor, moat, which speaks to Tor Launcher over a JSON API interface. * ADD new `bridgedb.distributors` package. * ADD new `bridgedb.distributors.common` package for code which is shared by multiple distributors. * FIXES #22871: https://bugs.torproject.org/22871 --- .coveragerc | 1 + README.rst | 10 +- bridgedb.conf | 104 ++- bridgedb/captcha.py | 1 - bridgedb/configure.py | 12 +- bridgedb/distributors/__init__.py | 3 - bridgedb/distributors/common/__init__.py | 2 + bridgedb/distributors/common/http.py | 80 +++ bridgedb/distributors/email/__init__.py | 18 +- bridgedb/distributors/email/distributor.py | 3 + bridgedb/distributors/https/__init__.py | 6 +- bridgedb/distributors/https/distributor.py | 11 + bridgedb/distributors/https/server.py | 63 +- bridgedb/distributors/moat/__init__.py | 4 + bridgedb/distributors/moat/distributor.py | 64 ++ bridgedb/distributors/moat/request.py | 133 ++++ bridgedb/distributors/moat/server.py | 776 ++++++++++++++++++++ bridgedb/main.py | 59 +- bridgedb/test/https_helpers.py | 5 +- bridgedb/test/moat_helpers.py | 110 +++ bridgedb/test/test_distributors_common_http.py | 100 +++ bridgedb/test/test_distributors_moat_request.py | 52 ++ bridgedb/test/test_distributors_moat_server.py | 918 ++++++++++++++++++++++++ bridgedb/test/test_https_server.py | 70 -- bridgedb/test/test_main.py | 65 +- scripts/setup-tests | 7 + setup.py | 2 + 27 files changed, 2470 insertions(+), 209 deletions(-) diff --git a/.coveragerc b/.coveragerc index 737310b..32d0c83 100644 --- a/.coveragerc +++ b/.coveragerc @@ -24,6 +24,7 @@ exclude_lines = if options[.verbosity.] def __repr__ if __name__ == .__main__.: + except Exception as impossible: # Ignore source code which cannot be found: ignore_errors = True # Exit with status code 2 if under this percentage is covered: diff --git a/README.rst b/README.rst index a2d3b27..3dbe647 100644 --- a/README.rst +++ b/README.rst @@ -444,7 +444,7 @@ The client MUST send a ``POST /meek/moat/fetch`` containing the following JSON:: { 'data': { 'version': '0.1.0', - 'type': 'moat client supported transports', + 'type': 'client-transports', 'supported': [ 'TRANSPORT', 'TRANSPORT', ... ], } } @@ -471,7 +471,7 @@ server will respond with the list of transports which is *does* support:: { 'data': { 'version': '0.1.0', - 'type': 'moat server supported transports', + 'type': 'moat-transports', 'supported': [ 'TRANSPORT', 'TRANSPORT', ... ], } } @@ -483,7 +483,7 @@ following JSON containing a CAPTCHA challenge:: { 'data': { 'id': 1, - 'type': 'moat farfetchd challenge', + 'type': 'moat-challenge', 'version': '0.1.0', 'transport': TRANSPORT, 'image': CAPTCHA, @@ -513,7 +513,7 @@ To propose a solution to a CAPTCHA, the client MUST send a request for ``POST { 'data': { 'id': 2, - 'type': 'moat farfetchd solution', + 'type': 'moat-solution', 'version': '0.1.0', 'transport': TRANSPORT, 'challenge': CHALLENGE, @@ -547,7 +547,7 @@ server responds ``200 OK`` with the following JSON:: { 'data': { 'id': 3, - 'type': 'moat bridges', + 'type': 'moat-bridges', 'version': '0.1.0', 'bridges': [ 'BRIDGE_LINE', ... ], 'qrcode': QRCODE, diff --git a/bridgedb.conf b/bridgedb.conf index 25a9cbf..d0366cf 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -150,12 +150,6 @@ STATUS_FILE = "networkstatus-bridges" # IGNORE_NETWORKSTATUS = True -# Certificate file and private key for the HTTPS Distributor. To create a -# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files -# in your current directory. -HTTPS_CERT_FILE="cert" -HTTPS_KEY_FILE="privkey.pem" - #---------------- # Output Files \ Where to store created data #------------------------------------------------------------------------------ @@ -294,15 +288,104 @@ SUPPORTED_TRANSPORTS = { DEFAULT_TRANSPORT = 'obfs4' #------------------------------- +# Moat Distribution Options \ +#------------------------------------------------------------------------------ +# +# These options configure the behaviour of a web interface which speaks JSON API +# to a remote application in order to present said application with the +# necessary information for creating a user interface for bridge distribution +# mechanism, similar to the web interface of BridgeDB's HTTPS Distributor. If +# MOAT_DIST is enabled, make sure that the MOAT_CERT_FILE and MOAT_KEY_FILE +# options point to the correct location of your SSL certificate and key! +# ------------------------------------------------------------------------------ + +# (boolean) True to enable distribution via Moat; False otherwise. +MOAT_DIST = True + +# (boolean) True to only allow Moat distribution via a Meek tunnel. False to +# only allow Moat distribution via untunneled HTTP(S). +MOAT_DIST_VIA_MEEK_ONLY = True + +# Certificate file and private key for the Moar Distributor. To create a +# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files +# in your current directory. +MOAT_TLS_CERT_FILE="moat-tls.crt" +MOAT_TLS_KEY_FILE="moat-tls.pem" + +# (string) The Fully-Qualified Domain Name (FQDN) of the server that the Moat +# and/or HTTPS distributor(s) is/are publicly reachable at. +if MOAT_DIST_VIA_MEEK_ONLY: + MOAT_SERVER_PUBLIC_ROOT = '/meek/moat' +else: + MOAT_SERVER_PUBLIC_ROOT = '/moat' + +# How many bridges do we give back in an answer (either HTTP or HTTPS)? +MOAT_BRIDGES_PER_ANSWER = 3 + +# (list) An ordered list of the preferred transports which moat should +# distribute, in order from most preferable to least preferable. +MOAT_TRANSPORT_PREFERENCE_LIST = ["obfs4", "vanilla"] + +# (string or None) The IP address where we listen for HTTPS connections. If +# ``None``, listen on the default interface. +MOAT_HTTPS_IP = '127.0.0.1' + +# (integer or None) The port to listen on for incoming HTTPS connections. +MOAT_HTTPS_PORT = 6791 + +# (string or None) The IP address to listen on for unencrypted HTTP +# connections. Set to ``None`` to disable unencrypted connections to the web +# interface. +MOAT_HTTP_IP = None + +# (integer or None) The port to listen on for incoming HTTP connections. +MOAT_HTTP_PORT = None + +# If true, there is a trusted proxy relaying incoming messages to us: take +# the *last* entry from its X-Forwarded-For header as the client's IP. +MOAT_USE_IP_FROM_FORWARDED_HEADER = True + +# How many clusters do we group IPs in when distributing bridges based on IP? +# Note that if PROXY_LIST_FILES is set (below), what we actually do here +# is use one higher than the number here, and the extra cluster is used +# for answering requests made by IP addresses in the PROXY_LIST_FILES file. +MOAT_N_IP_CLUSTERS = 4 + +# (string or None) The period at which the available bridges rotates to a +# separate set of bridges. This setting can be used in the form +# +# "COUNT PERIOD" where +# COUNT is an integer +# PERIOD is one of "second", "minute", "hour", "day", +# "week", or "month" (or any plural form). +# +# For example, setting HTTPS_ROTATION_PERIOD = "3 days" will result in the set +# of bridges which are available through the web interface (either HTTP or +# HTTPS) getting rotated once every three days. Setting this to None disables +# rotation entirely. +MOAT_ROTATION_PERIOD = "3 hours" + +# The location of the files which store the HMAC secret key and RSA keypair +# (for checking captcha responses): +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key' +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key' + +#------------------------------- # HTTP(S) Distribution Options \ #------------------------------------------------------------------------------ # # These options configure the behaviour of the web interface bridge -# distribution mechanism. If HTTPS_DIST is enabled, make sure that the above +# distribution mechanism. If HTTPS_DIST is enabled, make sure that the # HTTPS_CERT_FILE and HTTPS_KEY_FILE options point to the correct location of # your SSL certificate and key! #------------------------------------------------------------------------------ +# Certificate file and private key for the HTTPS Distributor. To create a +# self-signed cert, run ``scripts/make-ssl-cert`` it will create these files +# in your current directory. +HTTPS_CERT_FILE="cert" +HTTPS_KEY_FILE="privkey.pem" + # (string) The Fully-Qualified Domain Name (FQDN) of the server that the HTTP # and/or HTTPS distributor(s) is/are publicly reachable at. SERVER_PUBLIC_FQDN = 'bridges.torproject.org' @@ -585,14 +668,17 @@ EMAIL_GPG_PASSPHRASE_FILE = None # Once a bridge is assigned to either of the first two groups, it stays there # persistently. The bridges are allocated to these groups in a proportion of # -# ``HTTPS_SHARE`` : ``EMAIL_SHARE`` : ``RESERVED_SHARE`` +# ``MOAT_SHARE`` : ``HTTPS_SHARE`` : ``EMAIL_SHARE`` : ``RESERVED_SHARE`` # ------------------------------------------------------------------------------ +# The proportion of bridges to allocate to Moat distribution. +MOAT_SHARE = 20 + # The proportion of bridges to allocate to HTTP distribution. HTTPS_SHARE = 10 # The proportion of bridges to allocate to Email distribution. -EMAIL_SHARE = 5 +EMAIL_SHARE = 2 # An integer specifying the proportion of bridges which should remain # unallocated, for backup usage and manual distribution. diff --git a/bridgedb/captcha.py b/bridgedb/captcha.py index ad89aea..b66972c 100644 --- a/bridgedb/captcha.py +++ b/bridgedb/captcha.py @@ -290,7 +290,6 @@ class GimpCaptcha(Captcha): if hmacIsValid: try: answerBlob = secretKey.decrypt(encBlob) - timestamp = answerBlob[:12].lstrip('0') then = cls.sched.nextIntervalStarts(int(timestamp)) now = int(time.time()) diff --git a/bridgedb/configure.py b/bridgedb/configure.py index 1a5e949..0bc4dd1 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -110,6 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", + "MOAT_CERT_FILE", "MOAT_KEY_FILE", "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", @@ -120,11 +121,18 @@ def loadConfig(configFile=None, configCls=None): else: setattr(config, attr, os.path.abspath(os.path.expanduser(setting))) - for attr in ["HTTPS_ROTATION_PERIOD", "EMAIL_ROTATION_PERIOD"]: + for attr in ["MOAT_ROTATION_PERIOD", + "HTTPS_ROTATION_PERIOD", + "EMAIL_ROTATION_PERIOD"]: setting = getattr(config, attr, None) # Default to None setattr(config, attr, setting) - for attr in ["IGNORE_NETWORKSTATUS", "CSP_ENABLED", "CSP_REPORT_ONLY", + for attr in ["IGNORE_NETWORKSTATUS", + "MOAT_CSP_ENABLED", + "MOAT_CSP_REPORT_ONLY", + "MOAT_CSP_INCLUDE_SELF", + "CSP_ENABLED", + "CSP_REPORT_ONLY", "CSP_INCLUDE_SELF"]: setting = getattr(config, attr, True) # Default to True setattr(config, attr, setting) diff --git a/bridgedb/distributors/__init__.py b/bridgedb/distributors/__init__.py index 951ef44..cc5f7a0 100644 --- a/bridgedb/distributors/__init__.py +++ b/bridgedb/distributors/__init__.py @@ -1,5 +1,2 @@ """Methods for distributing bridges.""" -import email -import https -#import moat diff --git a/bridgedb/distributors/common/__init__.py b/bridgedb/distributors/common/__init__.py new file mode 100644 index 0000000..fac0768 --- /dev/null +++ b/bridgedb/distributors/common/__init__.py @@ -0,0 +1,2 @@ +"""Common resources and utilities for bridge distribution systems.""" + diff --git a/bridgedb/distributors/common/http.py b/bridgedb/distributors/common/http.py new file mode 100644 index 0000000..86c26b8 --- /dev/null +++ b/bridgedb/distributors/common/http.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_distributors_common_http -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see included AUTHORS file +# :copyright: (c) 2017, The Tor Project, Inc. +# (c) 2017, Isis Lovecruft +# :license: see LICENSE for licensing information + +""" +.. py:module:: bridgedb.distributors.common.http + :synopsis: Common utilities for HTTP-based distributors. + +bridgedb.distributors.common.http +================================== + +Common utilities for HTTP-based distributors. +""" + +import logging +import os + +from bridgedb.parse.addr import isIPAddress + + +#: The fully-qualified domain name for any and all web servers we run. +SERVER_PUBLIC_FQDN = None + + +def setFQDN(fqdn, https=True): + """Set the global :data:`SERVER_PUBLIC_FQDN` variable. + + :param str fqdn: The public, fully-qualified domain name of the HTTP + server that will serve this resource. + :param bool https: If ``True``, then ``'https://'`` will be prepended to + the FQDN. This is primarily used to create a + ``Content-Security-Policy`` header that will only allow resources to + be sourced via HTTPS, otherwise, if ``False``, it allow resources to + be sourced via any transport protocol. + """ + if https: + fqdn = 'https://' + fqdn + + logging.info("Setting HTTP server public FQDN to %r" % fqdn) + + global SERVER_PUBLIC_FQDN + SERVER_PUBLIC_FQDN = fqdn + +def getFQDN(): + """Get the setting for the HTTP server's public FQDN from the global + :data:`SERVER_PUBLIC_FQDN variable. + + :rtype: str or None + """ + return SERVER_PUBLIC_FQDN + +def getClientIP(request, useForwardedHeader=False): + """Get the client's IP address from the ``'X-Forwarded-For:'`` + header, or from the :api:`request <twisted.web.server.Request>`. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. + :param bool useForwardedHeader: If ``True``, attempt to get the client's + IP address from the ``'X-Forwarded-For:'`` header. + :rtype: ``None`` or :any:`str` + :returns: The client's IP address, if it was obtainable. + """ + ip = None + + if useForwardedHeader: + header = request.getHeader("X-Forwarded-For") + if header: + ip = header.split(",")[-1].strip() + if not isIPAddress(ip): + logging.warn("Got weird X-Forwarded-For value %r" % header) + ip = None + else: + ip = request.getClientIP() + + return ip diff --git a/bridgedb/distributors/email/__init__.py b/bridgedb/distributors/email/__init__.py index 8f9ed03..5c0ef50 100644 --- a/bridgedb/distributors/email/__init__.py +++ b/bridgedb/distributors/email/__init__.py @@ -1,4 +1,20 @@ -"""Servers for BridgeDB's email bridge distributor.""" +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2012-2017 Isis Lovecruft +# (c) 2007-2017, The Tor Project, Inc. +# (c) 2007-2017, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +'''Package containing modules for BridgeDB's email bridge distributor. + +.. py:module:: bridgedb.distributors.email + :synopsis: Package containing modules for BridgeDB's email bridge + distributor. +''' import autoresponder import distributor diff --git a/bridgedb/distributors/email/distributor.py b/bridgedb/distributors/email/distributor.py index 06a0c50..b76c26c 100644 --- a/bridgedb/distributors/email/distributor.py +++ b/bridgedb/distributors/email/distributor.py @@ -228,3 +228,6 @@ class EmailDistributor(Distributor): # Since prepopulateRings is called every half hour when the bridge # descriptors are re-parsed, we should clean the database then. self.cleanDatabase() + + logging.info("Bridges allotted for %s distribution: %d" + % (self.name, len(self.hashring))) diff --git a/bridgedb/distributors/https/__init__.py b/bridgedb/distributors/https/__init__.py index 6c51a5f..7da8173 100644 --- a/bridgedb/distributors/https/__init__.py +++ b/bridgedb/distributors/https/__init__.py @@ -1,5 +1,5 @@ """Servers for BridgeDB's HTTPS bridge distributor.""" -import distributor -import request -import server +#import distributor +#import request +#import server diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py index 791229b..5ff9d83 100644 --- a/bridgedb/distributors/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -262,6 +262,17 @@ class HTTPSDistributor(Distributor): self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) + logging.info("Bridges allotted for %s distribution: %d" + % (self.name, len(self.hashring))) + + logging.info("\tNum bridges:\tFilter set:") + for (ringname, (filterFn, subring)) in self.hashring.filterRings.items(): + filterSet = ' '.join(self.hashring.extractFilterNames(ringname)) + logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) + + logging.info("Total subrings for %s: %d" + % (self.name, len(self.hashring.filterRings))) + def insert(self, bridge): """Assign a bridge to this distributor.""" self.hashring.insert(bridge) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index b756659..352a838 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -50,6 +50,9 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha +from bridgedb.distributors.common.http import setFQDN +from bridgedb.distributors.common.http import getFQDN +from bridgedb.distributors.common.http import getClientIP from bridgedb.distributors.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress @@ -60,8 +63,9 @@ from bridgedb.schedule import ScheduledInterval from bridgedb.util import replaceControlChars +#: The path to the HTTPS distributor's web templates. (Should be the +#: "templates" directory in the same directory as this file.) TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') -rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, # due to potential speed increases. This means that the atimes of the Mako @@ -76,61 +80,9 @@ lookup = TemplateLookup(directories=[TEMPLATE_DIR], collection_size=500) logging.debug("Set template root to %s" % TEMPLATE_DIR) -#: This server's public, fully-qualified domain name. -SERVER_PUBLIC_FQDN = None - - -def setFQDN(fqdn, https=True): - """Set the global :data:`SERVER_PUBLIC_FQDN` variable. - - :param str fqdn: The public, fully-qualified domain name of the HTTP - server that will serve this resource. - :param bool https: If ``True``, then ``'https://'`` will be prepended to - the FQDN. This is primarily used to create a - ``Content-Security-Policy`` header that will only allow resources to - be sourced via HTTPS, otherwise, if ``False``, it allow resources to - be sourced via any transport protocol. - """ - if https: - fqdn = 'https://' + fqdn - - logging.info("Setting HTTP server public FQDN to %r" % fqdn) - - global SERVER_PUBLIC_FQDN - SERVER_PUBLIC_FQDN = fqdn - -def getFQDN(): - """Get the setting for the HTTP server's public FQDN from the global - :data:`SERVER_PUBLIC_FQDN variable. - - :rtype: str or None - """ - return SERVER_PUBLIC_FQDN - -def getClientIP(request, useForwardedHeader=False): - """Get the client's IP address from the ``'X-Forwarded-For:'`` - header, or from the :api:`request <twisted.web.server.Request>`. - - :type request: :api:`twisted.web.http.Request` - :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. - :param bool useForwardedHeader: If ``True``, attempt to get the client's - IP address from the ``'X-Forwarded-For:'`` header. - :rtype: ``None`` or :any:`str` - :returns: The client's IP address, if it was obtainable. - """ - ip = None - - if useForwardedHeader: - header = request.getHeader("X-Forwarded-For") - if header: - ip = header.split(",")[-1].strip() - if not isIPAddress(ip): - logging.warn("Got weird X-Forwarded-For value %r" % header) - ip = None - else: - ip = request.getClientIP() +#: Localisations which BridgeDB supports which should be rendered right-to-left. +rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') - return ip def replaceErrorPage(request, error, template_name=None, html=True): """Create a general error page for displaying in place of tracebacks. @@ -334,6 +286,7 @@ class ErrorResource(CSPResource): render_POST = render_GET + resource404 = ErrorResource('error-404.html', code=404) resource500 = ErrorResource('error-500.html', code=500) maintenance = ErrorResource('error-503.html', code=503) diff --git a/bridgedb/distributors/moat/__init__.py b/bridgedb/distributors/moat/__init__.py new file mode 100644 index 0000000..221663b --- /dev/null +++ b/bridgedb/distributors/moat/__init__.py @@ -0,0 +1,4 @@ +"""Servers for BridgeDB's Moat bridge distributor.""" + +#import distributor +#import server diff --git a/bridgedb/distributors/moat/distributor.py b/bridgedb/distributors/moat/distributor.py new file mode 100644 index 0000000..a8f0e1d --- /dev/null +++ b/bridgedb/distributors/moat/distributor.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_moat_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> +# :copyright: (c) 2013-2017, Isis Lovecruft +# (c) 2013-2017, Matthew Finkel +# (c) 2007-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +""" +bridgedb.distributors.moat.distributor +========================== + +A Distributor that hands out bridges through a web interface. + +.. inheritance-diagram:: MoatDistributor + :parts: 1 +""" + +from bridgedb.distributors.https.distributor import HTTPSDistributor + +class MoatDistributor(HTTPSDistributor): + """A bridge distributor for Moat, a system which uses a JSON API to + provide a remote application with data necessary to the creation of a + user interface for distributing bridges. + + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. + :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. + """ + + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ + super(MoatDistributor, self).__init__(totalSubrings, key, proxies, + answerParameters) diff --git a/bridgedb/distributors/moat/request.py b/bridgedb/distributors/moat/request.py new file mode 100644 index 0000000..71ae3cd --- /dev/null +++ b/bridgedb/distributors/moat/request.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8; test-case-name: bridgedb.test.test_distributors_moat_request; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft <isis(a)torproject.org> 0xA3ADB67A2CDB8B35 +# :copyright: (c) 2007-2017, The Tor Project, Inc. +# (c) 2013-2017, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +""" +.. py:module:: bridgedb.distributors.moat.request + :synopsis: Classes for parsing and storing information about requests for + bridges which are sent to the moat distributor. + +bridgedb.distributors.moat.request +================================== + +Classes for parsing and storing information about requests for bridges +which are sent to the moat distributor. + +.. inheritance-diagram:: MoatBridgeRequest + +:: + + bridgedb.distributors.moat.request + | + |_ MoatBridgeRequest - A request for bridges which was received through + the moat distributor. + +.. +""" + +from __future__ import print_function + +import ipaddr +import logging +import re + +from bridgedb import bridgerequest +from bridgedb import geo +from bridgedb.parse import addr + + +#: A regular expression for matching the Pluggable Transport methodname in +#: HTTP GET request parameters. +TRANSPORT_REGEXP = "[_a-zA-Z][_a-zA-Z0-9]*" +TRANSPORT_PATTERN = re.compile(TRANSPORT_REGEXP) + +UNBLOCKED_REGEXP = "[a-zA-Z]{2}" +UNBLOCKED_PATTERN = re.compile(UNBLOCKED_REGEXP) + + +class MoatBridgeRequest(bridgerequest.BridgeRequestBase): + """We received a request for bridges through the moat distributor.""" + + def __init__(self, addClientCountryCode=False): + """Process a new bridge request received through the + :class:`~bridgedb.distributors.moat.distributor.MoatDistributor`. + + :param bool addClientCountryCode: If ``True``, then calling + :meth:`withoutBlockInCountry` will attempt to add the client's own + country code, geolocated from their IP, to the ``notBlockedIn`` + countries list. + """ + super(MoatBridgeRequest, self).__init__() + self.addClientCountryCode = addClientCountryCode + + def withIPversion(self): + """Determine if the request **parameters** were for bridges with IPv6 + addresses or not. + + .. note:: If the client's forwarded IP address was IPv6, then we assume + the client wanted IPv6 bridges. + """ + if addr.isIPAddress(self.client): + if self.client.version == 6: + logging.info("Moat request for bridges with IPv6 addresses.") + self.withIPv6() + + def withoutBlockInCountry(self, data): + """Determine which countries the bridges for this **request** should + not be blocked in. + + If :data:`addClientCountryCode` is ``True``, the the client's own + geolocated country code will be added to the to the + :data:`notBlockedIn` list. + + :param dict data: The decoded data from the JSON API request. + """ + countryCodes = data.get("unblocked", list()) + + for countryCode in countryCodes: + try: + country = UNBLOCKED_PATTERN.match(countryCode).group() + except (TypeError, AttributeError): + pass + else: + if country: + self.notBlockedIn.append(country.lower()) + logging.info("Moat request for bridges not blocked in: %r" + % country) + + if self.addClientCountryCode: + # Look up the country code of the input IP, and request bridges + # not blocked in that country. + if addr.isIPAddress(self.client): + country = geo.getCountryCode(ipaddr.IPAddress(self.client)) + if country: + self.notBlockedIn.append(country.lower()) + logging.info( + ("Moat client's bridges also shouldn't be blocked " + "in their GeoIP country code: %s") % country) + + def withPluggableTransportType(self, data): + """This request included a specific Pluggable Transport identifier. + + Add any Pluggable Transport methodname found in the JSON API + request field named "transport". + + :param dict data: The decoded data from the JSON API request. + """ + methodname = type('')(data.get("transport", "")) + + try: + transport = TRANSPORT_PATTERN.match(methodname).group() + except (TypeError, AttributeError): + pass + else: + if transport: + self.transports.append(transport) + logging.info("Moat request for transport type: %r" % transport) diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py new file mode 100644 index 0000000..daf62fa --- /dev/null +++ b/bridgedb/distributors/moat/server.py @@ -0,0 +1,776 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_distributors_moat_server -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: please see included AUTHORS file +# :copyright: (c) 2017, The Tor Project, Inc. +# (c) 2017, Isis Lovecruft +# :license: see LICENSE for licensing information + +""" +.. py:module:: bridgedb.distributors.moat.server + :synopsis: Server which implements JSON API to interface with Tor Browser + clients through a meek tunnel. + +bridgedb.distributors.moat.server +================================= + +Server which implements JSON API to interface with Tor Browser clients through a +meek tunnel. + +.. inheritance-diagram:: JsonAPIResource JsonAPIErrorResource CustomErrorHandlingResource JsonAPIDataResource CaptchaResource CaptchaCheckResource CaptchaFetchResource + :parts: 1 +""" + +from __future__ import print_function + +import base64 +import json +import logging +import time + +from functools import partial + +from ipaddr import IPAddress + +from twisted.internet import reactor +from twisted.internet.error import CannotListenError +from twisted.web import resource +from twisted.web.server import Site + +from bridgedb import captcha +from bridgedb import crypto +from bridgedb.distributors.common.http import setFQDN +from bridgedb.distributors.common.http import getFQDN +from bridgedb.distributors.common.http import getClientIP +from bridgedb.distributors.moat.request import MoatBridgeRequest +from bridgedb.qrcodes import generateQR +from bridgedb.schedule import Unscheduled +from bridgedb.schedule import ScheduledInterval +from bridgedb.util import replaceControlChars + + +#: The current version of the moat JSON API that we speak +MOAT_API_VERSION = '0.1.0' + +#: The root path to resources for the moat server +SERVER_PUBLIC_ROOT = None + +#: An ordered list of the preferred transports which moat should +#: distribute, in order from most preferable to least preferable. +TRANSPORT_PREFERENCE_LIST = None + +#: All of the pluggable transports BridgeDB currently supports. +SUPPORTED_TRANSPORTS = None + + +def getFQDNAndRoot(): + """Get the server's public FQDN plus the root directory for the web server. + """ + root = getRoot() + fqdn = getFQDN() + + if not root.startswith('/') and not fqdn.endswith('/'): + return '/'.join([fqdn, root]) + else: + return ''.join([fqdn, root]) + +def setRoot(root): + """Set the global :data:`SERVER_PUBLIC_ROOT` variable. + + :param str root: The path to the root directory for the web server. + """ + logging.info("Setting Moat server public root to %r" % root) + + global SERVER_PUBLIC_ROOT + SERVER_PUBLIC_ROOT = root + +def getRoot(): + """Get the setting for the HTTP server's public FQDN from the global + :data:`SERVER_PUBLIC_FQDN variable. + + :rtype: str or None + """ + return SERVER_PUBLIC_ROOT + +def setPreferredTransports(preferences): + """Set the global ``TRANSPORT_PREFERENCE_LIST``.""" + global TRANSPORT_PREFERENCE_LIST + TRANSPORT_PREFERENCE_LIST = preferences + +def getPreferredTransports(): + """Get the global ``TRANSPORT_PREFERENCE_LIST``. + + :rtype: list + :returns: A list of preferences for which pluggable transports to distribute + to moat clients. + """ + return TRANSPORT_PREFERENCE_LIST + +def setSupportedTransports(transports): + """Set the global ``SUPPORTED_TRANSPORTS``. + + :param dist transports: The ``SUPPORTED_TRANSPORTS`` dict from a + bridgedb.conf file. + """ + supported = [k for (k, v) in transports.items() if v] + + if not "vanilla" in supported: + supported.append("vanilla") + + global SUPPORTED_TRANSPORTS + SUPPORTED_TRANSPORTS = supported + +def getSupportedTransports(): + """Get the global ``SUPPORTED_TRANSPORTS``. + + :rtype: list + :returns: A list all pluggable transports we support. + """ + return SUPPORTED_TRANSPORTS + + +class JsonAPIResource(resource.Resource): + """A resource which conforms to the `JSON API spec <http://jsonapi.org/>`__. + """ + + def __init__(self, useForwardedHeader=True): + resource.Resource.__init__(self) + self.useForwardedHeader = useForwardedHeader + + def getClientIP(self, request): + """Get the client's IP address from the ``'X-Forwarded-For:'`` + header, or from the :api:`request <twisted.web.server.Request>`. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a + :api:`twisted.web.resource.Resource`. + :rtype: ``None`` or :any:`str` + :returns: The client's IP address, if it was obtainable. + """ + return getClientIP(request, self.useForwardedHeader) + + def formatDataForResponse(self, data, request): + """Format a dictionary of ``data`` into JSON and add necessary response + headers. + + This method will set the appropriate response headers: + * `Content-Type: application/vnd.api+json` + * `Server: moat/VERSION` + + :type data: dict + :param data: Some data to respond with. This will be formatted as JSON. + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. + :returns: The encoded data. + """ + request.responseHeaders.addRawHeader(b"Content-Type", b"application/vnd.api+json") + request.responseHeaders.addRawHeader(b"Server", b"moat/%s" % MOAT_API_VERSION) + + if data: + rendered = json.dumps(data) + else: + rendered = b"" + + return rendered + + +class JsonAPIErrorResource(JsonAPIResource): + """A JSON API resource which explains that some error has occured.""" + + isLeaf = True + + def __init__(self, id=0, type="", code=200, status="OK", detail=""): + """Create a :api:`twisted.web.resource.Resource` for a JSON API errors + object. + """ + resource.Resource.__init__(self) + self.id = id + self.type = type + self.code = code + self.status = status + self.detail = detail + + def render_GET(self, request): + # status codes and messages are at the JSON API layer, not HTTP layer: + data = { + 'errors': [{ + 'id': self.id, + 'type': self.type, + 'version': MOAT_API_VERSION, + 'code': self.code, + 'status': self.status, + 'detail': self.detail, + }] + } + return self.formatDataForResponse(data, request) + + render_POST = render_GET + + +resource403 = JsonAPIErrorResource(code=403, status="Forbidden") +resource406 = JsonAPIErrorResource(code=406, status="Not Acceptable") +resource415 = JsonAPIErrorResource(code=415, status="Unsupported Media Type") +resource419 = JsonAPIErrorResource(code=419, status="No You're A Teapot") +resource501 = JsonAPIErrorResource(code=501, status="Not Implemented") + + +class CustomErrorHandlingResource(resource.Resource): + """A :api:`twisted.web.resource.Resource` which wraps the + :api:`twisted.web.resource.Resource.getChild` method in order to use + custom error handling pages. + """ + def getChild(self, path, request): + logging.debug("[501] %s %s" % (request.method, request.uri)) + + response = resource501 + response.detail = "moat version %s does not implement %s %s" % \ + (MOAT_API_VERSION, request.method, request.uri) + return response + + +class JsonAPIDataResource(JsonAPIResource): + """A resource which returns some JSON API data.""" + + def __init__(self, useForwardedHeader=True): + JsonAPIResource.__init__(self, useForwardedHeader) + + def checkRequestHeaders(self, request): + """The JSON API specification requires servers to respond with certain HTTP + status codes and message if the client's request headers are inappropriate in + any of the following ways: + + * Servers MUST respond with a 415 Unsupported Media Type status code if + a request specifies the header Content-Type: application/vnd.api+json + with any media type parameters. + + * Servers MUST respond with a 406 Not Acceptable status code if a + request’s Accept header contains the JSON API media type and all + instances of that media type are modified with media type parameters. + """ + supports_json_api = False + accept_json_api_header = False + accept_header_is_ok = False + + if request.requestHeaders.hasHeader("Content-Type"): + headers = request.requestHeaders.getRawHeaders("Content-Type") + # The "pragma: no cover"s are because, no matter what I do, I cannot + # for the life of me trick twisted's test infrastructure to not send + # some variant of these headers. ¯\_(ツ)_/¯ + for contentType in headers: # pragma: no cover + # The request must have the Content-Type set to 'application/vnd.api+json': + if contentType == 'application/vnd.api+json': + supports_json_api = True + # The request must not specify a Content-Type with media parameters: + if ';' in contentType: + supports_json_api = False + + if not supports_json_api: + return resource415 + + # If the request has an Accept header which contains + # 'application/vnd.api+json' then at least one instance of that type + # must have no parameters: + if request.requestHeaders.hasHeader("Accept"): # pragma: no cover + headers = request.requestHeaders.getRawHeaders("Accept") + for accept in headers: + if accept.startswith('application/vnd.api+json'): + accept_json_api_header = True + if ';' not in accept: + accept_header_is_ok = True + + if accept_json_api_header and not accept_header_is_ok: # pragma: no cover + return resource406 + + +class CaptchaResource(JsonAPIDataResource): + """A CAPTCHA.""" + + def __init__(self, hmacKey=None, publicKey=None, secretKey=None, + useForwardedHeader=True): + JsonAPIDataResource.__init__(self, useForwardedHeader) + self.hmacKey = hmacKey + self.publicKey = publicKey + self.secretKey = secretKey + + +class CaptchaFetchResource(CaptchaResource): + """A resource to retrieve a CAPTCHA challenge.""" + + isLeaf = True + + def __init__(self, hmacKey=None, publicKey=None, secretKey=None, + captchaDir="captchas", useForwardedHeader=True): + """DOCDOC + + :param bytes hmacKey: The master HMAC key, used for validating CAPTCHA + challenge strings in :meth:`captcha.GimpCaptcha.check`. The file + where this key is stored can be set via the + ``GIMP_CAPTCHA_HMAC_KEYFILE`` option in the config file. + are stored. See the ``GIMP_CAPTCHA_DIR`` config setting. + :param str secretkey: A PKCS#1 OAEP-padded, private RSA key, used for + verifying the client's solution to the CAPTCHA. See + :func:`bridgedb.crypto.getRSAKey` and the + ``GIMP_CAPTCHA_RSA_KEYFILE`` config setting. + :param str publickey: A PKCS#1 OAEP-padded, public RSA key, used for + creating the ``captcha_challenge_field`` string to give to a + client. + :param str captchaDir: The directory where the cached CAPTCHA images + :param bool useForwardedHeader: If ``True``, obtain the client's IP + address from the ``X-Forwarded-For`` HTTP header. + """ + CaptchaResource.__init__(self, hmacKey, publicKey, secretKey, + useForwardedHeader) + self.captchaDir = captchaDir + self.supportedTransports = getSupportedTransports() + + def getCaptchaImage(self, request): + """Get a random CAPTCHA image from our **captchaDir**. + + Creates a :class:`~bridgedb.captcha.GimpCaptcha`, and calls its + :meth:`~bridgedb.captcha.GimpCaptcha.get` method to return a random + CAPTCHA and challenge string. + + :type request: :api:`twisted.web.http.Request` + :param request: A client's initial request for some other resource + which is protected by this one (i.e. protected by a CAPTCHA). + :returns: A 2-tuple of ``(image, challenge)``, where:: + - ``image`` is a string holding a binary, JPEG-encoded image. + - ``challenge`` is a unique string associated with the request. + """ + # Create a new HMAC key, specific to requests from this client: + clientIP = self.getClientIP(request) + clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) + capt = captcha.GimpCaptcha(self.publicKey, self.secretKey, + clientHMACKey, self.captchaDir) + try: + capt.get() + except captcha.GimpCaptchaError as error: + logging.debug(error) + except Exception as impossible: + logging.error("Unhandled error while retrieving Gimp captcha!") + logging.error(impossible) + + return (capt.image, capt.challenge) + + def getPreferredTransports(self, supportedTransports): + """Choose which transport a client should request, based on their list + of ``supportedTransports``. + + :param list supportedTransports: A list of transports the client + reported that they support (as returned from + :meth:`~bridgedb.distributors.moat.server.CaptchaFetchResource.extractSupportedTransports`). + :rtype: str or list + :returns: A string specifying the chosen transport, provided there is an + overlap between which transports BridgeDB and the client support. + Otherwise, if there is no overlap, returns a list of all the + transports which BridgeDB *does* support. + """ + preferenceOrder = getPreferredTransports() + preferred = None + + for pt in preferenceOrder: + if pt in supportedTransports: + preferred = pt + + # If we couldn't pick the best one that we both support, return the + # whole list of what we're able to distribute: + if not preferred: + preferred = getSupportedTransports() + + return preferred + + def extractSupportedTransports(self, request): + """Extract the transports a client supports from their POST request. + + :param str request: A JSON blob containing the following + fields: + * "version": The moat protocol version. + * "type": "client-transports". + * "supported": ['TRANSPORT', … ] + where: + * TRANSPORT is a string identifying a transport, e.g. "obfs3" or + "obfs4". Currently supported transport identifiers are: + "vanilla", "fte", "obfs3", "obfs4", "scramblesuit". + :rtype: list + :returns: The list of transports the client supports. + """ + supported = [] + + try: + encoded_data = request.content.read() + data = json.loads(encoded_data)["data"][0] + + if data["type"] != "client-transports": + raise ValueError( + "Bad JSON API object type: expected %s got %s" % + ('client-transports', data["type"])) + elif data["version"] != MOAT_API_VERSION: + raise ValueError( + "Client requested protocol version %s, but we're using %s" % + (data["version"], MOAT_API_VERSION)) + elif not data["supported"]: + raise ValueError( + "Client didn't provide any supported transports") + else: + supported = data["supported"] + except KeyError as err: + logging.debug(("Error processing client POST request: Client JSON " + "API data missing '%s' field") % (err)) + except ValueError as err: + logging.warn("Error processing client POST request: %s" % err) + except Exception as impossible: + logging.error("Unhandled error while extracting moat client transports!") + logging.error(impossible) + + return supported + + def render_POST(self, request): + """Retrieve a captcha from the moat API server and serve it to the client. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object for a CAPTCHA. + :rtype: str + :returns: A JSON blob containing the following fields: + * "version": The moat protocol version. + * "image": A base64-encoded CAPTCHA JPEG image. + * "challenge": A base64-encoded, encrypted challenge. The client + will need to hold on to the and pass it back later, along with + their challenge response. + * "error": An ASCII error message. + Any of the above JSON fields may be "null". + """ + error = self.checkRequestHeaders(request) + + if error: # pragma: no cover + return error.render(request) + + supported = self.extractSupportedTransports(request) + preferred = self.getPreferredTransports(supported) + image, challenge = self.getCaptchaImage(request) + + data = { + 'data': [{ + 'id': 1, + 'type': 'moat-challenge', + 'version': MOAT_API_VERSION, + 'transport': preferred, + 'image': image, + 'challenge': challenge, # The challenge is already base64-encoded + }] + } + + try: + data["data"][0]["image"] = base64.b64encode(image) + except Exception as impossible: + logging.error("Could not construct or encode captcha!") + logging.error(impossible) + + return self.formatDataForResponse(data, request) + + +class CaptchaCheckResource(CaptchaResource): + """A resource to verify a CAPTCHA solution and distribute bridges.""" + + isLeaf = True + + def __init__(self, distributor, schedule, N=1, + hmacKey=None, publicKey=None, secretKey=None, + useForwardedHeader=True): + """Create a new resource for checking CAPTCHA solutions and returning + bridges to a client. + + :type distributor: :class:`MoatDistributor` + :param distributor: The mechanism to retrieve bridges for this + distributor. + :type schedule: :class:`~bridgedb.schedule.ScheduledInterval` + :param schedule: The time period used to tweak the bridge selection + procedure. + :param int N: The number of bridges to hand out per query. + :param bool useForwardedHeader: Whether or not we should use the the + X-Forwarded-For header instead of the source IP address. + """ + CaptchaResource.__init__(self, hmacKey, publicKey, secretKey, + useForwardedHeader) + self.distributor = distributor + self.schedule = schedule + self.nBridgesToGive = N + self.useForwardedHeader = useForwardedHeader + + def getBridgeLines(self, ip, data): + """Get bridge lines for a client's HTTP request. + + :param str ip: The client's IP address. + :param dict data: The decoded JSON API data from the client's request. + :rtype: list or None + :returns: A list of bridge lines. + """ + bridgeLines = None + interval = self.schedule.intervalStart(time.time()) + + logging.debug("Replying to JSON API request from %s." % ip) + + if ip and data: + bridgeRequest = MoatBridgeRequest() + bridgeRequest.client = IPAddress(ip) + bridgeRequest.isValid(True) + bridgeRequest.withIPversion() + bridgeRequest.withPluggableTransportType(data) + bridgeRequest.withoutBlockInCountry(data) + bridgeRequest.generateFilters() + + bridges = self.distributor.getBridges(bridgeRequest, interval) + bridgeLines = [replaceControlChars(bridge.getBridgeLine(bridgeRequest)) + for bridge in bridges] + + return bridgeLines + + def extractClientSolution(self, data): + """Extract the client's CAPTCHA solution from a POST request. + + This is used after receiving a POST request from a client (which + should contain their solution to the CAPTCHA), to extract the solution + and challenge strings. + + :param dict data: The decoded JSON API data from the client's request. + :returns: A redirect for a request for a new CAPTCHA if there was a + problem. Otherwise, returns a 2-tuple of strings, the first is the + client's CAPTCHA solution from the text input area, and the second + is the challenge string. + """ + qrcode = False + transport = None + challenge, solution = None, None + + try: + if data["type"] != "moat-solution": + raise ValueError( + "Bad JSON API object type: expected %s got %s" % + ("moat-solution", data["type"])) + elif data["id"] != 2: + raise ValueError( + "Bad JSON API data id: expected 2 got %s" % + (data["id"])) + elif data["version"] != MOAT_API_VERSION: + raise ValueError( + "Client requested protocol version %s, but we're using %s" % + (data["version"], MOAT_API_VERSION)) + elif data["transport"] not in getSupportedTransports(): + raise ValueError( + "Transport '%s' is not currently supported" % + data["transport"]) + else: + qrcode = data["qrcode"] + transport = type('')(data["transport"]) + challenge = type('')(data["challenge"]) + solution = type('')(data["solution"]) + except KeyError as err: + logging.warn(("Error processing client POST request: " + "Client JSON API data missing '%s' field.") % err) + except ValueError as err: + logging.warn("Error processing client POST request: %s" % err.message) + except Exception as impossible: + logging.error(impossible) + + return (qrcode, transport, challenge, solution) + + def checkSolution(self, challenge, solution, clientIP): + """Process a solved CAPTCHA via + :meth:`bridgedb.captcha.GimpCaptcha.check`. + + :param str challenge: A base64-encoded, encrypted challenge. + :param str solution: The client's solution to the captcha + :param str clientIP: The client's IP address. + :rtupe: bool + :returns: True, if the CAPTCHA solution was valid; False otherwise. + """ + valid = False + clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) + + try: + valid = captcha.GimpCaptcha.check(challenge, solution, + self.secretKey, clientHMACKey) + except Exception as impossible: + logging.error(impossible) + raise impossible + finally: + logging.debug("%sorrect captcha from %r: %r." % + ("C" if valid else "Inc", clientIP, solution)) + + return valid + + def failureResponse(self, id, request): + """Respond with status code "419 No You're A Teapot".""" + error_response = resource419 + error_response.type = 'moat-bridges' + + if id == 4: + error_response.id = 4 + error_response.detail = "The CAPTCHA solution was incorrect." + elif id == 5: + error_response.id = 5 + error_response.detail = "The CAPTCHA challenge timed out." + + return error_response.render(request) + + def render_POST(self, request): + """Process a client's CAPTCHA solution. + + If the client's CAPTCHA solution is valid (according to + :meth:`checkSolution`), process and serve their original + request. Otherwise, redirect them back to a new CAPTCHA page. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object, including POST arguments which + should include two key/value pairs: one key being + ``'captcha_challenge_field'``, and the other, + ``'captcha_response_field'``. These POST arguments should be + obtained from :meth:`render_GET`. + :rtype: str + :returns: A rendered HTML page containing a ReCaptcha challenge image + for the client to solve. + """ + valid = False + error = self.checkRequestHeaders(request) + + if error: # pragma: no cover + return error.render(request) + + data = { + "data": [{ + "id": 3, + "type": 'moat-bridges', + "version": MOAT_API_VERSION, + "bridges": None, + "qrcode": None, + }] + } + + try: + encoded_client_data = request.content.read() + client_data = json.loads(encoded_client_data)["data"][0] + clientIP = self.getClientIP(request) + + (include_qrcode, transport, + challenge, solution) = self.extractClientSolution(client_data) + + valid = self.checkSolution(challenge, solution, clientIP) + except captcha.CaptchaExpired: + logging.debug("The challenge had timed out") + return self.failureResponse(5, request) + except Exception as impossible: + logging.warn("Unhandled exception while processing a POST /fetch request!") + logging.error(impossible) + return self.failureResponse(4, request) + + if valid: + qrcode = None + bridgeLines = self.getBridgeLines(clientIP, client_data) + + if include_qrcode: + qrjpeg = generateQR(bridgeLines) + if qrjpeg: + qrcode = 'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + + data["data"][0]["qrcode"] = qrcode + data["data"][0]["bridges"] = bridgeLines + + return self.formatDataForResponse(data, request) + else: + return self.failureResponse(4, request) + + +def addMoatServer(config, distributor): + """Set up a web server for moat bridge distribution. + + :type config: :class:`bridgedb.persistent.Conf` + :param config: A configuration object from + :mod:`bridgedb.main`. Currently, we use these options:: + GIMP_CAPTCHA_DIR + SERVER_PUBLIC_FQDN + SUPPORTED_TRANSPORTS + MOAT_DIST + MOAT_DIST_VIA_MEEK_ONLY + MOAT_TLS_CERT_FILE + MOAT_TLS_KEY_FILE + MOAT_SERVER_PUBLIC_ROOT + MOAT_HTTPS_IP + MOAT_HTTPS_PORT + MOAT_HTTP_IP + MOAT_HTTP_PORT + MOAT_BRIDGES_PER_ANSWER + MOAT_TRANSPORT_PREFERENCE_LIST + MOAT_USE_IP_FROM_FORWARDED_HEADER + MOAT_ROTATION_PERIOD + MOAT_GIMP_CAPTCHA_HMAC_KEYFILE + MOAT_GIMP_CAPTCHA_RSA_KEYFILE + :type distributor: :class:`bridgedb.distributors.moat.distributor.MoatDistributor` + :param distributor: A bridge distributor. + :raises SystemExit: if the servers cannot be started. + :rtype: :api:`twisted.web.server.Site` + :returns: A webserver. + """ + captcha = None + fwdHeaders = config.MOAT_USE_IP_FROM_FORWARDED_HEADER + numBridges = config.MOAT_BRIDGES_PER_ANSWER + + logging.info("Starting moat servers...") + + setFQDN(config.SERVER_PUBLIC_FQDN) + setRoot(config.MOAT_SERVER_PUBLIC_ROOT) + setSupportedTransports(config.SUPPORTED_TRANSPORTS) + setPreferredTransports(config.MOAT_TRANSPORT_PREFERENCE_LIST) + + # Get the master HMAC secret key for CAPTCHA challenges, and then + # create a new HMAC key from it for use on the server. + captchaKey = crypto.getKey(config.MOAT_GIMP_CAPTCHA_HMAC_KEYFILE) + hmacKey = crypto.getHMAC(captchaKey, "Moat-Captcha-Key") + # Load or create our encryption keys: + secretKey, publicKey = crypto.getRSAKey(config.MOAT_GIMP_CAPTCHA_RSA_KEYFILE) + sched = Unscheduled() + + if config.MOAT_ROTATION_PERIOD: + count, period = config.MOAT_ROTATION_PERIOD.split() + sched = ScheduledInterval(count, period) + + sitePublicDir = getRoot() + + meek = CustomErrorHandlingResource() + moat = CustomErrorHandlingResource() + fetch = CaptchaFetchResource(hmacKey, publicKey, secretKey, + config.GIMP_CAPTCHA_DIR, fwdHeaders) + check = CaptchaCheckResource(distributor, sched, numBridges, + hmacKey, publicKey, secretKey, fwdHeaders) + + moat.putChild("fetch", fetch) + moat.putChild("check", check) + meek.putChild("moat", moat) + + root = CustomErrorHandlingResource() + root.putChild("meek", meek) + + site = Site(root) + site.displayTracebacks = False + + if config.MOAT_HTTP_PORT: # pragma: no cover + ip = config.MOAT_HTTP_IP or "" + port = config.MOAT_HTTP_PORT or 80 + try: + reactor.listenTCP(port, site, interface=ip) + except CannotListenError as error: + raise SystemExit(error) + logging.info("Started Moat HTTP server on %s:%d" % (str(ip), int(port))) + + if config.MOAT_HTTPS_PORT: # pragma: no cover + ip = config.MOAT_HTTPS_IP or "" + port = config.MOAT_HTTPS_PORT or 443 + try: + from twisted.internet.ssl import DefaultOpenSSLContextFactory + factory = DefaultOpenSSLContextFactory(config.MOAT_TLS_KEY_FILE, + config.MOAT_TLS_CERT_FILE) + reactor.listenSSL(port, site, factory, interface=ip) + except CannotListenError as error: + raise SystemExit(error) + logging.info("Started Moat TLS server on %s:%d" % (str(ip), int(port))) + + return site diff --git a/bridgedb/main.py b/bridgedb/main.py index 1601cf1..905af2d 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -33,6 +33,7 @@ from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig from bridgedb.distributors.email.distributor import EmailDistributor from bridgedb.distributors.https.distributor import HTTPSDistributor +from bridgedb.distributors.moat.distributor import MoatDistributor from bridgedb.parse import descriptors from bridgedb.parse.blacklist import parseBridgeBlacklistFile @@ -220,9 +221,10 @@ def createBridgeRings(cfg, proxyList, key): known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an + :returns: A :class:`~bridgedb.Bridges.BridgeSplitter` hashring, an :class:`~bridgedb.distributors.https.distributor.HTTPSDistributor` or None, and an - :class:`~bridgedb.distributors.email.distributor.EmailDistributor` or None. + :class:`~bridgedb.distributors.email.distributor.EmailDistributor` or None, and an + :class:`~bridgedb.distributors.moat.distributor.MoatDistributor` or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -233,7 +235,18 @@ def createBridgeRings(cfg, proxyList, key): ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS, needFlags=cfg.FORCE_FLAGS) - emailDistributor = ipDistributor = None + emailDistributor = ipDistributor = moatDistributor = None + + # As appropriate, create a Moat distributor. + if cfg.MOAT_DIST and cfg.MOAT_SHARE: + logging.debug("Setting up Moat Distributor...") + moatDistributor = MoatDistributor( + cfg.MOAT_N_IP_CLUSTERS, + crypto.getHMAC(key, "Moat-Dist-Key"), + proxyList, + answerParameters=ringParams) + hashring.addRing(moatDistributor.hashring, "moat", cfg.MOAT_SHARE) + # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") @@ -265,7 +278,7 @@ def createBridgeRings(cfg, proxyList, key): for pseudoRing in cfg.FILE_BUCKETS.keys(): hashring.addPseudoRing(pseudoRing) - return hashring, emailDistributor, ipDistributor + return hashring, emailDistributor, ipDistributor, moatDistributor def run(options, reactor=reactor): """This is BridgeDB's main entry point and main runtime loop. @@ -323,12 +336,14 @@ def run(options, reactor=reactor): from bridgedb.distributors.email.server import addServer as addSMTPServer from bridgedb.distributors.https.server import addWebServer + from bridgedb.distributors.moat.server import addMoatServer # Load the master key, or create a new one. key = crypto.getKey(config.MASTER_KEY_FILE) proxies = proxy.ProxySet() emailDistributor = None ipDistributor = None + moatDistributor = None # Save our state state.proxies = proxies @@ -388,7 +403,8 @@ def run(options, reactor=reactor): logging.info("Reparsing bridge descriptors...") (hashring, emailDistributorTmp, - ipDistributorTmp) = createBridgeRings(cfg, state.proxies, key) + ipDistributorTmp, + moatDistributorTmp) = createBridgeRings(cfg, state.proxies, key) logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. @@ -398,33 +414,19 @@ def run(options, reactor=reactor): if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings - logging.info("Bridges allotted for %s distribution: %d" - % (emailDistributorTmp.name, - len(emailDistributorTmp.hashring))) else: logging.warn("No email distributor created!") if ipDistributorTmp is not None: ipDistributorTmp.prepopulateRings() # create default rings - - logging.info("Bridges allotted for %s distribution: %d" - % (ipDistributorTmp.name, - len(ipDistributorTmp.hashring))) - logging.info("\tNum bridges:\tFilter set:") - - nSubrings = 0 - ipSubrings = ipDistributorTmp.hashring.filterRings - for (ringname, (filterFn, subring)) in ipSubrings.items(): - nSubrings += 1 - filterSet = ' '.join( - ipDistributorTmp.hashring.extractFilterNames(ringname)) - logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) - - logging.info("Total subrings for %s: %d" - % (ipDistributorTmp.name, nSubrings)) else: logging.warn("No HTTP(S) distributor created!") + if moatDistributorTmp is not None: + moatDistributorTmp.prepopulateRings() + else: + logging.warn("No Moat distributor created!") + # Dump bridge pool assignments to disk. try: logging.debug("Dumping pool assignments to file: '%s'" @@ -443,6 +445,9 @@ def run(options, reactor=reactor): if inThread: # XXX shutdown the distributors if they were previously running # and should now be disabled + if moatDistributorTmp: + reactor.callFromThread(replaceBridgeRings, + moatDistributor, moatDistributorTmp) if ipDistributorTmp: reactor.callFromThread(replaceBridgeRings, ipDistributor, ipDistributorTmp) @@ -452,7 +457,7 @@ def run(options, reactor=reactor): else: # We're still starting up. Return these distributors so # they are configured in the outer-namespace - return emailDistributorTmp, ipDistributorTmp + return emailDistributorTmp, ipDistributorTmp, moatDistributorTmp global _reloadFn _reloadFn = reload @@ -461,9 +466,11 @@ def run(options, reactor=reactor): if reactor: # pragma: no cover # And actually load it to start parsing. Get back our distributors. - emailDistributor, ipDistributor = reload(False) + emailDistributor, ipDistributor, moatDistributor = reload(False) # Configure all servers: + if config.MOAT_DIST and config.MOAT_SHARE: + addMoatServer(config, moatDistributor) if config.HTTPS_DIST and config.HTTPS_SHARE: addWebServer(config, ipDistributor) if config.EMAIL_DIST and config.EMAIL_SHARE: diff --git a/bridgedb/test/https_helpers.py b/bridgedb/test/https_helpers.py index d54e544..fad841e 100644 --- a/bridgedb/test/https_helpers.py +++ b/bridgedb/test/https_helpers.py @@ -347,9 +347,8 @@ class DummyRequest(RequestHelperDummyRequest): self.content = io.StringIO() self.headers = {} # Needed for Twisted>14.0.2 - #self.outgoingHeaders = {} - #self.responseHeaders = Headers() - #self.requestHeaders = Headers() + self.responseHeaders = Headers() + self.requestHeaders = Headers() def writeContent(self, data): """Add some **data** to the faked body of this request. diff --git a/bridgedb/test/moat_helpers.py b/bridgedb/test/moat_helpers.py new file mode 100644 index 0000000..c7f7718 --- /dev/null +++ b/bridgedb/test/moat_helpers.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2017, Isis Lovecruft +# (c) 2007-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information + + +"""Helpers for testing the HTTPS Distributor and its servers.""" + + +import io + +from bridgedb.persistent import Conf + +from . import util + + +GIMP_CAPTCHA_DIR = 'captchas' +SERVER_PUBLIC_FQDN = 'bridges.torproject.org' +SUPPORTED_TRANSPORTS = { + 'obfs2': False, + 'obfs3': True, + 'obfs4': True, + 'scramblesuit': True, + 'fte': True, +} +MOAT_DIST = True +MOAT_DIST_VIA_MEEK_ONLY = True +MOAT_TLS_CERT_FILE="moat-tls.crt" +MOAT_TLS_KEY_FILE="moat-tls.pem" +if MOAT_DIST_VIA_MEEK_ONLY: + MOAT_SERVER_PUBLIC_ROOT = '/meek/moat' +else: + MOAT_SERVER_PUBLIC_ROOT = '/moat' +MOAT_BRIDGES_PER_ANSWER = 3 +MOAT_TRANSPORT_PREFERENCE_LIST = ["obfs4", "vanilla"] +MOAT_HTTPS_IP = '127.0.0.1' +MOAT_HTTPS_PORT = None +MOAT_HTTP_IP = None +MOAT_HTTP_PORT = None +MOAT_USE_IP_FROM_FORWARDED_HEADER = True +MOAT_N_IP_CLUSTERS = 4 +MOAT_ROTATION_PERIOD = "3 hours" +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = 'moat_captcha_hmac_key' +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = 'moat_captcha_rsa_key' + +TEST_CONFIG_FILE = io.StringIO(unicode("""\ +GIMP_CAPTCHA_DIR = %r +SERVER_PUBLIC_FQDN = %r +SUPPORTED_TRANSPORTS = %r +MOAT_DIST = %r +MOAT_DIST_VIA_MEEK_ONLY = %r +MOAT_TLS_CERT_FILE = %r +MOAT_TLS_KEY_FILE = %r +MOAT_SERVER_PUBLIC_ROOT = %r +MOAT_HTTPS_IP = %r +MOAT_HTTPS_PORT = %r +MOAT_HTTP_IP = %r +MOAT_HTTP_PORT = %r +MOAT_BRIDGES_PER_ANSWER = %r +MOAT_TRANSPORT_PREFERENCE_LIST = %r +MOAT_USE_IP_FROM_FORWARDED_HEADER = %r +MOAT_N_IP_CLUSTERS = %r +MOAT_ROTATION_PERIOD = %r +MOAT_GIMP_CAPTCHA_HMAC_KEYFILE = %r +MOAT_GIMP_CAPTCHA_RSA_KEYFILE = %r +""" % (GIMP_CAPTCHA_DIR, + SERVER_PUBLIC_FQDN, + SUPPORTED_TRANSPORTS, + MOAT_DIST, + MOAT_DIST_VIA_MEEK_ONLY, + MOAT_TLS_CERT_FILE, + MOAT_TLS_KEY_FILE, + MOAT_SERVER_PUBLIC_ROOT, + MOAT_HTTPS_IP, + MOAT_HTTPS_PORT, + MOAT_HTTP_IP, + MOAT_HTTP_PORT, + MOAT_BRIDGES_PER_ANSWER, + MOAT_TRANSPORT_PREFERENCE_LIST, + MOAT_USE_IP_FROM_FORWARDED_HEADER, + MOAT_N_IP_CLUSTERS, + MOAT_ROTATION_PERIOD, + MOAT_GIMP_CAPTCHA_HMAC_KEYFILE, + MOAT_GIMP_CAPTCHA_RSA_KEYFILE))) + +def _createConfig(configFile=TEST_CONFIG_FILE): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(configFile.read(), '<string>', 'exec') + exec compiled in configuration + config = Conf(**configuration) + return config + + +class DummyMoatDistributor(object): + """A mocked :class:`bridgedb.distributors.moat.distributor.MoatDistributor` + which is used to test + :class:`bridgedb.distributors.moat.server.CaptchaFetchResource`. + """ + _bridge_class = util.DummyBridge + _bridgesPerResponseMin = 3 + + def getBridges(self, bridgeRequest=None, epoch=None): + """Needed because it's called in + :meth:`BridgesResource.getBridgeRequestAnswer`.""" + return [self._bridge_class() for _ in range(self._bridgesPerResponseMin)] diff --git a/bridgedb/test/test_distributors_common_http.py b/bridgedb/test/test_distributors_common_http.py new file mode 100644 index 0000000..60759e4 --- /dev/null +++ b/bridgedb/test/test_distributors_common_http.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.common.http`.""" + +from __future__ import print_function + +import logging +import os + +from twisted.trial import unittest +from twisted.web.test import requesthelper + +from bridgedb.distributors.common import http as server + +from bridgedb.test.https_helpers import DummyRequest + + +# For additional logger output for debugging, comment out the following: +logging.disable(50) +# and then uncomment the following line: +#server.logging.getLogger().setLevel(10) + + +class SetFQDNTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.https.server.setFQDN` and + :func:`bridgedb.distributors.https.server.setFQDN`. + """ + + def setUp(self): + self.originalFQDN = server.SERVER_PUBLIC_FQDN + + def tearDown(self): + server.SERVER_PUBLIC_FQDN = self.originalFQDN + + def test_setFQDN_https(self): + """Calling ``server.setFQDN([…], https=True)`` should prepend + ``"https://"`` to the module :data:`server.SERVER_PUBLIC_FQDN` + variable. + """ + server.setFQDN('example.com', https=True) + self.assertEqual(server.SERVER_PUBLIC_FQDN, "https://example.com") + + def test_setFQDN_http(self): + """Calling ``server.setFQDN([…], https=False)`` should not prepend + anything at all to the module :data:`server.SERVER_PUBLIC_FQDN` + variable. + """ + server.setFQDN('example.com', https=False) + self.assertEqual(server.SERVER_PUBLIC_FQDN, "example.com") + + +class GetClientIPTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.https.server.getClientIP`.""" + + def createRequestWithIPs(self): + """Set the IP address returned from ``request.getClientIP()`` to + '3.3.3.3', and the IP address reported in the 'X-Forwarded-For' header + to '2.2.2.2'. + """ + request = DummyRequest(['']) + request.headers.update({'x-forwarded-for': '2.2.2.2'}) + # See :api:`twisted.test.requesthelper.DummyRequest.getClientIP` + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.method = b'GET' + return request + + def test_getClientIP_XForwardedFor(self): + """getClientIP() should return the IP address from the + 'X-Forwarded-For' header when ``useForwardedHeader=True``. + """ + request = self.createRequestWithIPs() + clientIP = server.getClientIP(request, useForwardedHeader=True) + self.assertEqual(clientIP, '2.2.2.2') + + def test_getClientIP_XForwardedFor_bad_ip(self): + """getClientIP() should return None if the IP address from the + 'X-Forwarded-For' header is bad/invalid and + ``useForwardedHeader=True``. + """ + request = self.createRequestWithIPs() + request.headers.update({'x-forwarded-for': 'pineapple'}) + clientIP = server.getClientIP(request, useForwardedHeader=True) + self.assertEqual(clientIP, None) + + def test_getClientIP_fromRequest(self): + """getClientIP() should return the IP address from the request instance + when ``useForwardedHeader=False``. + """ + request = self.createRequestWithIPs() + clientIP = server.getClientIP(request) + self.assertEqual(clientIP, '3.3.3.3') diff --git a/bridgedb/test/test_distributors_moat_request.py b/bridgedb/test/test_distributors_moat_request.py new file mode 100644 index 0000000..ed7f493 --- /dev/null +++ b/bridgedb/test/test_distributors_moat_request.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.moat.request`.""" + +from __future__ import print_function + +from twisted.trial import unittest + +from bridgedb.distributors.moat import request + + +class MoatBridgeRequest(unittest.TestCase): + """Unittests for :class:`bridgedb.distributors.moat.request.MoatBridgeRequest`.""" + + def setUp(self): + self.bridgeRequest = request.MoatBridgeRequest() + + def test_withoutBlockInCountry(self): + data = {'unblocked': ['us', 'ir', 'sy']} + + self.bridgeRequest.withoutBlockInCountry(data) + self.bridgeRequest.generateFilters() + + self.assertItemsEqual(['byTransportNotBlockedIn(None,us,4)', + 'byTransportNotBlockedIn(None,ir,4)', + 'byTransportNotBlockedIn(None,sy,4)'], + [x.__name__ for x in self.bridgeRequest.filters]) + + def test_withoutBlockInCountry_not_a_valid_country_code(self): + data = {'unblocked': ['3']} + self.bridgeRequest.withoutBlockInCountry(data) + + def test_withoutBlockInCountry_unicode(self): + data = {'unblocked': ['föö']} + self.bridgeRequest.withoutBlockInCountry(data) + + def test_withoutBlockInCountry_not_a_valid_transport(self): + data = {'unblocked': ['3']} + self.bridgeRequest.withPluggableTransportType(data) + + def test_withPluggableTransportType_unicode(self): + data = {'transport': 'bifröst'} + self.bridgeRequest.withPluggableTransportType(data) diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py new file mode 100644 index 0000000..2145541 --- /dev/null +++ b/bridgedb/test/test_distributors_moat_server.py @@ -0,0 +1,918 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2014-2017, Isis Lovecruft +# (c) 2014-2017, The Tor Project, Inc. +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.distributors.moat.server`.""" + +from __future__ import print_function + +import base64 +import io +import json +import logging +import os +import shutil +import tempfile + +from twisted.trial import unittest +from twisted.web.resource import Resource +from twisted.web.test import requesthelper + +from bridgedb import crypto +from bridgedb.distributors.moat import server +from bridgedb.schedule import ScheduledInterval + +from bridgedb.test.moat_helpers import _createConfig +from bridgedb.test.moat_helpers import DummyMoatDistributor +from bridgedb.test.https_helpers import DummyRequest +from bridgedb.test.util import DummyBridge +from bridgedb.test.util import DummyMaliciousBridge + + +# For additional logger output for debugging, comment out the following: +logging.disable(50) +# and then uncomment the following line: +#server.logging.getLogger().setLevel(10) + + +# These keys are hardcoded so that we can test both expired and +# unexpired CAPTCHAs which were/are produced with them. +CAPTCHA_KEY = base64.b64decode("tFh0hkskhDBulvBtYkYy7qlQhyZh2MKsIOfaAmmvinQ=") +HMAC_KEY = base64.b64decode("vJPta7PflEb/qalt5Klgn9wyfgs=") + +SECRET_KEY = crypto.PKCS1_OAEP.new(crypto.RSA.importKey("""\ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAtz551eYMzSE8a56TGFRt4+bhZOgVCShBjaXv0LfuFtF4KXhv +cMXG8Zo9jw0M8HCk9YLhb40bLaeSxemcoaHiuC/zxgL8ECQ7GMVO6vi409UWRnl7 +VpCqyFPXg+V89TM67IsMejT8CDG4g62DYgSQ0Fpl5AsFd3RN2mQ38qdrmoBTXxQt +cmMXkEgDN20ZoVRFbWa+KurX9pv74wtNqUY2uUMxyGwXIrBfi6O3cyyjUAoSXsJT +iuKmgcuzE9+fDKzMftEf+k8OSs/DQWYARefQBCTndzha9ICwxFM5L3CWsPQxKtS4 +mkILRGLZSHYeswynwH98Swimgyv2FSqOZN9eYQIDAQABAoIBAQC0S1JQ9RKvWf46 +3UFZdOjSjb5DLF5WLjehiR0WPYKTDPKvywHK8a221c2vzGVoxUxpC6eHvEx7dR9i +f2JPXhrWose1kgY0U5GZ47isVKB2PHi4SpriJ2EBzgyEh+2UzB0z0/Qo4a0A2vrz +BGv6qwdZGTibUYTFbbeUI3sw0y16SxHPeGaYtfyLcx5/Nwd92V9NZ1pyaivVt4Uu +XgHnYxG7Y1uurSv6sdPouUN+H1o9msBm6EMRXjDrCAyVr0gmH2KQmnWedwSiY+eh +n5hP8jkOEQntxirMM5AR8N/KSXJdt9nccRPbamInn4eK1wlOzJJzwLpu8hhwJ+fc +6dWfBP9hAoGBAMxiSDtYqai6hTongIyKasycm14F0Xzcd5dVBwW1ANLdWbC32ti9 +n7i6TD7Fe2vfmiNpylMLbYLt5zODHT3eb3l1cWWpvkrMmxkvfj3to9sJfCg2p+jt +WzxugmSu9qO03C9nhSsFlD2Li754nPVCa5gk/1Gnxt9+RRI2Qzb7r1GFAoGBAOWF +eHJAO2SmZBc9doRqUgQXcZin/60f9PpxPqHtT8NzBItSxPSlkGYspDJaxPZfl5Y1 +7MZMNN6vBTitKXO3JXYZwCD+Y8s+trd/qm/vbco2SinWlgRlZyqcyBCvGKfBMUg/ +Nr8mRPyZ18yBEj5OtYadJcRCERk3ERWSt/ndLwItAoGAC/20KS8xfQG8cUYCB7zT +OT/y6ZhDyySQK6PEbrRI4RY1feW7hD3T0h2z/XbOn+yVeYBqa2bfPPBCQUZu/8M+ +HQ0j4wgLbw4EB30+1dlMZLxwuVdDkKnkUW5WXhvZwo8I4AsdyAFiyh2WzEz9QHJu +J5X8GMlUJKae3MusM9yeU5UCgYEAw8unYzd+My9qZRTunKkiTBE/u61dA/AmCNtA +Rdxu1dmxf7TdBaKTW0Yr0DT0nwQPCXn5AXSTCYAeoSm/GdKb53KyHrNEqGZYcpM6 +7wA+FWlYvPYsxZVHe+eBGBJ2ouzAwNQEPO5FnYMTv4Y/7N0yJ6K5TAHcGjmKnm+p ++EICTwUCgYEArF7zfcaxRXQtNKKEIYR9Q+zL/+fQEF2lxre61UNxpS0CmDrrAZwu +oa7cfLxocTUpYp7atUINzwVQgVd6AWta3v/PoXhkxo/CRd7pheyeH/ypFa7vAkzO +zZAlI9uLR/7XevId2W7b8U+8AWtxi3RLXSId9QzGRvjkoThBAKfLM30= +-----END RSA PRIVATE KEY-----""")) + +PUBLIC_KEY = crypto.PKCS1_OAEP.new(crypto.RSA.importKey("""\ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtz551eYMzSE8a56TGFRt +4+bhZOgVCShBjaXv0LfuFtF4KXhvcMXG8Zo9jw0M8HCk9YLhb40bLaeSxemcoaHi +uC/zxgL8ECQ7GMVO6vi409UWRnl7VpCqyFPXg+V89TM67IsMejT8CDG4g62DYgSQ +0Fpl5AsFd3RN2mQ38qdrmoBTXxQtcmMXkEgDN20ZoVRFbWa+KurX9pv74wtNqUY2 +uUMxyGwXIrBfi6O3cyyjUAoSXsJTiuKmgcuzE9+fDKzMftEf+k8OSs/DQWYARefQ +BCTndzha9ICwxFM5L3CWsPQxKtS4mkILRGLZSHYeswynwH98Swimgyv2FSqOZN9e +YQIDAQAB +-----END PUBLIC KEY-----""")) + + +class MiscellaneousTests(unittest.TestCase): + """Tests for helper functions in :mod:`bridgedb.distributors.moat.server`.""" + + def setUp(self): + self.config = _createConfig() + + def test_setRoot(self): + """If we call `setRoot()` with `root="/meek/moat"` then the public + root directory of the server should be "/meek/moat". + """ + server.setRoot("/meek/moat") + self.assertEqual(server.getRoot(), "/meek/moat") + + def test_getFQDNAndRoot(self): + """If the FQDN is set to "bridges.torproject.org" and we call `setRoot()` + with `root="/meek/moat"` then the public root directory of the server + should be "/meek/moat". + """ + server.setFQDN("bridges.torproject.org", https=True) + server.setRoot("/meek/moat") + + self.assertEqual(server.getFQDNAndRoot(), + "https://bridges.torproject.org/meek/moat") + + def test_getFQDNAndRoot_no_slash(self): + """If the FQDN is set to "bridges.torproject.org" and we call `setRoot()` + with `root="meek/moat"` then the public root directory of the server + should be "bridges.torproject.org/meek/moat". + """ + server.setFQDN("bridges.torproject.org", https=True) + server.setRoot("meek/moat") # missing the "/" prefix + + self.assertEqual(server.getFQDNAndRoot(), + "https://bridges.torproject.org/meek/moat") + + def test_setPreferredTransports(self): + """Setting the pluggable transport preference list to ["dinosaur"] + should set it thusly. + """ + prefs = ["dinosaur"] + + server.setPreferredTransports(prefs) + + self.assertEqual(server.getPreferredTransports(), prefs) + + def test_setSupportedPreferences(self): + """Taking the ``SUPPORTED_TRANSPORTS`` config option (a dict) and + passing it to ``setSupportedTransports()`` should convert it into a list + for the items in the dict whose value was ``True``. + """ + server.setSupportedTransports(self.config.SUPPORTED_TRANSPORTS) + + self.assertItemsEqual(server.getSupportedTransports(), + ["obfs4", "obfs3", "scramblesuit", "fte", "vanilla"]) + + +class JsonAPIResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIResource`.""" + + def setUp(self): + self.pagename = b'' + self.resource = server.JsonAPIResource() + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + def test_getClientIP(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + self.resource.getClientIP(request) + + def test_formatDataForResponse(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + data = {'data': { 'version': 'wow', + 'dinosaurs': 'cool', + 'triceratops': 'awesome', + 'velociraptors': 'terrifying', }} + + rendered = self.resource.formatDataForResponse(data, request) + + self.assertTrue(rendered) + self.assertTrue(request.responseHeaders.hasHeader('content-type')) + self.assertTrue(request.responseHeaders.hasHeader('server')) + self.assertEqual(request.responseHeaders.getRawHeaders('content-type'), + ['application/vnd.api+json']) + + def test_formatDataForResponse_no_data(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + rendered = self.resource.formatDataForResponse(None, request) + + self.assertEqual(rendered, b'') + + +class JsonAPIErrorResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIErrorResource`.""" + + def setUp(self): + self.pagename = b'' + self.root = Resource() + + def use_resource(self, resource): + self.resource = resource + self.root.putChild(self.pagename, self.resource) + + def do_render_for_method(self, method): + request = DummyRequest([self.pagename]) + request.method = method + + rendered = self.resource.render(request) + + self.assertTrue(rendered) + self.assertTrue(request.responseHeaders.hasHeader('content-type')) + self.assertTrue(request.responseHeaders.hasHeader('server')) + self.assertEqual(request.responseHeaders.getRawHeaders('content-type'), + ['application/vnd.api+json']) + + decoded = json.loads(rendered) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + + self.assertEqual(len(errors), 1) + + error = errors[0] + + return error + + def test_render_GET(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'GET') + + def test_render_POST(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'POST') + + def test_resource200_render_GET(self): + self.use_resource(server.JsonAPIErrorResource()) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['id'], 0) + self.assertEqual(error['type'], '') + self.assertEqual(error['code'], 200) + self.assertEqual(error['status'], 'OK') + self.assertEqual(error['detail'], '') + + def test_resource403_render_GET(self): + self.use_resource(server.resource403) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 403) + self.assertEqual(error['status'], 'Forbidden') + + def test_resource406_render_GET(self): + self.use_resource(server.resource406) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 406) + self.assertEqual(error['status'], 'Not Acceptable') + + def test_resource415_render_GET(self): + self.use_resource(server.resource415) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 415) + self.assertEqual(error['status'], 'Unsupported Media Type') + + def test_resource419_render_GET(self): + self.use_resource(server.resource419) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 419) + self.assertEqual(error['status'], "No You're A Teapot") + + def test_resource501_render_GET(self): + self.use_resource(server.resource501) + error = self.do_render_for_method(b'GET') + + self.assertEqual(error['code'], 501) + self.assertEqual(error['status'], 'Not Implemented') + + +class CustomErrorHandlingResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CustomErrorHandlingResource`.""" + + def setUp(self): + self.pagename = b'' + self.resource = server.CustomErrorHandlingResource() + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + def test_getChild(self): + request = DummyRequest(['foo']) + request.method = b'GET' + response_resource = self.resource.getChild('/foo', request) + + self.assertTrue(response_resource) + self.assertIsInstance(response_resource, server.JsonAPIErrorResource) + + response = response_resource.render(request) + detail = json.loads(response)['errors'][0]['detail'] + + self.assertIn('does not implement GET http://dummy/', detail) + + +class JsonAPIDataResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.JsonAPIDataResource`.""" + + def setUp(self): + self.resource = server.JsonAPIDataResource() + + def test_checkRequestHeaders_no_headers(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + + def test_checkRequestHeaders_different_content_type(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + request.requestHeaders.addRawHeader('Content-Type', 'application/html') + + def test_checkRequestHeaders_with_media_type(self): + request = DummyRequest(['']) + self.resource.checkRequestHeaders(request) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json;mp3') + + +class CaptchaFetchResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CaptchaFetchResource`.""" + + def setUp(self): + self.topDir = os.getcwd().rstrip('_trial_temp') + self.captchaDir = os.path.join(self.topDir, 'captchas') + self.captchaKey = CAPTCHA_KEY + self.hmacKey = HMAC_KEY + self.secretKey, self.publicKey = SECRET_KEY, PUBLIC_KEY + self.resource = server.CaptchaFetchResource(self.hmacKey, + self.publicKey, + self.secretKey, + self.captchaDir) + self.pagename = b'fetch' + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + self.make_captcha_directory() + + def make_captcha_directory(self): + if not os.path.isdir(self.captchaDir): + os.mkdir(self.captchaDir) + + def create_POST_with_data(self, data): + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + request.writeContent(data) + + return request + + def create_valid_POST(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def create_valid_POST_with_unsupported_transports(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4', 'dinosaur', 'karlthefog'], + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def test_init(self): + self.assertTrue(self.resource) + + def test_checkRequestHeaders_missing_content_type(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.removeHeader('Content-Type') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_missing_accept(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.removeHeader('Accept') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_content_type_with_media_parameters(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json;mp3') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_checkRequestHeaders_accept_with_media_parameters(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json;mp3') + request.method = b'POST' + request.writeContent(encoded_data) + + def test_getCaptchaImage(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + image, challenge = self.resource.getCaptchaImage(request) + + self.assertIsNotNone(image) + self.assertIsNotNone(challenge) + + def test_getCaptchaImage_empty_captcha_dir(self): + request = DummyRequest([self.pagename]) + request.method = b'GET' + + captchaDirOrig = self.resource.captchaDir + captchaDirNew = tempfile.mkdtemp() + self.resource.captchaDir = captchaDirNew + image, challenge = self.resource.getCaptchaImage(request) + self.resource.captchaDir = captchaDirOrig + shutil.rmtree(captchaDirNew) + + self.assertIsNone(image) + self.assertIsNone(challenge) + + def test_extractSupportedTransports_missing_type(self): + data = { + 'data': [{ + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_missing_version(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_missing_supported(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_wrong_type(self): + data = { + 'data': [{ + 'type': 'totoro', + 'version': server.MOAT_API_VERSION, + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_wrong_version(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': '0.0.1', # this version never existed + 'supported': ['obfs4'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_none_supported(self): + data = { + 'data': [{ + 'type': 'client-transports', + 'version': server.MOAT_API_VERSION, + 'supported': [], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + supported = self.resource.extractSupportedTransports(request) + + def test_extractSupportedTransports_preferred_transport(self): + request = self.create_valid_POST() + supported = self.resource.extractSupportedTransports(request) + + self.assertEqual(supported, ['obfs4']) + + def test_extractSupportedTransports_preferred_and_unknown_transports(self): + request = self.create_valid_POST_with_unsupported_transports() + supported = self.resource.extractSupportedTransports(request) + + self.assertEqual(supported, ['obfs4', 'dinosaur', 'karlthefog']) + + def test_getPreferredTransports_preferred_transport(self): + preferred = self.resource.getPreferredTransports(['obfs4']) + + self.assertEqual(preferred, 'obfs4') + + def test_getPreferredTransports_unknown_transport(self): + preferred = self.resource.getPreferredTransports(['dinosaur']) + + self.assertItemsEqual(preferred, + ['obfs4', 'obfs3', 'fte', 'scramblesuit', 'vanilla']) + + def assert_data_is_ok(self, decoded): + self.assertIsNone(decoded.get('errors')) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + + self.assertEqual(len(datas), 1) + + data = datas[0] + + self.assertEqual(data["type"], "moat-challenge") + self.assertEqual(data["version"], server.MOAT_API_VERSION) + self.assertIsNotNone(data["challenge"]) + self.assertIsNotNone(data["image"]) + self.assertIsNotNone(data["transport"]) + + def test_render_POST(self): + request = self.create_valid_POST() + response = self.resource.render(request) + + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assert_data_is_ok(decoded) + + +class CaptchaCheckResourceTests(unittest.TestCase): + """Tests for :class:`bridgedb.distributors.moat.server.CaptchaCheckResource`.""" + + def setUp(self): + self.topDir = os.getcwd().rstrip('_trial_temp') + self.captchaDir = os.path.join(self.topDir, 'captchas') + self.captchaKey = CAPTCHA_KEY + self.hmacKey = HMAC_KEY + self.secretKey, self.publicKey = SECRET_KEY, PUBLIC_KEY + self.distributor = DummyMoatDistributor() + self.schedule = ScheduledInterval("10", "minutes") + self.resource = server.CaptchaCheckResource(self.distributor, + self.schedule, 3, + self.hmacKey, + self.publicKey, + self.secretKey, + useForwardedHeader=False) + self.pagename = b'check' + self.root = Resource() + self.root.putChild(self.pagename, self.resource) + + self.solution = 'Tvx74PMy' + self.expiredChallenge = ( + "Vu-adMmSRsgr9PmPpGAznhrBQlys3zMkczIG2YQ7AngWqWnVn2y-LdAl8iHkrqkNhn" + "iyre02ZlUf5KD_KDqh_Km3dIoksOMW3eUuargLLnhIUldJ4PvSXPb7pwGev_FDY4gF" + "QDcmkrhFZm6RPzFWRgJjyY-2v6HRrmAMAGjGXSXnAc-8tDvVFSpo5Cce-saZou5W4G" + "TjzVcyG0WkXELA2nX8rozIDIr3mUyB1vb3f53KbW5b_oCEVC_LCSoxqjnS6ZSQpNzK" + "iz_PdOD2GIGPeclwiHAWM1pOS4cQVsTQR_z4ojZbpLiSp35n4Qbb11YOoreovZzlbS" + "7W38rAsTirkdeugcNq82AxKP3phEkyRcw--CzV") + + def create_POST_with_data(self, data): + request = DummyRequest([self.pagename]) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.method = b'POST' + + request.writeContent(data) + + return request + + def create_valid_POST_with_challenge(self, challenge): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': challenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + encoded_data = json.dumps(data) + + return self.create_POST_with_data(encoded_data) + + def create_valid_POST_make_new_challenge(self): + request = DummyRequest([self.pagename]) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + resource = server.CaptchaFetchResource(self.hmacKey, self.publicKey, + self.secretKey, self.captchaDir, + useForwardedHeader=False) + image, challenge = resource.getCaptchaImage(request) + + request = self.create_valid_POST_with_challenge(challenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + return request + + def test_withoutBlockIn(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + 'unblocked': ['us', 'ir', 'sy'], + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + + self.resource.render(request) + + def test_extractClientSolution(self): + request = self.create_valid_POST_make_new_challenge() + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + qrcode, transport, challenge, solution = self.resource.extractClientSolution(content) + + self.assertFalse(qrcode) + self.assertIsNotNone(transport) + self.assertIsNotNone(challenge) + self.assertIsNotNone(solution) + + def test_extractClientSolution_missing_id(self): + data = { + 'data': [{ + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_id(self): + data = { + 'data': [{ + 'id': 69, # nice + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_weird_transport(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'dinosaur', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_version(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': '0.0.1', # this version never existed + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_extractClientSolution_wrong_type(self): + data = { + 'data': [{ + 'id': 2, + 'type': 'boat-revolution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': self.expiredChallenge, + 'solution': self.solution, + 'qrcode': False, + }] + } + qrcode, transport, challenge, solution = self.resource.extractClientSolution(data['data'][0]) + + self.assertFalse(qrcode) + self.assertIsNone(transport) + self.assertIsNone(challenge) + self.assertIsNone(solution) + + def test_failureResponse_5(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + response = self.resource.failureResponse(5, request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + self.assertEqual(len(errors), 1) + + error = errors[0] + self.assertEqual(error['status'], "No You're A Teapot") + self.assertEqual(error['code'], 419) + self.assertEqual(error['detail'], "The CAPTCHA challenge timed out.") + self.assertEqual(error['id'], 5) + + def test_checkSolution(self): + request = self.create_valid_POST_make_new_challenge() + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + clientIP = self.resource.getClientIP(request) + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + qrcode, transport, challenge, solution = self.resource.extractClientSolution(content) + result = self.resource.checkSolution(challenge, solution, clientIP) + + self.assertTrue(result) + + def test_render_POST_expired(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('errors')) + + errors = decoded['errors'] + self.assertEqual(len(errors), 1) + + error = errors[0] + self.assertEqual(error['status'], "No You're A Teapot") + self.assertEqual(error['code'], 419) + self.assertEqual(error['detail'], "The CAPTCHA solution was incorrect.") + self.assertEqual(error['version'], server.MOAT_API_VERSION) + self.assertEqual(error['type'], "moat-bridges") + self.assertEqual(error['id'], 4) + + def test_getBridgeLines(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + encoded_content = request.content.read() + content = json.loads(encoded_content)['data'][0] + + bridgelines = self.resource.getBridgeLines('3.3.3.3', content) + + self.assertTrue(bridgelines) + + def test_getBridgeLines_no_data(self): + request = self.create_valid_POST_with_challenge(self.expiredChallenge) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + + bridgelines = self.resource.getBridgeLines('3.3.3.3', None) + + self.assertIsNone(bridgelines) + + def test_render_POST_unexpired(self): + request = self.create_valid_POST_make_new_challenge() + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + self.assertEqual(len(datas), 1) + + data = datas[0] + self.assertIsNone(data['qrcode']) + self.assertIsNotNone(data['bridges']) + self.assertEqual(data['version'], server.MOAT_API_VERSION) + self.assertEqual(data['type'], 'moat-bridges') + self.assertEqual(data['id'], 3) + + def test_render_POST_unexpired_with_qrcode(self): + request = DummyRequest([self.pagename]) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('Content-Type', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('Accept', 'application/vnd.api+json') + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + resource = server.CaptchaFetchResource(self.hmacKey, self.publicKey, + self.secretKey, self.captchaDir, + useForwardedHeader=False) + image, challenge = resource.getCaptchaImage(request) + + data = { + 'data': [{ + 'id': 2, + 'type': 'moat-solution', + 'version': server.MOAT_API_VERSION, + 'transport': 'obfs4', + 'challenge': challenge, + 'solution': self.solution, + 'qrcode': True, + }] + } + encoded_data = json.dumps(data) + request = self.create_POST_with_data(encoded_data) + request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) + request.requestHeaders.addRawHeader('X-Forwarded-For', '3.3.3.3') + + response = self.resource.render(request) + decoded = json.loads(response) + + self.assertTrue(decoded) + self.assertIsNotNone(decoded.get('data')) + + datas = decoded['data'] + self.assertEqual(len(datas), 1) + + data = datas[0] + self.assertIsNotNone(data['qrcode']) + self.assertIsNotNone(data['bridges']) + self.assertEqual(data['version'], server.MOAT_API_VERSION) + self.assertEqual(data['type'], 'moat-bridges') + self.assertEqual(data['id'], 3) + + +class AddMoatServerTests(unittest.TestCase): + """Tests for :func:`bridgedb.distributors.moat.server.addMoatServer()`.""" + + def setUp(self): + self.config = _createConfig() + self.distributor = DummyMoatDistributor() + + def test_addMoatServer(self): + server.addMoatServer(self.config, self.distributor) diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 2c41359..dbd177f 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -43,76 +43,6 @@ logging.disable(50) #server.logging.getLogger().setLevel(10) -class SetFQDNTests(unittest.TestCase): - """Tests for :func:`bridgedb.distributors.https.server.setFQDN` and - :func:`bridgedb.distributors.https.server.setFQDN`. - """ - - def setUp(self): - self.originalFQDN = server.SERVER_PUBLIC_FQDN - - def tearDown(self): - server.SERVER_PUBLIC_FQDN = self.originalFQDN - - def test_setFQDN_https(self): - """Calling ``server.setFQDN([…], https=True)`` should prepend - ``"https://"`` to the module :data:`server.SERVER_PUBLIC_FQDN` - variable. - """ - server.setFQDN('example.com', https=True) - self.assertEqual(server.SERVER_PUBLIC_FQDN, "https://example.com") - - def test_setFQDN_http(self): - """Calling ``server.setFQDN([…], https=False)`` should not prepend - anything at all to the module :data:`server.SERVER_PUBLIC_FQDN` - variable. - """ - server.setFQDN('example.com', https=False) - self.assertEqual(server.SERVER_PUBLIC_FQDN, "example.com") - - -class GetClientIPTests(unittest.TestCase): - """Tests for :func:`bridgedb.distributors.https.server.getClientIP`.""" - - def createRequestWithIPs(self): - """Set the IP address returned from ``request.getClientIP()`` to - '3.3.3.3', and the IP address reported in the 'X-Forwarded-For' header - to '2.2.2.2'. - """ - request = DummyRequest(['']) - request.headers.update({'x-forwarded-for': '2.2.2.2'}) - # See :api:`twisted.test.requesthelper.DummyRequest.getClientIP` - request.client = requesthelper.IPv4Address('TCP', '3.3.3.3', 443) - request.method = b'GET' - return request - - def test_getClientIP_XForwardedFor(self): - """getClientIP() should return the IP address from the - 'X-Forwarded-For' header when ``useForwardedHeader=True``. - """ - request = self.createRequestWithIPs() - clientIP = server.getClientIP(request, useForwardedHeader=True) - self.assertEqual(clientIP, '2.2.2.2') - - def test_getClientIP_XForwardedFor_bad_ip(self): - """getClientIP() should return None if the IP address from the - 'X-Forwarded-For' header is bad/invalid and - ``useForwardedHeader=True``. - """ - request = self.createRequestWithIPs() - request.headers.update({'x-forwarded-for': 'pineapple'}) - clientIP = server.getClientIP(request, useForwardedHeader=True) - self.assertEqual(clientIP, None) - - def test_getClientIP_fromRequest(self): - """getClientIP() should return the IP address from the request instance - when ``useForwardedHeader=False``. - """ - request = self.createRequestWithIPs() - clientIP = server.getClientIP(request) - self.assertEqual(clientIP, '3.3.3.3') - - class ReplaceErrorPageTests(unittest.TestCase): """Tests for :func:`bridgedb.distributors.https.server.replaceErrorPage`.""" diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py index ab34055..2a124b1 100644 --- a/bridgedb/test/test_main.py +++ b/bridgedb/test/test_main.py @@ -302,12 +302,12 @@ class BridgedbTests(unittest.TestCase): hashring. """ proxyList = None - (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, - proxyList, - self.key) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + self.config, proxyList, self.key) + # Should have an HTTPSDistributor ring, an EmailDistributor ring, - # and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + # a MoatDistributor right, and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4) def test_main_createBridgeRings_with_proxyList(self): """main.createBridgeRings() should add three hashrings to the @@ -316,12 +316,12 @@ class BridgedbTests(unittest.TestCase): exitRelays = ['1.1.1.1', '2.2.2.2', '3.3.3.3'] proxyList = main.proxy.ProxySet() proxyList.addExitRelays(exitRelays) - (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, - proxyList, - self.key) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + self.config, proxyList, self.key) + # Should have an HTTPSDistributor ring, an EmailDistributor ring, - # and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + # a MoatDistributor ring, and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4) self.assertGreater(len(httpsDist.proxies), 0) self.assertItemsEqual(exitRelays, httpsDist.proxies) @@ -332,11 +332,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.HTTPS_DIST = False - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an EmailDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an EmailDistributor ring, a MoatDistributor ring, and an + # UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('https', hashring.rings) self.assertNotIn(httpsDist, hashring.ringsByName.values()) @@ -347,11 +348,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.EMAIL_DIST = False - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, a MoatDistributor ring, and an + # UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('email', hashring.rings) self.assertNotIn(emailDist, hashring.ringsByName.values()) @@ -362,11 +364,12 @@ class BridgedbTests(unittest.TestCase): proxyList = main.proxy.ProxySet() config = self.config config.RESERVED_SHARE = 0 - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, and an EmailDistributor ring: - self.assertEqual(len(hashring.ringsByName.keys()), 2) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, an EmailDistributor ring, and a + # MoatDistributor ring: + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertNotIn('unallocated', hashring.rings) def test_main_createBridgeRings_two_file_buckets(self): @@ -380,12 +383,12 @@ class BridgedbTests(unittest.TestCase): 'bridges-for-support-desk': 10, 'bridges-for-ooni-tests': 10, } - (hashring, emailDist, httpsDist) = main.createBridgeRings(config, - proxyList, - self.key) - # Should have an HTTPSDistributor ring, an EmailDistributor, and an - # UnallocatedHolder ring: - self.assertEqual(len(hashring.ringsByName.keys()), 3) + (hashring, emailDist, httpsDist, moatDist) = main.createBridgeRings( + config, proxyList, self.key) + + # Should have an HTTPSDistributor ring, an EmailDistributor, a + # MoatDistributor and an UnallocatedHolder ring: + self.assertEqual(len(hashring.ringsByName.keys()), 4) # Should have two pseudoRings: self.assertEqual(len(hashring.pseudoRings), 2) diff --git a/scripts/setup-tests b/scripts/setup-tests index 8e42762..18298f4 100755 --- a/scripts/setup-tests +++ b/scripts/setup-tests @@ -25,8 +25,15 @@ sed -r -i -e "s/(EMAIL_SMTP_PORT = )([1-9]{2,5})/\12525/" run/bridgedb.conf sed -r -i -e "s/(HTTP_UNENCRYPTED_BIND_IP = )(None)/\1'127.0.0.1'/" run/bridgedb.conf sed -r -i -e "s/(HTTP_UNENCRYPTED_PORT = )(None)/\16788/" run/bridgedb.conf sed -r -i -e "s/(SERVER_PUBLIC_FQDN = )(.*)/\1'127.0.0.1:6788'/" run/bridgedb.conf +# Enable plaintext HTTP distribution for the moat server as well +sed -r -i -e "s/(MOAT_HTTP_IP = )(None)/\1'127.0.0.1'/" run/bridgedb.conf +sed -r -i -e "s/(MOAT_HTTP_PORT = )(None)/\16790/" run/bridgedb.conf +# Create descriptors leekspin -n 100 cp -t run/from-authority networkstatus-bridges cached-extrainfo* bridge-descriptors cp -t run/from-bifroest networkstatus-bridges cached-extrainfo* bridge-descriptors +# Create TLS certificates ./scripts/make-ssl-cert cp -t run privkey.pem cert +cp privkey.pem run/moat-tls.pem +cp cert run/moat-tls.crt diff --git a/setup.py b/setup.py index a133424..1139a65 100644 --- a/setup.py +++ b/setup.py @@ -376,8 +376,10 @@ setuptools.setup( package_dir={'bridgedb': 'bridgedb'}, packages=['bridgedb', 'bridgedb.distributors', + 'bridgedb.distributors.common', 'bridgedb.distributors.email', 'bridgedb.distributors.https', + 'bridgedb.distributors.moat', 'bridgedb.parse', 'bridgedb.test', ],

1 0

[bridgedb/master] Merge branch 'release/0.6.0'
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 659b4ed3bfea6318ac88880d3fb3bad99b44af89 Merge: 513cd25 b4d32c7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 12 21:33:22 2017 +0000 Merge branch 'release/0.6.0' .coveragerc | 2 + .travis.requirements.txt | 2 +- CHANGELOG | 12 + Makefile | 3 + README.rst | 173 ++++ bridgedb.conf | 104 ++- bridgedb/Bridges.py | 2 +- bridgedb/captcha.py | 1 - bridgedb/configure.py | 16 +- bridgedb/distributors/__init__.py | 2 + bridgedb/distributors/common/__init__.py | 2 + bridgedb/distributors/common/http.py | 80 ++ bridgedb/distributors/email/__init__.py | 24 + bridgedb/{ => distributors}/email/autoresponder.py | 24 +- bridgedb/{ => distributors}/email/distributor.py | 13 +- bridgedb/{ => distributors}/email/dkim.py | 6 +- bridgedb/{ => distributors}/email/request.py | 12 +- bridgedb/{ => distributors}/email/server.py | 30 +- bridgedb/{ => distributors}/email/templates.py | 6 +- bridgedb/{ => distributors}/https/__init__.py | 4 + bridgedb/{ => distributors}/https/distributor.py | 19 +- bridgedb/{ => distributors}/https/request.py | 8 +- bridgedb/{ => distributors}/https/server.py | 73 +- .../https/templates/assets/css/bootstrap.min.css | 0 .../https/templates/assets/css/error.css | 0 .../templates/assets/css/font-awesome-ie7.min.css | 0 .../templates/assets/css/font-awesome.min.css | 0 .../https/templates/assets/css/main.css | 0 .../https/templates/assets/css/rtl.css | 0 .../templates/assets/font/fontawesome-webfont.eot | Bin .../templates/assets/font/fontawesome-webfont.svg | 0 .../templates/assets/font/fontawesome-webfont.ttf | Bin .../templates/assets/font/fontawesome-webfont.woff | Bin .../https/templates/assets/font/lato-bold.woff | Bin .../https/templates/assets/font/lato-italic.woff | Bin .../https/templates/assets/font/lato-regular.woff | Bin .../templates/assets/images/404-excavator.svg | 0 .../templates/assets/images/404-hills-left.svg | 0 .../templates/assets/images/404-hills-right.svg | 0 .../assets/images/500-hills-left-shade.svg | 0 .../templates/assets/images/500-hills-left.svg | 0 .../assets/images/500-hills-right-shade.svg | 0 .../templates/assets/images/500-hills-right.svg | 0 .../https/templates/assets/images/500-road.svg | 0 .../assets/images/maintenance-hills-left-shade.svg | 0 .../assets/images/maintenance-hills-left.svg | 0 .../images/maintenance-hills-right-shade.svg | 0 .../assets/images/maintenance-hills-right.svg | 0 .../assets/images/maintenance-tractor.svg | 0 .../https/templates/assets/js/bridges.js | 0 .../https/templates/assets/tor-roots-blue.svg | 0 .../https/templates/assets/tor.svg | 0 .../{ => distributors}/https/templates/base.html | 0 .../https/templates/bridges.html | 0 .../https/templates/captcha.html | 0 .../https/templates/error-404.html | 0 .../https/templates/error-500.html | 0 .../https/templates/error-503.html | 0 .../{ => distributors}/https/templates/howto.html | 0 .../{ => distributors}/https/templates/index.html | 0 .../https/templates/options.html | 0 .../{ => distributors}/https/templates/robots.txt | 0 bridgedb/distributors/moat/__init__.py | 4 + bridgedb/distributors/moat/distributor.py | 64 ++ bridgedb/distributors/moat/request.py | 133 +++ bridgedb/distributors/moat/server.py | 776 +++++++++++++++++ bridgedb/email/__init__.py | 1 - bridgedb/{Main.py => main.py} | 95 ++- bridgedb/parse/addr.py | 4 +- bridgedb/parse/blacklist.py | 12 +- bridgedb/parse/options.py | 8 +- bridgedb/persistent.py | 4 +- bridgedb/proxy.py | 2 +- bridgedb/runner.py | 2 +- bridgedb/strings.py | 4 +- bridgedb/test/email_helpers.py | 14 +- bridgedb/test/https_helpers.py | 9 +- bridgedb/test/legacy_Tests.py | 8 +- bridgedb/test/moat_helpers.py | 110 +++ .../{test_bridgedb.py => test_bridgedb_script.py} | 0 bridgedb/test/test_distributors_common_http.py | 100 +++ bridgedb/test/test_distributors_moat_request.py | 52 ++ bridgedb/test/test_distributors_moat_server.py | 918 +++++++++++++++++++++ bridgedb/test/test_email_autoresponder.py | 14 +- bridgedb/test/test_email_distributor.py | 12 +- bridgedb/test/test_email_dkim.py | 4 +- bridgedb/test/test_email_request.py | 4 +- bridgedb/test/test_email_server.py | 20 +- bridgedb/test/test_email_templates.py | 4 +- bridgedb/test/test_https_distributor.py | 6 +- bridgedb/test/test_https_request.py | 4 +- bridgedb/test/test_https_server.py | 108 +-- bridgedb/test/{test_Main.py => test_main.py} | 255 +++--- bridgedb/test/test_parse_blacklist.py | 56 ++ doc/sphinx/requirements-for-sphinx-builds.txt | 11 +- doc/sphinx/source/bridgedb.distributors.email.rst | 15 + doc/sphinx/source/bridgedb.distributors.https.rst | 12 + doc/sphinx/source/bridgedb.email.rst | 15 - doc/sphinx/source/bridgedb.https.rst | 12 - .../{bridgedb.Main.rst => bridgedb.main.rst} | 6 +- doc/sphinx/source/bridgedb.rst | 6 +- doc/sphinx/source/conf.py | 27 +- requirements.txt | 2 +- scripts/bridgedb | 2 +- scripts/setup-tests | 7 + setup.py | 11 +- 106 files changed, 3071 insertions(+), 480 deletions(-)

1 0

[bridgedb/develop] Run scheduled tasks after server startup.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 1112e2e650318c961e5eb8b4af574e9a825682b3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 20:55:10 2017 +0000 Run scheduled tasks after server startup. --- bridgedb/Main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridgedb/Main.py b/bridgedb/Main.py index 53b70c2..edd93cf 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -488,7 +488,10 @@ def run(options, reactor=reactor): for name, seconds in config.TASKS.items(): if seconds: try: - tasks[name].start(abs(seconds)) + # Set now to False to get the servers up and running when + # first started, rather than spend a bunch of time in + # scheduled tasks. + tasks[name].start(abs(seconds), now=False) except KeyError: logging.info("Task %s is disabled and will not run." % name) else:

1 0

[bridgedb/develop] Add Makefile directive for making TAGS file.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 8ebc712bd0706c607fb30222cb304c258bf14d83 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 23:55:35 2017 +0000 Add Makefile directive for making TAGS file. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 906ce94..428f004 100644 --- a/Makefile +++ b/Makefile @@ -86,3 +86,6 @@ upload: clean python setup.py bdist_egg upload --sign #python setup.py bdist_wheel upload --sign python setup.py sdist --formats=gztar,zip upload --sign + +tags: + find ./bridgedb -type f -name "*.py" -print | xargs etags

1 0

[bridgedb/develop] Move bridgedb.Main to bridgedb.main.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 985923c9fa58a268fd8679f9e0193dd9411c8e59 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 22:58:39 2017 +0000 Move bridgedb.Main to bridgedb.main. --- bridgedb/configure.py | 4 +- bridgedb/distributors/https/server.py | 2 +- bridgedb/{Main.py => main.py} | 4 +- bridgedb/proxy.py | 2 +- bridgedb/runner.py | 2 +- bridgedb/test/legacy_Tests.py | 2 +- .../{test_bridgedb.py => test_bridgedb_script.py} | 0 bridgedb/test/{test_Main.py => test_main.py} | 126 ++++++++++----------- .../{bridgedb.Main.rst => bridgedb.main.rst} | 6 +- doc/sphinx/source/conf.py | 2 +- scripts/bridgedb | 2 +- 11 files changed, 76 insertions(+), 76 deletions(-) diff --git a/bridgedb/configure.py b/bridgedb/configure.py index 02eea65..1a5e949 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -47,9 +47,9 @@ def loadConfig(configFile=None, configCls=None): .. _faster: http://lucumr.pocoo.org/2011/2/1/exec-in-python/ :ivar bool itsSafeToUseLogging: This is called in - :func:`bridgedb.Main.run` before + :func:`bridgedb.main.run` before :func:`bridgedb.safelog.configureLogging`. When called from - :func:`~bridgedb.Main.run`, the **configCls** parameter is not given, + :func:`~bridgedb.main.run`, the **configCls** parameter is not given, because that is the first time that a :class:`config <bridgedb.configure.Conf>` has been created. If a :class:`logging.Logger` is created in this function, then logging will diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index 85a32b3..b756659 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -1008,7 +1008,7 @@ def addWebServer(config, distributor): :type config: :class:`bridgedb.persistent.Conf` :param config: A configuration object from - :mod:`bridgedb.Main`. Currently, we use these options:: + :mod:`bridgedb.main`. Currently, we use these options:: HTTP_UNENCRYPTED_PORT HTTP_UNENCRYPTED_BIND_IP HTTP_USE_IP_FROM_FORWARDED_HEADER diff --git a/bridgedb/Main.py b/bridgedb/main.py similarity index 99% rename from bridgedb/Main.py rename to bridgedb/main.py index edd93cf..f6d16ec 100644 --- a/bridgedb/Main.py +++ b/bridgedb/main.py @@ -63,7 +63,7 @@ def load(state, hashring, clear=False): hashring before parsing for new ones. """ if not state: - logging.fatal("bridgedb.Main.load() could not retrieve state!") + logging.fatal("bridgedb.main.load() could not retrieve state!") sys.exit(2) if clear: @@ -517,7 +517,7 @@ def runSubcommand(options, config): :type options: :class:`bridgedb.opt.MainOptions` :param options: A pre-parsed options class containing any arguments and options given in the commandline we were called with. - :type config: :class:`bridgedb.Main.Conf` + :type config: :class:`bridgedb.main.Conf` :param config: The current configuration. :raises: :exc:`SystemExit` when all subCommands and subOptions have finished running. diff --git a/bridgedb/proxy.py b/bridgedb/proxy.py index 4611cf7..39cd109 100644 --- a/bridgedb/proxy.py +++ b/bridgedb/proxy.py @@ -33,7 +33,7 @@ def downloadTorExits(proxyList, ipaddress, port=443, protocol=None): """Run a script which downloads a list of Tor exit relays which allow their clients to exit to the given **ipaddress** and **port**. - :param proxyList: The :class:`ProxySet` instance from :mod:`bridgedb.Main`. + :param proxyList: The :class:`ProxySet` instance from :mod:`bridgedb.main`. :param str ipaddress: The IP address that each Tor exit relay should be capable of connecting to for clients, as specified by its ExitPolicy. :param int port: The port corresponding to the above **ipaddress** that diff --git a/bridgedb/runner.py b/bridgedb/runner.py index 4591a3a..57297cb 100644 --- a/bridgedb/runner.py +++ b/bridgedb/runner.py @@ -128,7 +128,7 @@ def doDumpBridges(config): This function handles the commandline '--dump-bridges' option. - :type config: :class:`bridgedb.Main.Conf` + :type config: :class:`bridgedb.main.Conf` :param config: The current configuration. """ import bridgedb.Bucket as bucket diff --git a/bridgedb/test/legacy_Tests.py b/bridgedb/test/legacy_Tests.py index 77409b4..00e1d12 100644 --- a/bridgedb/test/legacy_Tests.py +++ b/bridgedb/test/legacy_Tests.py @@ -18,7 +18,7 @@ import time from datetime import datetime import bridgedb.Bridges -import bridgedb.Main +import bridgedb.main import bridgedb.schedule import bridgedb.Storage import re diff --git a/bridgedb/test/test_bridgedb.py b/bridgedb/test/test_bridgedb_script.py similarity index 100% rename from bridgedb/test/test_bridgedb.py rename to bridgedb/test/test_bridgedb_script.py diff --git a/bridgedb/test/test_Main.py b/bridgedb/test/test_main.py similarity index 79% rename from bridgedb/test/test_Main.py rename to bridgedb/test/test_main.py index b157fbf..fddb8a9 100644 --- a/bridgedb/test/test_Main.py +++ b/bridgedb/test/test_main.py @@ -9,7 +9,7 @@ # (c) 2007-2017, all entities within the AUTHORS file # :license: see included LICENSE for information -"""Tests for :mod:`bridgedb.Main`.""" +"""Tests for :mod:`bridgedb.main`.""" from __future__ import print_function @@ -26,7 +26,7 @@ from time import sleep from twisted.internet.threads import deferToThread from twisted.trial import unittest -from bridgedb import Main +from bridgedb import main from bridgedb.parse.options import parseOptions @@ -50,7 +50,7 @@ p reject 1-65535 def mockUpdateBridgeHistory(bridges, timestamps): """A mocked version of :func:`bridgedb.Stability.updateBridgeHistory` which doesn't access the database (so that we can test functions which - call it, like :func:`bridgedb.Main.load`). + call it, like :func:`bridgedb.main.load`). """ for fingerprint, stamps in timestamps.items()[:]: for timestamp in stamps: @@ -71,8 +71,8 @@ class MockHashring(object): pass -class MainTests(unittest.TestCase): - """Integration tests for :func:`bridgedb.Main.load`.""" +class BridgedbTests(unittest.TestCase): + """Integration tests for :func:`bridgedb.main.load`.""" def _appendToFile(self, file, data): """Append **data** to **file**.""" @@ -117,7 +117,7 @@ class MainTests(unittest.TestCase): def _cbCallUpdateBridgeHistory(self, d, hashring): """Fake some timestamps for the bridges in the hashring, and then call - Main.updateBridgeHistory(). + main.updateBridgeHistory(). """ def timestamp(): return datetime.fromtimestamp(random.randint(1324285117, 1524285117)) @@ -128,7 +128,7 @@ class MainTests(unittest.TestCase): for fingerprint, _ in bridges.items(): timestamps[fingerprint] = [timestamp(), timestamp(), timestamp()] - return Main.updateBridgeHistory(bridges, timestamps) + return main.updateBridgeHistory(bridges, timestamps) def _eb_Failure(self, failure): """If something produces a twisted.python.failure.Failure, fail the @@ -159,7 +159,7 @@ class MainTests(unittest.TestCase): """ # Get the bridgedb.conf file in the top-level directory of this repo: self.configFile = os.path.join(TOPDIR, 'bridgedb.conf') - self.config = Main.loadConfig(self.configFile) + self.config = main.loadConfig(self.configFile) # Copy the referenced descriptor files from bridgedb/run/ to CWD: self.config.STATUS_FILE = self._copyDescFilesHere([self.config.STATUS_FILE])[0] @@ -167,7 +167,7 @@ class MainTests(unittest.TestCase): self.config.EXTRA_INFO_FILES = self._copyDescFilesHere(self.config.EXTRA_INFO_FILES) # Initialise the state - self.state = Main.persistent.State(**self.config.__dict__) + self.state = main.persistent.State(**self.config.__dict__) self.key = base64.b64decode('TvPS1y36BFguBmSOvhChgtXB2Lt+BOw0mGfz9SZe12Y=') # Create a pseudo hashring @@ -175,104 +175,104 @@ class MainTests(unittest.TestCase): # Functions which some tests mock, which we'll need to re-replace # later in tearDown(): - self._orig_updateBridgeHistory = Main.updateBridgeHistory + self._orig_updateBridgeHistory = main.updateBridgeHistory self._orig_sys_argv = sys.argv def tearDown(self): """Replace the mocked mockUpdateBridgeHistory() function with the real function, Stability.updateBridgeHistory(). """ - Main.updateBridgeHistory = self._orig_updateBridgeHistory + main.updateBridgeHistory = self._orig_updateBridgeHistory sys.argv = self._orig_sys_argv - def test_Main_updateBridgeHistory(self): - """Main.updateBridgeHistory should update some timestamps for some + def test_main_updateBridgeHistory(self): + """main.updateBridgeHistory should update some timestamps for some bridges. """ # Mock the updateBridgeHistory() function so that we don't try to # access the database: - Main.updateBridgeHistory = mockUpdateBridgeHistory + main.updateBridgeHistory = mockUpdateBridgeHistory # Get the bridges into the mocked hashring - d = deferToThread(Main.load, self.state, self.hashring) + d = deferToThread(main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) d.addCallback(self._cbCallUpdateBridgeHistory, self.hashring) d.addErrback(self._eb_Failure) return d - def test_Main_load(self): - """Main.load() should run without error.""" - d = deferToThread(Main.load, self.state, self.hashring) + def test_main_load(self): + """main.load() should run without error.""" + d = deferToThread(main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d - def test_Main_load_no_state(self): - """Main.load() should raise SystemExit without a state object.""" - self.assertRaises(SystemExit, Main.load, None, self.hashring) + def test_main_load_no_state(self): + """main.load() should raise SystemExit without a state object.""" + self.assertRaises(SystemExit, main.load, None, self.hashring) - def test_Main_load_clear(self): + def test_main_load_clear(self): """When called with clear=True, load() should run and clear the hashrings. """ - d = deferToThread(Main.load, self.state, self.hashring, clear=True) + d = deferToThread(main.load, self.state, self.hashring, clear=True) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d - def test_Main_load_collect_timestamps(self): - """When COLLECT_TIMESTAMPS=True, Main.load() should call - Main.updateBridgeHistory(). + def test_main_load_collect_timestamps(self): + """When COLLECT_TIMESTAMPS=True, main.load() should call + main.updateBridgeHistory(). """ # Mock the addOrUpdateBridgeHistory() function so that we don't try to # access the database: - Main.updateBridgeHistory = mockUpdateBridgeHistory + main.updateBridgeHistory = mockUpdateBridgeHistory state = self.state state.COLLECT_TIMESTAMPS = True # The reactor is deferring this to a thread, so the test execution # here isn't actually covering the Storage.updateBridgeHistory() # function: - Main.load(state, self.hashring) + main.load(state, self.hashring) - def test_Main_load_malformed_networkstatus(self): + def test_main_load_malformed_networkstatus(self): """When called with a networkstatus file with an invalid descriptor, - Main.load() should raise a ValueError. + main.load() should raise a ValueError. """ self._appendToFile(self.state.STATUS_FILE, NETWORKSTATUS_MALFORMED) - self.assertRaises(ValueError, Main.load, self.state, self.hashring) + self.assertRaises(ValueError, main.load, self.state, self.hashring) - def test_Main_reloadFn(self): - """Main._reloadFn() should return True.""" - self.assertTrue(Main._reloadFn()) + def test_main_reloadFn(self): + """main._reloadFn() should return True.""" + self.assertTrue(main._reloadFn()) - def test_Main_handleSIGHUP(self): - """Main._handleSIGHUP() should return True.""" + def test_main_handleSIGHUP(self): + """main._handleSIGHUP() should return True.""" raise unittest.SkipTest("_handleSIGHUP touches the reactor.") - self.assertTrue(Main._handleSIGHUP()) + self.assertTrue(main._handleSIGHUP()) - def test_Main_createBridgeRings(self): - """Main.createBridgeRings() should add three hashrings to the + def test_main_createBridgeRings(self): + """main.createBridgeRings() should add three hashrings to the hashring. """ proxyList = None - (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, proxyList, self.key) # Should have an HTTPSDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 3) - def test_Main_createBridgeRings_with_proxyList(self): - """Main.createBridgeRings() should add three hashrings to the + def test_main_createBridgeRings_with_proxyList(self): + """main.createBridgeRings() should add three hashrings to the hashring and add the proxyList to the IPBasedDistibutor. """ exitRelays = ['1.1.1.1', '2.2.2.2', '3.3.3.3'] - proxyList = Main.proxy.ProxySet() + proxyList = main.proxy.ProxySet() proxyList.addExitRelays(exitRelays) - (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(self.config, proxyList, self.key) # Should have an HTTPSDistributor ring, an EmailDistributor ring, @@ -281,14 +281,14 @@ class MainTests(unittest.TestCase): self.assertGreater(len(httpsDist.proxies), 0) self.assertItemsEqual(exitRelays, httpsDist.proxies) - def test_Main_createBridgeRings_no_https_dist(self): - """When HTTPS_DIST=False, Main.createBridgeRings() should add only + def test_main_createBridgeRings_no_https_dist(self): + """When HTTPS_DIST=False, main.createBridgeRings() should add only two hashrings to the hashring. """ - proxyList = Main.proxy.ProxySet() + proxyList = main.proxy.ProxySet() config = self.config config.HTTPS_DIST = False - (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(config, proxyList, self.key) # Should have an EmailDistributor ring, and an UnallocatedHolder ring: @@ -296,14 +296,14 @@ class MainTests(unittest.TestCase): self.assertNotIn('https', hashring.rings) self.assertNotIn(httpsDist, hashring.ringsByName.values()) - def test_Main_createBridgeRings_no_email_dist(self): - """When EMAIL_DIST=False, Main.createBridgeRings() should add only + def test_main_createBridgeRings_no_email_dist(self): + """When EMAIL_DIST=False, main.createBridgeRings() should add only two hashrings to the hashring. """ - proxyList = Main.proxy.ProxySet() + proxyList = main.proxy.ProxySet() config = self.config config.EMAIL_DIST = False - (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(config, proxyList, self.key) # Should have an HTTPSDistributor ring, and an UnallocatedHolder ring: @@ -311,32 +311,32 @@ class MainTests(unittest.TestCase): self.assertNotIn('email', hashring.rings) self.assertNotIn(emailDist, hashring.ringsByName.values()) - def test_Main_createBridgeRings_no_reserved_share(self): - """When RESERVED_SHARE=0, Main.createBridgeRings() should add only + def test_main_createBridgeRings_no_reserved_share(self): + """When RESERVED_SHARE=0, main.createBridgeRings() should add only two hashrings to the hashring. """ - proxyList = Main.proxy.ProxySet() + proxyList = main.proxy.ProxySet() config = self.config config.RESERVED_SHARE = 0 - (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(config, proxyList, self.key) # Should have an HTTPSDistributor ring, and an EmailDistributor ring: self.assertEqual(len(hashring.ringsByName.keys()), 2) self.assertNotIn('unallocated', hashring.rings) - def test_Main_createBridgeRings_two_file_buckets(self): - """When FILE_BUCKETS has two filenames in it, Main.createBridgeRings() + def test_main_createBridgeRings_two_file_buckets(self): + """When FILE_BUCKETS has two filenames in it, main.createBridgeRings() should add three hashrings to the hashring, then add two "pseudo-rings". """ - proxyList = Main.proxy.ProxySet() + proxyList = main.proxy.ProxySet() config = self.config config.FILE_BUCKETS = { 'bridges-for-support-desk': 10, 'bridges-for-ooni-tests': 10, } - (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = main.createBridgeRings(config, proxyList, self.key) # Should have an HTTPSDistributor ring, an EmailDistributor, and an @@ -348,8 +348,8 @@ class MainTests(unittest.TestCase): self.assertIn('pseudo_bridges-for-support-desk', hashring.pseudoRings) self.assertIn('pseudo_bridges-for-ooni-tests', hashring.pseudoRings) - def test_Main_run(self): - """Main.run() should run and then finally raise SystemExit.""" + def test_main_run(self): + """main.run() should run and then finally raise SystemExit.""" config = """ BRIDGE_FILES = ["../run/bridge-descriptors"] EXTRA_INFO_FILES = ["../run/cached-extrainfo", "../run/cached-extrainfo.new"] @@ -434,4 +434,4 @@ FILE_BUCKETS = {}""" sys.argv = ['bridgedb', '-r', os.getcwd(), '-c', configFile] options = parseOptions() - self.assertRaises(SystemExit, Main.run, options, reactor=None) + self.assertRaises(SystemExit, main.run, options, reactor=None) diff --git a/doc/sphinx/source/bridgedb.Main.rst b/doc/sphinx/source/bridgedb.main.rst similarity index 56% rename from doc/sphinx/source/bridgedb.Main.rst rename to doc/sphinx/source/bridgedb.main.rst index 982b567..cf9f9ec 100644 --- a/doc/sphinx/source/bridgedb.Main.rst +++ b/doc/sphinx/source/bridgedb.main.rst @@ -1,7 +1,7 @@ -bridgedb.Main ----------------- +bridgedb.main +----------------- -.. automodule:: bridgedb.Main +.. automodule:: bridgedb.main :members: :undoc-members: :private-members: diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 906d5fe..8eb0063 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -27,6 +27,7 @@ sys.path.insert(0, os.path.abspath(os.path.join( import bridgedb +import bridgedb.main import bridgedb.bridgerequest import bridgedb.bridges import bridgedb.captcha @@ -47,7 +48,6 @@ import bridgedb.distributors.https import bridgedb.distributors.https.distributor import bridgedb.distributors.https.request import bridgedb.distributors.https.server -import bridgedb.Main import bridgedb.parse import bridgedb.parse.addr import bridgedb.parse.descriptors diff --git a/scripts/bridgedb b/scripts/bridgedb index 2ec37b1..876fde9 100644 --- a/scripts/bridgedb +++ b/scripts/bridgedb @@ -16,7 +16,7 @@ import sys from os import getcwd -from bridgedb.Main import run +from bridgedb.main import run from bridgedb.parse.options import parseOptions

1 0

[bridgedb/develop] Make new bridgedb.distributors package and put distributor code there.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit fca25fc8d9295ab50cb2a1be39199c213ddf8751 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 20:44:24 2017 +0000 Make new bridgedb.distributors package and put distributor code there. --- bridgedb/Bridges.py | 2 +- bridgedb/Main.py | 16 ++++---- bridgedb/distributors/__init__.py | 5 +++ bridgedb/distributors/email/__init__.py | 8 ++++ bridgedb/{ => distributors}/email/autoresponder.py | 24 +++++------ bridgedb/{ => distributors}/email/distributor.py | 10 ++--- bridgedb/{ => distributors}/email/dkim.py | 6 +-- bridgedb/{ => distributors}/email/request.py | 12 +++--- bridgedb/{ => distributors}/email/server.py | 30 +++++++------- bridgedb/{ => distributors}/email/templates.py | 6 +-- bridgedb/{ => distributors}/https/__init__.py | 4 ++ bridgedb/{ => distributors}/https/distributor.py | 8 ++-- bridgedb/{ => distributors}/https/request.py | 8 ++-- bridgedb/{ => distributors}/https/server.py | 8 ++-- .../https/templates/assets/css/bootstrap.min.css | 0 .../https/templates/assets/css/error.css | 0 .../templates/assets/css/font-awesome-ie7.min.css | 0 .../templates/assets/css/font-awesome.min.css | 0 .../https/templates/assets/css/main.css | 0 .../https/templates/assets/css/rtl.css | 0 .../templates/assets/font/fontawesome-webfont.eot | Bin .../templates/assets/font/fontawesome-webfont.svg | 0 .../templates/assets/font/fontawesome-webfont.ttf | Bin .../templates/assets/font/fontawesome-webfont.woff | Bin .../https/templates/assets/font/lato-bold.woff | Bin .../https/templates/assets/font/lato-italic.woff | Bin .../https/templates/assets/font/lato-regular.woff | Bin .../templates/assets/images/404-excavator.svg | 0 .../templates/assets/images/404-hills-left.svg | 0 .../templates/assets/images/404-hills-right.svg | 0 .../assets/images/500-hills-left-shade.svg | 0 .../templates/assets/images/500-hills-left.svg | 0 .../assets/images/500-hills-right-shade.svg | 0 .../templates/assets/images/500-hills-right.svg | 0 .../https/templates/assets/images/500-road.svg | 0 .../assets/images/maintenance-hills-left-shade.svg | 0 .../assets/images/maintenance-hills-left.svg | 0 .../images/maintenance-hills-right-shade.svg | 0 .../assets/images/maintenance-hills-right.svg | 0 .../assets/images/maintenance-tractor.svg | 0 .../https/templates/assets/js/bridges.js | 0 .../https/templates/assets/tor-roots-blue.svg | 0 .../https/templates/assets/tor.svg | 0 .../{ => distributors}/https/templates/base.html | 0 .../https/templates/bridges.html | 0 .../https/templates/captcha.html | 0 .../https/templates/error-404.html | 0 .../https/templates/error-500.html | 0 .../https/templates/error-503.html | 0 .../{ => distributors}/https/templates/howto.html | 0 .../{ => distributors}/https/templates/index.html | 0 .../https/templates/options.html | 0 .../{ => distributors}/https/templates/robots.txt | 0 bridgedb/email/__init__.py | 1 - bridgedb/parse/addr.py | 4 +- bridgedb/persistent.py | 4 +- bridgedb/strings.py | 4 +- bridgedb/test/email_helpers.py | 14 +++---- bridgedb/test/https_helpers.py | 4 +- bridgedb/test/legacy_Tests.py | 6 +-- bridgedb/test/test_email_autoresponder.py | 14 +++---- bridgedb/test/test_email_distributor.py | 12 +++--- bridgedb/test/test_email_dkim.py | 4 +- bridgedb/test/test_email_request.py | 4 +- bridgedb/test/test_email_server.py | 20 +++++----- bridgedb/test/test_email_templates.py | 4 +- bridgedb/test/test_https_distributor.py | 6 +-- bridgedb/test/test_https_request.py | 4 +- bridgedb/test/test_https_server.py | 44 ++++++++++----------- doc/sphinx/source/conf.py | 22 +++++------ setup.py | 9 +++-- 71 files changed, 172 insertions(+), 155 deletions(-) diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index ac403b6..50d70ae 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -541,7 +541,7 @@ class FilteredBridgeSplitter(object): :ivar bridges: DOCDOC :type distributorName: str :ivar distributorName: The name of this splitter's distributor. See - :meth:`~bridgedb.https.distributor.HTTPSDistributor.setDistributorName`. + :meth:`~bridgedb.distributors.https.distributor.HTTPSDistributor.setDistributorName`. """ self.key = key self.filterRings = {} diff --git a/bridgedb/Main.py b/bridgedb/Main.py index 4273653..53b70c2 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -31,8 +31,8 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig -from bridgedb.email.distributor import EmailDistributor -from bridgedb.https.distributor import HTTPSDistributor +from bridgedb.distributors.email.distributor import EmailDistributor +from bridgedb.distributors.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors from bridgedb.parse.blacklist import parseBridgeBlacklistFile @@ -216,8 +216,8 @@ def createBridgeRings(cfg, proxyList, key): :param bytes key: Hashring master key :rtype: tuple :returns: A BridgeSplitter hashring, an - :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an - :class:`~bridgedb.email.distributor.EmailDistributor` or None. + :class:`~bridgedb.distributors.https.distributor.HTTPSDistributor` or None, and an + :class:`~bridgedb.distributors.email.distributor.EmailDistributor` or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -316,8 +316,8 @@ def run(options, reactor=reactor): state = persistent.State(config=config) - from bridgedb.email.server import addServer as addSMTPServer - from bridgedb.https.server import addWebServer + from bridgedb.distributors.email.server import addServer as addSMTPServer + from bridgedb.distributors.https.server import addWebServer # Load the master key, or create a new one. key = crypto.getKey(config.MASTER_KEY_FILE) @@ -352,9 +352,9 @@ def run(options, reactor=reactor): :ivar proxyList: The container for the IP addresses of any currently known open proxies. :ivar ipDistributor: A - :class:`~bridgedb.https.distributor.HTTPSDistributor`. + :class:`~bridgedb.distributors.https.distributor.HTTPSDistributor`. :ivar emailDistributor: A - :class:`~bridgedb.email.distributor.EmailDistributor`. + :class:`~bridgedb.distributors.email.distributor.EmailDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some scheduled event, repetitive or otherwise, for the :class:`reactor diff --git a/bridgedb/distributors/__init__.py b/bridgedb/distributors/__init__.py new file mode 100644 index 0000000..951ef44 --- /dev/null +++ b/bridgedb/distributors/__init__.py @@ -0,0 +1,5 @@ +"""Methods for distributing bridges.""" + +import email +import https +#import moat diff --git a/bridgedb/distributors/email/__init__.py b/bridgedb/distributors/email/__init__.py new file mode 100644 index 0000000..8f9ed03 --- /dev/null +++ b/bridgedb/distributors/email/__init__.py @@ -0,0 +1,8 @@ +"""Servers for BridgeDB's email bridge distributor.""" + +import autoresponder +import distributor +import dkim +import request +import server +import templates diff --git a/bridgedb/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py similarity index 97% rename from bridgedb/email/autoresponder.py rename to bridgedb/distributors/email/autoresponder.py index 430a50c..ff65a73 100644 --- a/bridgedb/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -13,10 +13,10 @@ #_____________________________________________________________________________ """ -.. py:module:: bridgedb.email.autoresponder +.. py:module:: bridgedb.distributors.email.autoresponder :synopsis: Functionality for autoresponding to incoming emails. -bridgedb.email.autoresponder +bridgedb.distributors.email.autoresponder ============================ Functionality for autoresponding to incoming emails. @@ -26,7 +26,7 @@ Functionality for autoresponding to incoming emails. :: - bridgedb.email.autoresponder + bridgedb.distributors.email.autoresponder | |_ createResponseBody - Parse lines from an incoming email and determine | | how to respond. | |_ generateResponse - Create an email response. @@ -50,13 +50,13 @@ from twisted.python import failure from bridgedb import safelog from bridgedb.crypto import NEW_BUFFER_INTERFACE -from bridgedb.email import dkim -from bridgedb.email import request -from bridgedb.email import templates -from bridgedb.email.distributor import EmailRequestedHelp -from bridgedb.email.distributor import EmailRequestedKey -from bridgedb.email.distributor import TooSoonEmail -from bridgedb.email.distributor import IgnoreEmail +from bridgedb.distributors.email import dkim +from bridgedb.distributors.email import request +from bridgedb.distributors.email import templates +from bridgedb.distributors.email.distributor import EmailRequestedHelp +from bridgedb.distributors.email.distributor import EmailRequestedKey +from bridgedb.distributors.email.distributor import TooSoonEmail +from bridgedb.distributors.email.distributor import IgnoreEmail from bridgedb.parse import addr from bridgedb.parse.addr import canonicalizeEmailDomain from bridgedb.util import levenshteinDistance @@ -69,7 +69,7 @@ def createResponseBody(lines, context, client, lang='en'): :param list lines: The list of lines from the original request sent by the client. - :type context: :class:`bridgedb.email.server.MailServerContext` + :type context: :class:`bridgedb.distributors.email.server.MailServerContext` :param context: The context which contains settings for the email server. :type client: :api:`twisted.mail.smtp.Address` :param client: The client's email address which should be in the @@ -614,7 +614,7 @@ class SMTPAutoresponder(smtp.SMTPClient): 3. Check that those canonical domains match. 4. If the incoming message is from a domain which supports DKIM - signing, then run :func:`bridgedb.email.dkim.checkDKIM` as well. + signing, then run :func:`bridgedb.distributors.email.dkim.checkDKIM` as well. .. note:: Calling this method sets the :attr:`incoming.canonicalFromEmail` and diff --git a/bridgedb/email/distributor.py b/bridgedb/distributors/email/distributor.py similarity index 96% rename from bridgedb/email/distributor.py rename to bridgedb/distributors/email/distributor.py index c6e45d5..06a0c50 100644 --- a/bridgedb/email/distributor.py +++ b/bridgedb/distributors/email/distributor.py @@ -11,10 +11,10 @@ # :license: see LICENSE for licensing information """ -.. py:module:: bridgedb.email.distributor +.. py:module:: bridgedb.distributors.email.distributor :synopsis: A Distributor which hands out Bridges via an email interface. -bridgedb.email.autoresponder +bridgedb.distributors.email.autoresponder ============================ A :class:`~bridgedb.distribute.Distributor` which hands out :class:`bridges @@ -117,11 +117,11 @@ class EmailDistributor(Distributor): .. hint:: All checks on the email address (which should be stored in the ``bridgeRequest.client`` attribute), such as checks for whitelisting and canonicalization of domain name, are done in - :meth:`bridgedb.email.autoresponder.getMailTo` and - :meth:`bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + :meth:`bridgedb.distributors.email.autoresponder.getMailTo` and + :meth:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :type bridgeRequest: - :class:`~bridgedb.email.request.EmailBridgeRequest` + :class:`~bridgedb.distributors.email.request.EmailBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute diff --git a/bridgedb/email/dkim.py b/bridgedb/distributors/email/dkim.py similarity index 95% rename from bridgedb/email/dkim.py rename to bridgedb/distributors/email/dkim.py index c851c5c..be33d59 100644 --- a/bridgedb/email/dkim.py +++ b/bridgedb/distributors/email/dkim.py @@ -13,18 +13,18 @@ #_____________________________________________________________________________ """ -.. py:module:: bridgedb.email.dkim +.. py:module:: bridgedb.distributors.email.dkim :synopsis: Functions for checking DKIM verification results in email headers. -bridgedb.email.dkim +bridgedb.distributors.email.dkim =================== Functions for checking DKIM verification results in email headers. :: - bridgedb.email.dkim + bridgedb.distributors.email.dkim |_ checkDKIM - Check the DKIM verification results header. .. diff --git a/bridgedb/email/request.py b/bridgedb/distributors/email/request.py similarity index 94% rename from bridgedb/email/request.py rename to bridgedb/distributors/email/request.py index 71167c7..a490e56 100644 --- a/bridgedb/email/request.py +++ b/bridgedb/distributors/email/request.py @@ -13,11 +13,11 @@ #_____________________________________________________________________________ """ -.. py:module:: bridgedb.email.request +.. py:module:: bridgedb.distributors.email.request :synopsis: Classes for parsing and storing information about requests for bridges which are sent to the email distributor. -bridgedb.email.request +bridgedb.distributors.email.request ====================== Classes for parsing and storing information about requests for bridges @@ -28,7 +28,7 @@ which are sent to the email distributor. :: - bridgedb.email.request + bridgedb.distributors.email.request | |_ determineBridgeRequestOptions - Figure out which filters to apply, or | offer help. |_ EmailBridgeRequest - A request for bridges which was received through @@ -44,8 +44,8 @@ import logging import re from bridgedb import bridgerequest -from bridgedb.email.distributor import EmailRequestedHelp -from bridgedb.email.distributor import EmailRequestedKey +from bridgedb.distributors.email.distributor import EmailRequestedHelp +from bridgedb.distributors.email.distributor import EmailRequestedKey #: A regular expression for matching the Pluggable Transport method TYPE in @@ -109,7 +109,7 @@ class EmailBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self): """Process a new bridge request received through the - :class:`~bridgedb.email.distributor.EmailDistributor`. + :class:`~bridgedb.distributors.email.distributor.EmailDistributor`. """ super(EmailBridgeRequest, self).__init__() self._wantsKey = False diff --git a/bridgedb/email/server.py b/bridgedb/distributors/email/server.py similarity index 95% rename from bridgedb/email/server.py rename to bridgedb/distributors/email/server.py index 41862e7..229f118 100644 --- a/bridgedb/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -14,11 +14,11 @@ """ -.. py:module:: bridgedb.email.server +.. py:module:: bridgedb.distributors.email.server :synopsis: Servers which interface with clients and distribute bridges over SMTP. -bridgedb.email.server +bridgedb.distributors.email.server ===================== Servers which interface with clients and distribute bridges over SMTP. @@ -28,7 +28,7 @@ Servers which interface with clients and distribute bridges over SMTP. :: - bridgedb.email.server + bridgedb.distributors.email.server | |_ addServer - Set up a SMTP server which listens on the configured | EMAIL_PORT for incoming connections, and responds as | necessary to requests for bridges. @@ -66,9 +66,9 @@ from zope.interface import implements from bridgedb import __version__ from bridgedb import safelog from bridgedb.crypto import initializeGnuPG -from bridgedb.email import autoresponder -from bridgedb.email import templates -from bridgedb.email import request +from bridgedb.distributors.email import autoresponder +from bridgedb.distributors.email import templates +from bridgedb.distributors.email import request from bridgedb.parse import addr from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import canonicalizeEmailDomain @@ -109,7 +109,7 @@ class MailServerContext(object): """Create a context for storing configs for email bridge distribution. :type config: :class:`bridgedb.persistent.Conf` - :type distributor: :class:`~bridgedb.email.distributor.EmailDistributor` + :type distributor: :class:`~bridgedb.distributors.email.distributor.EmailDistributor` :param distributor: The distributor will handle getting the correct bridges (or none) for a client for us. :type schedule: :class:`bridgedb.schedule.ScheduledInterval` @@ -172,18 +172,18 @@ class SMTPMessage(object): :var bool ignoring: If ``True``, we're ignoring the rest of this message because it exceeded :data:`MailServerContext.maximumSize`. :var canonicalFromSMTP: See - :meth:`~bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :var canonicalFromEmail: See - :meth:`~bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :var canonicalDomainRules: See - :meth:`~bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :var message: (:api:`twisted.mail.smtp.rfc822.Message` or ``None``) The incoming email message. - :var responder: A :class:`~bridgedb.email.autoresponder.SMTPAutoresponder` + :var responder: A :class:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder` which parses and checks the incoming :data:`message`. If it decides to do so, it will build a - :meth:`~bridgedb.email.autoresponder.SMTPAutoresponder.reply` email - and :meth:`~bridgedb.email.autoresponder.SMTPAutoresponder.send` it. + :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.reply` email + and :meth:`~bridgedb.distributors.email.autoresponder.SMTPAutoresponder.send` it. """ implements(smtp.IMessage) @@ -432,7 +432,7 @@ class SMTPIncomingServerFactory(smtp.SMTPFactory): automation whenever we get a incoming connection on the SMTP port. .. warning:: - My :attr:`~bridgedb.email.server.SMTPIncomingServerFactory.context` + My :attr:`~bridgedb.distributors.email.server.SMTPIncomingServerFactory.context` isn't an OpenSSL context, as is used for the :api:`twisted.mail.smtp.ESMTPSender`. @@ -477,7 +477,7 @@ def addServer(config, distributor): :type config: :class:`bridgedb.configure.Conf` :param config: A configuration object. - :type distributor: :class:`bridgedb.email.distributor.EmailDistributor` + :type distributor: :class:`bridgedb.distributors.email.distributor.EmailDistributor` :param dist: A distributor which will handle database interactions, and will decide which bridges to give to who and when. """ diff --git a/bridgedb/email/templates.py b/bridgedb/distributors/email/templates.py similarity index 97% rename from bridgedb/email/templates.py rename to bridgedb/distributors/email/templates.py index 014881e..85dd105 100644 --- a/bridgedb/email/templates.py +++ b/bridgedb/distributors/email/templates.py @@ -11,11 +11,11 @@ #_____________________________________________________________________________ """ -.. py:module:: bridgedb.email.templates +.. py:module:: bridgedb.distributors.email.templates :synopsis: Templates for formatting emails sent out by the email distributor. -bridgedb.email.templates +bridgedb.distributors.email.templates ======================== Templates for formatting emails sent out by the email distributor. @@ -30,7 +30,7 @@ import os from datetime import datetime from bridgedb import strings -from bridgedb.email.distributor import MAX_EMAIL_RATE +from bridgedb.distributors.email.distributor import MAX_EMAIL_RATE def addCommands(template): diff --git a/bridgedb/https/__init__.py b/bridgedb/distributors/https/__init__.py similarity index 52% rename from bridgedb/https/__init__.py rename to bridgedb/distributors/https/__init__.py index f1210ec..6c51a5f 100644 --- a/bridgedb/https/__init__.py +++ b/bridgedb/distributors/https/__init__.py @@ -1 +1,5 @@ """Servers for BridgeDB's HTTPS bridge distributor.""" + +import distributor +import request +import server diff --git a/bridgedb/https/distributor.py b/bridgedb/distributors/https/distributor.py similarity index 98% rename from bridgedb/https/distributor.py rename to bridgedb/distributors/https/distributor.py index 4e980b7..791229b 100644 --- a/bridgedb/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -11,7 +11,7 @@ # :license: see LICENSE for licensing information """ -bridgedb.https.distributor +bridgedb.distributors.https.distributor ========================== A Distributor that hands out bridges through a web interface. @@ -119,7 +119,7 @@ class HTTPSDistributor(Distributor): ``1.2.178.234``) are placed within the same cluster, but Carol (with address ``1.3.11.33``) *might* end up in a different cluster. - >>> from bridgedb.https.distributor import HTTPSDistributor + >>> from bridgedb.distributors.https.distributor import HTTPSDistributor >>> HTTPSDistributor.getSubnet('1.2.3.4') '1.2.0.0/16' >>> HTTPSDistributor.getSubnet('1.2.211.154') @@ -274,7 +274,7 @@ class HTTPSDistributor(Distributor): def getBridges(self, bridgeRequest, interval): """Return a list of bridges to give to a user. - :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` + :type bridgeRequest: :class:`bridgedb.distributors.https.request.HTTPSBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's IP address. @@ -283,7 +283,7 @@ class HTTPSDistributor(Distributor): :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in the response. See - :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` + :meth:`bridgedb.distributors.https.server.WebResourceBridges.getBridgeRequestAnswer` for an example of how this is used. """ logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) diff --git a/bridgedb/https/request.py b/bridgedb/distributors/https/request.py similarity index 96% rename from bridgedb/https/request.py rename to bridgedb/distributors/https/request.py index fd80809..3236bee 100644 --- a/bridgedb/https/request.py +++ b/bridgedb/distributors/https/request.py @@ -10,11 +10,11 @@ #_____________________________________________________________________________ """ -.. py:module:: bridgedb.https.request +.. py:module:: bridgedb.distributors.https.request :synopsis: Classes for parsing and storing information about requests for bridges which are sent to the HTTPS distributor. -bridgedb.https.request +bridgedb.distributors.https.request ====================== Classes for parsing and storing information about requests for bridges @@ -24,7 +24,7 @@ which are sent to the HTTPS distributor. :: - bridgedb.https.request + bridgedb.distributors.https.request | |_ HTTPSBridgeRequest - A request for bridges which was received through the HTTPS distributor. @@ -58,7 +58,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self, addClientCountryCode=True): """Process a new bridge request received through the - :class:`~bridgedb.https.distributor.HTTPSDistributor`. + :class:`~bridgedb.distributors.https.distributor.HTTPSDistributor`. :param bool addClientCountryCode: If ``True``, then calling :meth:`withoutBlockInCountry` will attempt to add the client's own diff --git a/bridgedb/https/server.py b/bridgedb/distributors/https/server.py similarity index 99% rename from bridgedb/https/server.py rename to bridgedb/distributors/https/server.py index c785450..3e4284f 100644 --- a/bridgedb/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -8,11 +8,11 @@ # :license: see LICENSE for licensing information """ -.. py:module:: bridgedb.https.server +.. py:module:: bridgedb.distributors.https.server :synopsis: Servers which interface with clients and distribute bridges over HTTP(S). -bridgedb.https.server +bridgedb.distributors.https.server ===================== Servers which interface with clients and distribute bridges over HTTP(S). @@ -50,7 +50,7 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha -from bridgedb.https.request import HTTPSBridgeRequest +from bridgedb.distributors.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress from bridgedb.qrcodes import generateQR @@ -1032,7 +1032,7 @@ def addWebServer(config, distributor): CSP_ENABLED CSP_REPORT_ONLY CSP_INCLUDE_SELF - :type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor` + :type distributor: :class:`bridgedb.distributors.https.distributor.HTTPSDistributor` :param distributor: A bridge distributor. :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` diff --git a/bridgedb/https/templates/assets/css/bootstrap.min.css b/bridgedb/distributors/https/templates/assets/css/bootstrap.min.css similarity index 100% rename from bridgedb/https/templates/assets/css/bootstrap.min.css rename to bridgedb/distributors/https/templates/assets/css/bootstrap.min.css diff --git a/bridgedb/https/templates/assets/css/error.css b/bridgedb/distributors/https/templates/assets/css/error.css similarity index 100% rename from bridgedb/https/templates/assets/css/error.css rename to bridgedb/distributors/https/templates/assets/css/error.css diff --git a/bridgedb/https/templates/assets/css/font-awesome-ie7.min.css b/bridgedb/distributors/https/templates/assets/css/font-awesome-ie7.min.css similarity index 100% rename from bridgedb/https/templates/assets/css/font-awesome-ie7.min.css rename to bridgedb/distributors/https/templates/assets/css/font-awesome-ie7.min.css diff --git a/bridgedb/https/templates/assets/css/font-awesome.min.css b/bridgedb/distributors/https/templates/assets/css/font-awesome.min.css similarity index 100% rename from bridgedb/https/templates/assets/css/font-awesome.min.css rename to bridgedb/distributors/https/templates/assets/css/font-awesome.min.css diff --git a/bridgedb/https/templates/assets/css/main.css b/bridgedb/distributors/https/templates/assets/css/main.css similarity index 100% rename from bridgedb/https/templates/assets/css/main.css rename to bridgedb/distributors/https/templates/assets/css/main.css diff --git a/bridgedb/https/templates/assets/css/rtl.css b/bridgedb/distributors/https/templates/assets/css/rtl.css similarity index 100% rename from bridgedb/https/templates/assets/css/rtl.css rename to bridgedb/distributors/https/templates/assets/css/rtl.css diff --git a/bridgedb/https/templates/assets/font/fontawesome-webfont.eot b/bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.eot similarity index 100% rename from bridgedb/https/templates/assets/font/fontawesome-webfont.eot rename to bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.eot diff --git a/bridgedb/https/templates/assets/font/fontawesome-webfont.svg b/bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.svg similarity index 100% rename from bridgedb/https/templates/assets/font/fontawesome-webfont.svg rename to bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.svg diff --git a/bridgedb/https/templates/assets/font/fontawesome-webfont.ttf b/bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.ttf similarity index 100% rename from bridgedb/https/templates/assets/font/fontawesome-webfont.ttf rename to bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.ttf diff --git a/bridgedb/https/templates/assets/font/fontawesome-webfont.woff b/bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.woff similarity index 100% rename from bridgedb/https/templates/assets/font/fontawesome-webfont.woff rename to bridgedb/distributors/https/templates/assets/font/fontawesome-webfont.woff diff --git a/bridgedb/https/templates/assets/font/lato-bold.woff b/bridgedb/distributors/https/templates/assets/font/lato-bold.woff similarity index 100% rename from bridgedb/https/templates/assets/font/lato-bold.woff rename to bridgedb/distributors/https/templates/assets/font/lato-bold.woff diff --git a/bridgedb/https/templates/assets/font/lato-italic.woff b/bridgedb/distributors/https/templates/assets/font/lato-italic.woff similarity index 100% rename from bridgedb/https/templates/assets/font/lato-italic.woff rename to bridgedb/distributors/https/templates/assets/font/lato-italic.woff diff --git a/bridgedb/https/templates/assets/font/lato-regular.woff b/bridgedb/distributors/https/templates/assets/font/lato-regular.woff similarity index 100% rename from bridgedb/https/templates/assets/font/lato-regular.woff rename to bridgedb/distributors/https/templates/assets/font/lato-regular.woff diff --git a/bridgedb/https/templates/assets/images/404-excavator.svg b/bridgedb/distributors/https/templates/assets/images/404-excavator.svg similarity index 100% rename from bridgedb/https/templates/assets/images/404-excavator.svg rename to bridgedb/distributors/https/templates/assets/images/404-excavator.svg diff --git a/bridgedb/https/templates/assets/images/404-hills-left.svg b/bridgedb/distributors/https/templates/assets/images/404-hills-left.svg similarity index 100% rename from bridgedb/https/templates/assets/images/404-hills-left.svg rename to bridgedb/distributors/https/templates/assets/images/404-hills-left.svg diff --git a/bridgedb/https/templates/assets/images/404-hills-right.svg b/bridgedb/distributors/https/templates/assets/images/404-hills-right.svg similarity index 100% rename from bridgedb/https/templates/assets/images/404-hills-right.svg rename to bridgedb/distributors/https/templates/assets/images/404-hills-right.svg diff --git a/bridgedb/https/templates/assets/images/500-hills-left-shade.svg b/bridgedb/distributors/https/templates/assets/images/500-hills-left-shade.svg similarity index 100% rename from bridgedb/https/templates/assets/images/500-hills-left-shade.svg rename to bridgedb/distributors/https/templates/assets/images/500-hills-left-shade.svg diff --git a/bridgedb/https/templates/assets/images/500-hills-left.svg b/bridgedb/distributors/https/templates/assets/images/500-hills-left.svg similarity index 100% rename from bridgedb/https/templates/assets/images/500-hills-left.svg rename to bridgedb/distributors/https/templates/assets/images/500-hills-left.svg diff --git a/bridgedb/https/templates/assets/images/500-hills-right-shade.svg b/bridgedb/distributors/https/templates/assets/images/500-hills-right-shade.svg similarity index 100% rename from bridgedb/https/templates/assets/images/500-hills-right-shade.svg rename to bridgedb/distributors/https/templates/assets/images/500-hills-right-shade.svg diff --git a/bridgedb/https/templates/assets/images/500-hills-right.svg b/bridgedb/distributors/https/templates/assets/images/500-hills-right.svg similarity index 100% rename from bridgedb/https/templates/assets/images/500-hills-right.svg rename to bridgedb/distributors/https/templates/assets/images/500-hills-right.svg diff --git a/bridgedb/https/templates/assets/images/500-road.svg b/bridgedb/distributors/https/templates/assets/images/500-road.svg similarity index 100% rename from bridgedb/https/templates/assets/images/500-road.svg rename to bridgedb/distributors/https/templates/assets/images/500-road.svg diff --git a/bridgedb/https/templates/assets/images/maintenance-hills-left-shade.svg b/bridgedb/distributors/https/templates/assets/images/maintenance-hills-left-shade.svg similarity index 100% rename from bridgedb/https/templates/assets/images/maintenance-hills-left-shade.svg rename to bridgedb/distributors/https/templates/assets/images/maintenance-hills-left-shade.svg diff --git a/bridgedb/https/templates/assets/images/maintenance-hills-left.svg b/bridgedb/distributors/https/templates/assets/images/maintenance-hills-left.svg similarity index 100% rename from bridgedb/https/templates/assets/images/maintenance-hills-left.svg rename to bridgedb/distributors/https/templates/assets/images/maintenance-hills-left.svg diff --git a/bridgedb/https/templates/assets/images/maintenance-hills-right-shade.svg b/bridgedb/distributors/https/templates/assets/images/maintenance-hills-right-shade.svg similarity index 100% rename from bridgedb/https/templates/assets/images/maintenance-hills-right-shade.svg rename to bridgedb/distributors/https/templates/assets/images/maintenance-hills-right-shade.svg diff --git a/bridgedb/https/templates/assets/images/maintenance-hills-right.svg b/bridgedb/distributors/https/templates/assets/images/maintenance-hills-right.svg similarity index 100% rename from bridgedb/https/templates/assets/images/maintenance-hills-right.svg rename to bridgedb/distributors/https/templates/assets/images/maintenance-hills-right.svg diff --git a/bridgedb/https/templates/assets/images/maintenance-tractor.svg b/bridgedb/distributors/https/templates/assets/images/maintenance-tractor.svg similarity index 100% rename from bridgedb/https/templates/assets/images/maintenance-tractor.svg rename to bridgedb/distributors/https/templates/assets/images/maintenance-tractor.svg diff --git a/bridgedb/https/templates/assets/js/bridges.js b/bridgedb/distributors/https/templates/assets/js/bridges.js similarity index 100% rename from bridgedb/https/templates/assets/js/bridges.js rename to bridgedb/distributors/https/templates/assets/js/bridges.js diff --git a/bridgedb/https/templates/assets/tor-roots-blue.svg b/bridgedb/distributors/https/templates/assets/tor-roots-blue.svg similarity index 100% rename from bridgedb/https/templates/assets/tor-roots-blue.svg rename to bridgedb/distributors/https/templates/assets/tor-roots-blue.svg diff --git a/bridgedb/https/templates/assets/tor.svg b/bridgedb/distributors/https/templates/assets/tor.svg similarity index 100% rename from bridgedb/https/templates/assets/tor.svg rename to bridgedb/distributors/https/templates/assets/tor.svg diff --git a/bridgedb/https/templates/base.html b/bridgedb/distributors/https/templates/base.html similarity index 100% rename from bridgedb/https/templates/base.html rename to bridgedb/distributors/https/templates/base.html diff --git a/bridgedb/https/templates/bridges.html b/bridgedb/distributors/https/templates/bridges.html similarity index 100% rename from bridgedb/https/templates/bridges.html rename to bridgedb/distributors/https/templates/bridges.html diff --git a/bridgedb/https/templates/captcha.html b/bridgedb/distributors/https/templates/captcha.html similarity index 100% rename from bridgedb/https/templates/captcha.html rename to bridgedb/distributors/https/templates/captcha.html diff --git a/bridgedb/https/templates/error-404.html b/bridgedb/distributors/https/templates/error-404.html similarity index 100% rename from bridgedb/https/templates/error-404.html rename to bridgedb/distributors/https/templates/error-404.html diff --git a/bridgedb/https/templates/error-500.html b/bridgedb/distributors/https/templates/error-500.html similarity index 100% rename from bridgedb/https/templates/error-500.html rename to bridgedb/distributors/https/templates/error-500.html diff --git a/bridgedb/https/templates/error-503.html b/bridgedb/distributors/https/templates/error-503.html similarity index 100% rename from bridgedb/https/templates/error-503.html rename to bridgedb/distributors/https/templates/error-503.html diff --git a/bridgedb/https/templates/howto.html b/bridgedb/distributors/https/templates/howto.html similarity index 100% rename from bridgedb/https/templates/howto.html rename to bridgedb/distributors/https/templates/howto.html diff --git a/bridgedb/https/templates/index.html b/bridgedb/distributors/https/templates/index.html similarity index 100% rename from bridgedb/https/templates/index.html rename to bridgedb/distributors/https/templates/index.html diff --git a/bridgedb/https/templates/options.html b/bridgedb/distributors/https/templates/options.html similarity index 100% rename from bridgedb/https/templates/options.html rename to bridgedb/distributors/https/templates/options.html diff --git a/bridgedb/https/templates/robots.txt b/bridgedb/distributors/https/templates/robots.txt similarity index 100% rename from bridgedb/https/templates/robots.txt rename to bridgedb/distributors/https/templates/robots.txt diff --git a/bridgedb/email/__init__.py b/bridgedb/email/__init__.py deleted file mode 100644 index 604c648..0000000 --- a/bridgedb/email/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Servers for BridgeDB's email bridge distributor.""" diff --git a/bridgedb/parse/addr.py b/bridgedb/parse/addr.py index cd512d1..bf5a8a5 100644 --- a/bridgedb/parse/addr.py +++ b/bridgedb/parse/addr.py @@ -217,7 +217,7 @@ def canonicalizeEmailDomain(domain, domainmap): :param str domain: The domain portion of an email address to validate. It will be checked that it is one of the domains allowed to email requests for bridges to the - :class:`~bridgedb.email.distributor.EmailDistributor`. + :class:`~bridgedb.distributors.email.distributor.EmailDistributor`. :param dict domainmap: A map of permitted alternate domains (in lowercase) to their canonical domain names (in lowercase). This can be configured with the ``EMAIL_DOMAIN_MAP`` option in ``bridgedb.conf``, for @@ -442,7 +442,7 @@ def normalizeEmail(emailaddr, domainmap, domainrules, ignorePlus=True): The email address, **emailaddr**, will be parsed and validated, and then checked that it originated from one of the domains allowed to email requests for bridges to the - :class:`~bridgedb.email.distributor.EmailDistributor` via the + :class:`~bridgedb.distributors.email.distributor.EmailDistributor` via the :func:`canonicaliseEmailDomain` function. :param str emailaddr: An email address to normalise. diff --git a/bridgedb/persistent.py b/bridgedb/persistent.py index 4410b9b..c46c4a5 100644 --- a/bridgedb/persistent.py +++ b/bridgedb/persistent.py @@ -25,8 +25,8 @@ from twisted.spread import jelly from bridgedb import Bridges from bridgedb import filters -from bridgedb.email import distributor as emailDistributor -from bridgedb.https import distributor as httpsDistributor +from bridgedb.distributors.email import distributor as emailDistributor +from bridgedb.distributors.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet diff --git a/bridgedb/strings.py b/bridgedb/strings.py index 7d4b99e..1c40695 100644 --- a/bridgedb/strings.py +++ b/bridgedb/strings.py @@ -92,7 +92,7 @@ help to circumvent internet censorship in many cases.\n\n"""), } """These strings should go on the first "Welcome" email sent by the :mod:`~bridgedb.EmailServer`, as well as on the ``index.html`` template used -by the :mod:`~bridgedb.https.server`. They are used as an introduction to +by the :mod:`~bridgedb.distributors.https.server`. They are used as an introduction to explain what Tor bridges are, what bridges do, and why someone might want to use bridges. """ @@ -278,7 +278,7 @@ contain format specifiers (i.e. ``%s``) to what those format specifiers should be replaced with in a given template system. For example, a string which needs a pair of HTML ``("<a href=''">, "</a>")`` -tags (for the templates used by :mod:`bridgedb.https.server`) would need some +tags (for the templates used by :mod:`bridgedb.distributors.https.server`) would need some alternative replacements for the :mod:`EmailServer`, because the latter uses templates with a ``text/plain`` mimetype instead of HTML. For the ``EmailServer``, the format strings specifiers are replaced with an empty diff --git a/bridgedb/test/email_helpers.py b/bridgedb/test/email_helpers.py index 55b8033..645fc93 100644 --- a/bridgedb/test/email_helpers.py +++ b/bridgedb/test/email_helpers.py @@ -14,9 +14,9 @@ import io from bridgedb.persistent import Conf -from bridgedb.email.distributor import IgnoreEmail -from bridgedb.email.distributor import TooSoonEmail -from bridgedb.email.server import MailServerContext +from bridgedb.distributors.email.distributor import IgnoreEmail +from bridgedb.distributors.email.distributor import TooSoonEmail +from bridgedb.distributors.email.server import MailServerContext from bridgedb.schedule import Unscheduled from . import util @@ -121,7 +121,7 @@ def _createMailServerContext(config=None, distributor=None): class DummyEmailDistributor(object): - """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which is used + """A mocked :class:`bridgedb.distributors.email.distributor.EmailDistributor` which is used to test :class:`bridgedb.EmailServer`. """ @@ -145,9 +145,9 @@ class DummyEmailDistributor(object): class DummyEmailDistributorWithState(DummyEmailDistributor): - """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which raises - :exc:`bridgedb.email.distributor.TooSoonEmail` on the second email and - :exc:`bridgedb.email.distributor.IgnoreEmail` on the third. + """A mocked :class:`bridgedb.distributors.email.distributor.EmailDistributor` which raises + :exc:`bridgedb.distributors.email.distributor.TooSoonEmail` on the second email and + :exc:`bridgedb.distributors.email.distributor.IgnoreEmail` on the third. Note that the state tracking is done in a really dumb way. For example, we currently don't consider requests for help text or GnuPG keys to be a diff --git a/bridgedb/test/https_helpers.py b/bridgedb/test/https_helpers.py index 5c1c838..d54e544 100644 --- a/bridgedb/test/https_helpers.py +++ b/bridgedb/test/https_helpers.py @@ -113,8 +113,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE): class DummyHTTPSDistributor(object): - """A mocked :class:`bridgedb.https.distributor.HTTPSDistributor` which is - used to test :class:`bridgedb.https.server.BridgesResource`. + """A mocked :class:`bridgedb.distributors.https.distributor.HTTPSDistributor` which is + used to test :class:`bridgedb.distributors.https.server.BridgesResource`. """ _bridge_class = util.DummyBridge _bridgesPerResponseMin = 3 diff --git a/bridgedb/test/legacy_Tests.py b/bridgedb/test/legacy_Tests.py index 22b2e13..77409b4 100644 --- a/bridgedb/test/legacy_Tests.py +++ b/bridgedb/test/legacy_Tests.py @@ -26,9 +26,9 @@ import ipaddr from bridgedb.Stability import BridgeHistory -from bridgedb.email.distributor import EmailDistributor -from bridgedb.email.distributor import IgnoreEmail -from bridgedb.email.distributor import TooSoonEmail +from bridgedb.distributors.email.distributor import EmailDistributor +from bridgedb.distributors.email.distributor import IgnoreEmail +from bridgedb.distributors.email.distributor import TooSoonEmail from bridgedb.parse import addr from .util import bracketIPv6 diff --git a/bridgedb/test/test_email_autoresponder.py b/bridgedb/test/test_email_autoresponder.py index 6bdac9e..a1e376f 100644 --- a/bridgedb/test/test_email_autoresponder.py +++ b/bridgedb/test/test_email_autoresponder.py @@ -9,7 +9,7 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests for the :mod:`bridgedb.email.autoresponder` module.""" +"""Unittests for the :mod:`bridgedb.distributors.email.autoresponder` module.""" from __future__ import print_function @@ -23,9 +23,9 @@ from twisted.python.failure import Failure from twisted.trial import unittest from twisted.test import proto_helpers -from bridgedb.email import autoresponder -from bridgedb.email.server import SMTPMessage -from bridgedb.email.distributor import TooSoonEmail +from bridgedb.distributors.email import autoresponder +from bridgedb.distributors.email.server import SMTPMessage +from bridgedb.distributors.email.distributor import TooSoonEmail from bridgedb.test.email_helpers import _createConfig from bridgedb.test.email_helpers import _createMailServerContext @@ -33,7 +33,7 @@ from bridgedb.test.email_helpers import DummyEmailDistributorWithState class CreateResponseBodyTests(unittest.TestCase): - """Tests for :func:`bridgedb.email.autoresponder.createResponseBody`.""" + """Tests for :func:`bridgedb.distributors.email.autoresponder.createResponseBody`.""" def _moveGPGTestKeyfile(self): here = os.getcwd() @@ -289,7 +289,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" class SMTPAutoresponderTests(unittest.TestCase): - """Unittests for :class:`bridgedb.email.autoresponder.SMTPAutoresponder`.""" + """Unittests for :class:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder`.""" timeout = 10 @@ -322,7 +322,7 @@ class SMTPAutoresponderTests(unittest.TestCase): self.responder = self.message.responder # The following are needed to provide client disconnection methods for # the call to ``twisted.mail.smtp.SMTPClient.sendError`` in - # ``bridgedb.email.autoresponder.SMTPAutoresponder.sendError``: + # ``bridgedb.distributors.email.autoresponder.SMTPAutoresponder.sendError``: #protocol = proto_helpers.AccumulatingProtocol() #transport = proto_helpers.StringTransportWithDisconnection() self.tr = proto_helpers.StringTransportWithDisconnection() diff --git a/bridgedb/test/test_email_distributor.py b/bridgedb/test/test_email_distributor.py index 0d727b8..d95370f 100644 --- a/bridgedb/test/test_email_distributor.py +++ b/bridgedb/test/test_email_distributor.py @@ -7,7 +7,7 @@ # (c) 2007-2017, The Tor Project, Inc. # :license: see included LICENSE for information -"""Tests for :mod:`bridgedb.email.distributor`.""" +"""Tests for :mod:`bridgedb.distributors.email.distributor`.""" from __future__ import print_function @@ -21,10 +21,10 @@ from twisted.trial import unittest import bridgedb.Storage from bridgedb.bridges import Bridge -from bridgedb.email.distributor import EmailDistributor -from bridgedb.email.distributor import IgnoreEmail -from bridgedb.email.distributor import TooSoonEmail -from bridgedb.email.request import EmailBridgeRequest +from bridgedb.distributors.email.distributor import EmailDistributor +from bridgedb.distributors.email.distributor import IgnoreEmail +from bridgedb.distributors.email.distributor import TooSoonEmail +from bridgedb.distributors.email.request import EmailBridgeRequest from bridgedb.parse.addr import BadEmail from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import normalizeEmail @@ -38,7 +38,7 @@ BRIDGES = generateFakeBridges() class EmailDistributorTests(unittest.TestCase): - """Tests for :class:`bridgedb.email.distributor.EmailDistributor`.""" + """Tests for :class:`bridgedb.distributors.email.distributor.EmailDistributor`.""" # Fail any tests which take longer than 15 seconds. timeout = 15 diff --git a/bridgedb/test/test_email_dkim.py b/bridgedb/test/test_email_dkim.py index 499a3c1..dc4c53b 100644 --- a/bridgedb/test/test_email_dkim.py +++ b/bridgedb/test/test_email_dkim.py @@ -9,14 +9,14 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests for the :mod:`bridgedb.email.dkim` module.""" +"""Unittests for the :mod:`bridgedb.distributors.email.dkim` module.""" import io from twisted.mail.smtp import rfc822 from twisted.trial import unittest -from bridgedb.email import dkim +from bridgedb.distributors.email import dkim class CheckDKIMTests(unittest.TestCase): diff --git a/bridgedb/test/test_email_request.py b/bridgedb/test/test_email_request.py index 745ea71..0c87d36 100644 --- a/bridgedb/test/test_email_request.py +++ b/bridgedb/test/test_email_request.py @@ -9,7 +9,7 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests for the :mod:`bridgedb.email.request` module.""" +"""Unittests for the :mod:`bridgedb.distributors.email.request` module.""" from __future__ import print_function @@ -17,7 +17,7 @@ import ipaddr from twisted.trial import unittest -from bridgedb.email import request +from bridgedb.distributors.email import request class DetermineBridgeRequestOptionsTests(unittest.TestCase): diff --git a/bridgedb/test/test_email_server.py b/bridgedb/test/test_email_server.py index 946133c..edf2539 100644 --- a/bridgedb/test/test_email_server.py +++ b/bridgedb/test/test_email_server.py @@ -9,7 +9,7 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests for the :mod:`bridgedb.email.server` module.""" +"""Unittests for the :mod:`bridgedb.distributors.email.server` module.""" from __future__ import print_function @@ -31,9 +31,9 @@ from twisted.trial import unittest from zope.interface import implementedBy -from bridgedb.email import server -from bridgedb.email.distributor import EmailDistributor -from bridgedb.email.distributor import TooSoonEmail +from bridgedb.distributors.email import server +from bridgedb.distributors.email.distributor import EmailDistributor +from bridgedb.distributors.email.distributor import TooSoonEmail from bridgedb.parse.addr import BadEmail from bridgedb.schedule import Unscheduled @@ -43,7 +43,7 @@ from bridgedb.test.email_helpers import _createMailServerContext class SMTPMessageTests(unittest.TestCase): - """Unittests for :class:`bridgedb.email.server.SMTPMessage`.""" + """Unittests for :class:`bridgedb.distributors.email.server.SMTPMessage`.""" def setUp(self): self.config = _createConfig() @@ -59,7 +59,7 @@ class SMTPMessageTests(unittest.TestCase): def test_SMTPMessage_init(self): """Our ``message`` attribute should be a ``SMTPMessage`` object, and ``message.responder`` should be a - :class:`bridgedb.email.autoresponder.SMTPAutoresponder`. + :class:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder`. """ self.assertIsInstance(self.message, server.SMTPMessage) self.assertIsInstance(self.message.responder, @@ -320,7 +320,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin): segment. Includes the SMTP DATA EOM command ('.') at the end. If no keyword arguments are given, the defaults are fairly sane. - Suitable for testing a :class:`bridgedb.email.server.SMTPIncomingServerFactory`. + Suitable for testing a :class:`bridgedb.distributors.email.server.SMTPIncomingServerFactory`. :param str fromAddr: An email address for the 'From:' header. :param str toAddr: An email address for the 'To:' header. @@ -382,7 +382,7 @@ class SMTPTestCaseMixin(util.TestCaseMixin): class SMTPIncomingServerFactoryTests(SMTPTestCaseMixin, unittest.TestCase): - """Unittests for :class:`bridgedb.email.server.SMTPIncomingServerFactory`.""" + """Unittests for :class:`bridgedb.distributors.email.server.SMTPIncomingServerFactory`.""" def setUp(self): """Set up a localhost SMTPIncomingServerFactory handler incoming SMTP @@ -502,7 +502,7 @@ class SMTPIncomingServerFactoryTests(SMTPTestCaseMixin, unittest.TestCase): class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase): - """Unittests for :func:`bridgedb.email.server.addServer`.""" + """Unittests for :func:`bridgedb.distributors.email.server.addServer`.""" def setUp(self): """Create a MailServerContext and EmailDistributor.""" @@ -523,7 +523,7 @@ class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase): reactor.runUntilCurrent() def test_addServer(self): - """Call :func:`bridgedb.email.server.addServer` to test startup.""" + """Call :func:`bridgedb.distributors.email.server.addServer` to test startup.""" factory = server.addServer(self.config, self.dist) factory.timeout = None factory.protocol.timeout = None # Or else the reactor gets dirty diff --git a/bridgedb/test/test_email_templates.py b/bridgedb/test/test_email_templates.py index 0c74377..8464fe9 100644 --- a/bridgedb/test/test_email_templates.py +++ b/bridgedb/test/test_email_templates.py @@ -9,7 +9,7 @@ # (c) 2007-2013, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests for the :mod:`bridgedb.email.templates` module.""" +"""Unittests for the :mod:`bridgedb.distributors.email.templates` module.""" from __future__ import print_function from __future__ import unicode_literals @@ -20,7 +20,7 @@ from gettext import NullTranslations from twisted.mail.smtp import Address from twisted.trial import unittest -from bridgedb.email import templates +from bridgedb.distributors.email import templates class EmailTemplatesTests(unittest.TestCase): diff --git a/bridgedb/test/test_https_distributor.py b/bridgedb/test/test_https_distributor.py index 0d20960..791a940 100644 --- a/bridgedb/test/test_https_distributor.py +++ b/bridgedb/test/test_https_distributor.py @@ -9,7 +9,7 @@ # (c) 2007-2017, all entities within the AUTHORS file # :license: 3-clause BSD, see included LICENSE for information -"""Tests for :mod:`bridgedb.https.distributor`.""" +"""Tests for :mod:`bridgedb.distributors.https.distributor`.""" from __future__ import print_function @@ -23,8 +23,8 @@ from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import BridgeRingParameters from bridgedb.filters import byIPv4 from bridgedb.filters import byIPv6 -from bridgedb.https import distributor -from bridgedb.https.request import HTTPSBridgeRequest +from bridgedb.distributors.https import distributor +from bridgedb.distributors.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet from bridgedb.test.util import randomValidIPv4String diff --git a/bridgedb/test/test_https_request.py b/bridgedb/test/test_https_request.py index c262aef..2ea3fe6 100644 --- a/bridgedb/test/test_https_request.py +++ b/bridgedb/test/test_https_request.py @@ -14,7 +14,7 @@ from twisted.trial import unittest from bridgedb.bridgerequest import IRequestBridges -from bridgedb.https import request +from bridgedb.distributors.https import request class MockRequest(object): @@ -23,7 +23,7 @@ class MockRequest(object): class HTTPSBridgeRequestTests(unittest.TestCase): - """Unittests for :class:`bridgedb.https.request.HTTPSBridgeRequest`.""" + """Unittests for :class:`bridgedb.distributors.https.request.HTTPSBridgeRequest`.""" def setUp(self): """Setup test run.""" diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 3827bfe..2c41359 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -9,7 +9,7 @@ # :license: see LICENSE for licensing information #_____________________________________________________________________________ -"""Unittests for :mod:`bridgedb.https.server`.""" +"""Unittests for :mod:`bridgedb.distributors.https.server`.""" from __future__ import print_function @@ -27,7 +27,7 @@ from twisted.trial import unittest from twisted.web.resource import Resource from twisted.web.test import requesthelper -from bridgedb.https import server +from bridgedb.distributors.https import server from bridgedb.schedule import ScheduledInterval from bridgedb.test.https_helpers import _createConfig @@ -44,8 +44,8 @@ logging.disable(50) class SetFQDNTests(unittest.TestCase): - """Tests for :func:`bridgedb.https.server.setFQDN` and - :func:`bridgedb.https.server.setFQDN`. + """Tests for :func:`bridgedb.distributors.https.server.setFQDN` and + :func:`bridgedb.distributors.https.server.setFQDN`. """ def setUp(self): @@ -72,7 +72,7 @@ class SetFQDNTests(unittest.TestCase): class GetClientIPTests(unittest.TestCase): - """Tests for :func:`bridgedb.https.server.getClientIP`.""" + """Tests for :func:`bridgedb.distributors.https.server.getClientIP`.""" def createRequestWithIPs(self): """Set the IP address returned from ``request.getClientIP()`` to @@ -114,7 +114,7 @@ class GetClientIPTests(unittest.TestCase): class ReplaceErrorPageTests(unittest.TestCase): - """Tests for :func:`bridgedb.https.server.replaceErrorPage`.""" + """Tests for :func:`bridgedb.distributors.https.server.replaceErrorPage`.""" def setUp(self): self.resource500 = server.resource500 @@ -152,7 +152,7 @@ class ReplaceErrorPageTests(unittest.TestCase): errorPage) class ErrorResourceTests(unittest.TestCase): - """Tests for :class:`bridgedb.https.server.ErrorResource`.""" + """Tests for :class:`bridgedb.distributors.https.server.ErrorResource`.""" def setUp(self): self.request = DummyRequest(['']) @@ -174,7 +174,7 @@ class ErrorResourceTests(unittest.TestCase): class CustomErrorHandlingResourceTests(unittest.TestCase): - """Tests for :class:`bridgedb.https.server.CustomErrorHandlingResource`.""" + """Tests for :class:`bridgedb.distributors.https.server.CustomErrorHandlingResource`.""" def test_getChild(self): """``CustomErrorHandlingResource.getChild`` should return a rendered @@ -258,7 +258,7 @@ class CSPResourceTests(unittest.TestCase): class IndexResourceTests(unittest.TestCase): - """Test for :class:`bridgedb.https.server.IndexResource`.""" + """Test for :class:`bridgedb.distributors.https.server.IndexResource`.""" def setUp(self): self.pagename = '' @@ -283,7 +283,7 @@ class IndexResourceTests(unittest.TestCase): class HowtoResourceTests(unittest.TestCase): - """Test for :class:`bridgedb.https.server.HowtoResource`.""" + """Test for :class:`bridgedb.distributors.https.server.HowtoResource`.""" def setUp(self): self.pagename = 'howto.html' @@ -308,7 +308,7 @@ class HowtoResourceTests(unittest.TestCase): class CaptchaProtectedResourceTests(unittest.TestCase): - """Tests for :class:`bridgedb.https.server.CaptchaProtectedResource`.""" + """Tests for :class:`bridgedb.distributors.https.server.CaptchaProtectedResource`.""" def setUp(self): self.dist = None @@ -391,7 +391,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase): class GimpCaptchaProtectedResourceTests(unittest.TestCase): - """Tests for :mod:`bridgedb.https.server.GimpCaptchaProtectedResource`.""" + """Tests for :mod:`bridgedb.distributors.https.server.GimpCaptchaProtectedResource`.""" def setUp(self): """Create a :class:`server.BridgesResource` and protect it with @@ -518,7 +518,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): class ReCaptchaProtectedResourceTests(unittest.TestCase): - """Tests for :mod:`bridgedb.https.server.ReCaptchaProtectedResource`.""" + """Tests for :mod:`bridgedb.distributors.https.server.ReCaptchaProtectedResource`.""" def setUp(self): """Create a :class:`server.BridgesResource` and protect it with @@ -947,7 +947,7 @@ class BridgesResourceTests(unittest.TestCase): class OptionsResourceTests(unittest.TestCase): - """Tests for :class:`bridgedb.https.server.OptionsResource`.""" + """Tests for :class:`bridgedb.distributors.https.server.OptionsResource`.""" def setUp(self): """Create a :class:`server.OptionsResource`.""" @@ -973,7 +973,7 @@ class OptionsResourceTests(unittest.TestCase): class HTTPSServerServiceTests(unittest.TestCase): - """Unittests for :func:`bridgedb.email.server.addWebServer`.""" + """Unittests for :func:`bridgedb.distributors.email.server.addWebServer`.""" def setUp(self): """Create a config and an HTTPSDistributor.""" @@ -998,29 +998,29 @@ class HTTPSServerServiceTests(unittest.TestCase): reactor.runUntilCurrent() def test_addWebServer_GIMP_CAPTCHA_ENABLED(self): - """Call :func:`bridgedb.https.server.addWebServer` to test startup.""" + """Call :func:`bridgedb.distributors.https.server.addWebServer` to test startup.""" server.addWebServer(self.config, self.distributor) def test_addWebServer_RECAPTCHA_ENABLED(self): - """Call :func:`bridgedb.https.server.addWebServer` to test startup.""" + """Call :func:`bridgedb.distributors.https.server.addWebServer` to test startup.""" config = self.config config.RECAPTCHA_ENABLED = True server.addWebServer(config, self.distributor) def test_addWebServer_no_captchas(self): - """Call :func:`bridgedb.https.server.addWebServer` to test startup.""" + """Call :func:`bridgedb.distributors.https.server.addWebServer` to test startup.""" config = self.config config.GIMP_CAPTCHA_ENABLED = False server.addWebServer(config, self.distributor) def test_addWebServer_no_HTTPS_ROTATION_PERIOD(self): - """Call :func:`bridgedb.https.server.addWebServer` to test startup.""" + """Call :func:`bridgedb.distributors.https.server.addWebServer` to test startup.""" config = self.config config.HTTPS_ROTATION_PERIOD = None server.addWebServer(config, self.distributor) def test_addWebServer_CSP_ENABLED_False(self): - """Call :func:`bridgedb.https.server.addWebServer` with + """Call :func:`bridgedb.distributors.https.server.addWebServer` with ``CSP_ENABLED=False`` to test startup. """ config = self.config @@ -1028,7 +1028,7 @@ class HTTPSServerServiceTests(unittest.TestCase): server.addWebServer(config, self.distributor) def test_addWebServer_CSP_REPORT_ONLY_False(self): - """Call :func:`bridgedb.https.server.addWebServer` with + """Call :func:`bridgedb.distributors.https.server.addWebServer` with ``CSP_REPORT_ONLY=False`` to test startup. """ config = self.config @@ -1036,7 +1036,7 @@ class HTTPSServerServiceTests(unittest.TestCase): server.addWebServer(config, self.distributor) def test_addWebServer_CSP_INCLUDE_SELF_False(self): - """Call :func:`bridgedb.https.server.addWebServer` with + """Call :func:`bridgedb.distributors.https.server.addWebServer` with ``CSP_INCLUDE_SELF=False`` to test startup. """ config = self.config diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index f1f5fa4..906d5fe 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -34,19 +34,19 @@ import bridgedb.Bridges import bridgedb.Bucket import bridgedb.crypto import bridgedb.distribute -import bridgedb.email -import bridgedb.email.autoresponder -import bridgedb.email.distributor -import bridgedb.email.dkim -import bridgedb.email.request -import bridgedb.email.server -import bridgedb.email.templates +import bridgedb.distributors.email +import bridgedb.distributors.email.autoresponder +import bridgedb.distributors.email.distributor +import bridgedb.distributors.email.dkim +import bridgedb.distributors.email.request +import bridgedb.distributors.email.server +import bridgedb.distributors.email.templates import bridgedb.filters import bridgedb.geo -import bridgedb.https -import bridgedb.https.distributor -import bridgedb.https.request -import bridgedb.https.server +import bridgedb.distributors.https +import bridgedb.distributors.https.distributor +import bridgedb.distributors.https.request +import bridgedb.distributors.https.server import bridgedb.Main import bridgedb.parse import bridgedb.parse.addr diff --git a/setup.py b/setup.py index 955a720..a133424 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ repo_langs = os.path.join(pkgpath, '_langs.py') # The directory containing template files and other resources to serve on the # web server: -repo_templates = os.path.join(pkgpath, 'https', 'templates') +repo_templates = os.path.join(pkgpath, 'distributors', 'https', 'templates') # The directories to install non-sourcecode resources into should always be # given as relative paths, in order to force distutils to install relative to @@ -375,8 +375,9 @@ setuptools.setup( download_url='https://gitweb.torproject.org/bridgedb.git', package_dir={'bridgedb': 'bridgedb'}, packages=['bridgedb', - 'bridgedb.email', - 'bridgedb.https', + 'bridgedb.distributors', + 'bridgedb.distributors.email', + 'bridgedb.distributors.https', 'bridgedb.parse', 'bridgedb.test', ], @@ -395,7 +396,7 @@ setuptools.setup( message_extractors={ pkgpath: [ ('**.py', 'python', None), - ('https/templates/**.html', 'mako', None), + ('distributors/https/templates/**.html', 'mako', None), ] }, )

1 0

[bridgedb/develop] Fix bridgedb.main erroring unittests due to multi-bridgeauth code.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit c2ffe4f6785d846a083ffc347f13759e5d748c96 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 23:51:39 2017 +0000 Fix bridgedb.main erroring unittests due to multi-bridgeauth code. --- bridgedb/main.py | 7 +++++- bridgedb/test/test_main.py | 53 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/bridgedb/main.py b/bridgedb/main.py index f6d16ec..cb3eee1 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -46,7 +46,12 @@ def expandBridgeAuthDir(authdir, filename): """Expands a descriptor ``filename`` relative to which of the BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. """ - return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + path = filename + + if not authdir in filename or not os.path.isabs(filename): + path = os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + + return path def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py index fddb8a9..a2e975e 100644 --- a/bridgedb/test/test_main.py +++ b/bridgedb/test/test_main.py @@ -81,9 +81,12 @@ class BridgedbTests(unittest.TestCase): fh.flush() fh.close() - def _copyDescFilesHere(self, files): + def _copyDescFilesHere(self, authdirs, files): """Copy all the **files** to the _trial_tmp/ directory. + :param list authdirs: A list of strings representing the directories + from BridgeAuthorities, as in the BRIDGE_AUTHORITY_DIRECTORIES + config option. :param list files: A list of strings representing the paths to descriptor files. This should probably be taken from a ``bridgedb.persistent.Conf`` object which has parsed the @@ -95,18 +98,23 @@ class BridgedbTests(unittest.TestCase): """ updatedPaths = [] - for f in files: - base = os.path.basename(f) - src = os.path.join(CI_RUNDIR, base) - if os.path.isfile(src): - dst = os.path.join(HERE, base) - shutil.copy(src, dst) - updatedPaths.append(dst) - else: - self.skip = True - raise unittest.SkipTest( - "Can't find mock descriptor files in %s directory" % - CI_RUNDIR) + for d in authdirs: + for f in files: + base = os.path.basename(f) + src = os.path.join(CI_RUNDIR, d, base) + if os.path.isfile(src): + dstdir = os.path.join(HERE, d) + if not os.path.isdir(dstdir): + os.mkdir(dstdir) + self._directories_created.append(dstdir) + dst = os.path.join(dstdir, base) + shutil.copy(src, dst) + updatedPaths.append(dst) + else: + self.skip = True + raise unittest.SkipTest( + "Can't find mock descriptor files in %s directory" % + CI_RUNDIR) return updatedPaths @@ -157,14 +165,24 @@ class BridgedbTests(unittest.TestCase): directory, produce a state object from the loaded bridgedb.conf file, and make an HMAC key. """ + # We'll want to nuke these after the test runs + self._directories_created = [] + # Get the bridgedb.conf file in the top-level directory of this repo: self.configFile = os.path.join(TOPDIR, 'bridgedb.conf') self.config = main.loadConfig(self.configFile) + self.config.BRIDGE_AUTHORITY_DIRECTORIES = ["from-bifroest"] # Copy the referenced descriptor files from bridgedb/run/ to CWD: - self.config.STATUS_FILE = self._copyDescFilesHere([self.config.STATUS_FILE])[0] - self.config.BRIDGE_FILES = self._copyDescFilesHere(self.config.BRIDGE_FILES) - self.config.EXTRA_INFO_FILES = self._copyDescFilesHere(self.config.EXTRA_INFO_FILES) + self.config.STATUS_FILE = self._copyDescFilesHere( + self.config.BRIDGE_AUTHORITY_DIRECTORIES, + [self.config.STATUS_FILE])[0] + self.config.BRIDGE_FILES = self._copyDescFilesHere( + self.config.BRIDGE_AUTHORITY_DIRECTORIES, + self.config.BRIDGE_FILES) + self.config.EXTRA_INFO_FILES = self._copyDescFilesHere( + self.config.BRIDGE_AUTHORITY_DIRECTORIES, + self.config.EXTRA_INFO_FILES) # Initialise the state self.state = main.persistent.State(**self.config.__dict__) @@ -185,6 +203,9 @@ class BridgedbTests(unittest.TestCase): main.updateBridgeHistory = self._orig_updateBridgeHistory sys.argv = self._orig_sys_argv + for d in self._directories_created: + shutil.rmtree(d) + def test_main_updateBridgeHistory(self): """main.updateBridgeHistory should update some timestamps for some bridges.

1 0

[bridgedb/develop] Upgrade coveralls dependency to 1.2.0.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 93fc704cf3350313512df0d41d0ea7dd7b5901ac Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 20:54:38 2017 +0000 Upgrade coveralls dependency to 1.2.0. --- .travis.requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.requirements.txt b/.travis.requirements.txt index 092d4f2..e05e643 100644 --- a/.travis.requirements.txt +++ b/.travis.requirements.txt @@ -14,7 +14,7 @@ # #------------------------------------------------------------------------------ coverage==4.2 -coveralls==0.4.2 +coveralls==1.2.0 git+https://git.torproject.org/user/isis/leekspin.git@bad0bed11a9018f65555b… mechanize==0.2.5 sure==1.2.2

1 0

[bridgedb/develop] Add more tests for bridgedb.main.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 00a9288a141158552ee56790211ed230704775d7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 2 00:27:31 2017 +0000 Add more tests for bridgedb.main. --- bridgedb/main.py | 4 ++-- bridgedb/test/test_main.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/bridgedb/main.py b/bridgedb/main.py index cb3eee1..d155a28 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -459,7 +459,7 @@ def run(options, reactor=reactor): signal.signal(signal.SIGHUP, _handleSIGHUP) signal.signal(signal.SIGUSR1, _handleSIGUSR1) - if reactor: + if reactor: # pragma: no cover # And actually load it to start parsing. Get back our distributors. emailDistributor, ipDistributor = reload(False) @@ -508,7 +508,7 @@ def run(options, reactor=reactor): if reactor and not reactor.running: logging.info("Starting reactors.") reactor.run() - except KeyboardInterrupt: + except KeyboardInterrupt: # pragma: no cover logging.fatal("Received keyboard interrupt. Shutting down...") finally: if config.PIDFILE: diff --git a/bridgedb/test/test_main.py b/bridgedb/test/test_main.py index a2e975e..ab34055 100644 --- a/bridgedb/test/test_main.py +++ b/bridgedb/test/test_main.py @@ -71,6 +71,20 @@ class MockHashring(object): pass +class ExpandBridgeAuthDirTests(unittest.TestCase): + """Unittests for :func:`bridgedb.main.expandBridgeAuthDir`.""" + + def setUp(self): + self.authdir = "from-authority" + self.filename = "bridge-descriptors" + + def test_expandBridgeAuthDir_not_abs(self): + """A non-absolute path should turn into an absolute one.""" + result = main.expandBridgeAuthDir(self.authdir, self.filename) + + self.assertTrue(os.path.isabs(result)) + + class BridgedbTests(unittest.TestCase): """Integration tests for :func:`bridgedb.main.load`.""" @@ -229,6 +243,15 @@ class BridgedbTests(unittest.TestCase): d.addErrback(self._eb_Failure) return d + def test_main_load_then_reload(self): + """main.load() should run without error.""" + d = deferToThread(main.load, self.state, self.hashring) + d.addCallback(self._cbAssertFingerprints) + d.addErrback(self._eb_Failure) + d.addCallback(main._reloadFn) + d.addErrback(self._eb_Failure) + return d + def test_main_load_no_state(self): """main.load() should raise SystemExit without a state object.""" self.assertRaises(SystemExit, main.load, None, self.hashring)

1 0

[bridgedb/develop] Fix typo in docstring.
by isis＠torproject.org 15 Nov '17

15 Nov '17

commit 5a09a99ba05412e8f08323a54abeb927c024c57e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 1 20:56:03 2017 +0000 Fix typo in docstring. --- bridgedb/distributors/https/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index 3e4284f..85a32b3 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -81,7 +81,7 @@ SERVER_PUBLIC_FQDN = None def setFQDN(fqdn, https=True): - """Set the global :data:`SERVER_PUBLIC FQDN` variable. + """Set the global :data:`SERVER_PUBLIC_FQDN` variable. :param str fqdn: The public, fully-qualified domain name of the HTTP server that will serve this resource.

1 0