[bridgedb/develop] Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.

commit dc91ea443e66223999c5b7a70c4b42025c0ac25c Author: Isis Lovecruft isis@torproject.org Date: Tue Apr 21 07:32:49 2015 +0000

Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.

* FIXES part of #12506: https://bugs.torproject.org/12506 --- doc/sphinx/source/bridgedb.https.rst | 1 + doc/sphinx/source/conf.py | 1 + lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 302 ------------------- lib/bridgedb/Main.py | 21 +- lib/bridgedb/https/distributor.py | 328 ++++++++++++++++++++ lib/bridgedb/https/request.py | 2 +- lib/bridgedb/https/server.py | 2 +- lib/bridgedb/persistent.py | 3 +- lib/bridgedb/test/https_helpers.py | 4 +- lib/bridgedb/test/legacy_Tests.py | 68 +---- lib/bridgedb/test/test_Dist.py | 434 --------------------------- lib/bridgedb/test/test_https_distributor.py | 405 +++++++++++++++++++++++++ lib/bridgedb/test/test_https_server.py | 2 +- lib/bridgedb/test/util.py | 33 ++ 15 files changed, 797 insertions(+), 811 deletions(-)

diff --git a/doc/sphinx/source/bridgedb.https.rst b/doc/sphinx/source/bridgedb.https.rst index 1512065..36fa6c6 100644 --- a/doc/sphinx/source/bridgedb.https.rst +++ b/doc/sphinx/source/bridgedb.https.rst @@ -7,5 +7,6 @@ bridgedb.https :depth: 3

.. automodule:: bridgedb.https.__init__ +.. automodule:: bridgedb.https.distributor .. automodule:: bridgedb.https.request .. automodule:: bridgedb.https.server diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 9b43d0f..5038abb 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -43,6 +43,7 @@ import bridgedb.email.templates import bridgedb.filters import bridgedb.geo import bridgedb.https +import bridgedb.https.distributor import bridgedb.https.request import bridgedb.https.server import bridgedb.Main diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 2b56884..7a237fe 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -510,7 +510,7 @@ class FilteredBridgeSplitter(object): :ivar bridges: DOCDOC :type distributorName: str :ivar distributorName: The name of this splitter's distributor. See - :meth:`bridgedb.Dist.HTTPSDistributor.setDistributorName`. + :meth:`~bridgedb.https.distributor.HTTPSDistributor.setDistributorName`. """ self.key = key self.filterRings = {} diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 9b9e35c..1be705f 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -12,14 +12,11 @@

"""This module has functions to decide which bridges to hand out to whom."""

-import ipaddr import logging -import re import time

import bridgedb.Storage

-from bridgedb import proxy from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC @@ -47,305 +44,6 @@ class EmailRequestedKey(Exception): """Raised when an incoming email requested a copy of our GnuPG keys."""

-class HTTPSDistributor(Distributor): - """A Distributor that hands out bridges based on the IP address of an - incoming request and the current time period. - - :type proxies: :class:`~bridgedb.proxies.ProxySet` - :ivar proxies: All known proxies, which we treat differently. See - :param:`proxies`. - :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter` - :ivar hashring: A hashring that assigns bridges to subrings with fixed - proportions. Used to assign bridges into the subrings of this - distributor. - """ - - def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): - """Create a Distributor that decides which bridges to distribute based - upon the client's IP address and the current time. - - :param int totalSubrings: The number of subhashrings to group clients - into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, - then the actual number of clusters is one higher than - ``totalSubrings``, because the set of all known open proxies is - given its own subhashring. - :param bytes key: The master HMAC key for this distributor. All added - bridges are HMACed with this key in order to place them into the - hashrings. - :type proxies: :class:`~bridgedb.proxy.ProxySet` - :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known - Tor Exit relays and other known proxies. These will constitute - the extra cluster, and any client requesting bridges from one of - these **proxies** will be distributed bridges from a separate - subhashring that is specific to Tor/proxy users. - :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` - :param answerParameters: A mechanism for ensuring that the set of - bridges that this distributor answers a client with fit certain - parameters, i.e. that an answer has "at least two obfsproxy - bridges" or "at least one bridge on port 443", etc. - """ - super(HTTPSDistributor, self).__init__(key) - self.totalSubrings = totalSubrings - self.answerParameters = answerParameters - - if proxies: - logging.info("Added known proxies to HTTPS distributor...") - self.proxies = proxies - self.totalSubrings += 1 - self.proxySubring = self.totalSubrings - else: - logging.warn("No known proxies were added to HTTPS distributor!") - self.proxies = proxy.ProxySet() - self.proxySubring = 0 - - self.ringCacheSize = self.totalSubrings * 3 - - key2 = getHMAC(key, "Assign-Bridges-To-Rings") - key3 = getHMAC(key, "Order-Areas-In-Rings") - key4 = getHMAC(key, "Assign-Areas-To-Rings") - - self._clientToPositionHMAC = getHMACFunc(key3, hex=False) - self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) - self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) - self.name = 'HTTPS' - logging.debug("Added %s to %s distributor." % - (self.hashring.__class__.__name__, self.name)) - - def bridgesPerResponse(self, hashring=None): - return super(HTTPSDistributor, self).bridgesPerResponse(hashring) - - @classmethod - def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): - """Map all clients whose **ip**s are within the same subnet to the same - arbitrary string. - - .. hint:: For non-proxy IP addresses, any two IPv4 addresses within - the same ``/16`` subnet, or any two IPv6 addresses in the same - ``/32`` subnet, will get the same string. - - Subnets for this distributor are grouped into the number of rings - specified by the ``N_IP_CLUSTERS`` configuration option, such that - Alice (with the address ``1.2.3.4`` and Bob (with the address - ``1.2.178.234``) are placed within the same cluster, but Carol (with - address ``1.3.11.33``) *might* end up in a different cluster. - - >>> from bridgedb.Dist import HTTPSDistributor - >>> HTTPSDistributor.getSubnet('1.2.3.4') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('1.2.211.154') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') - '2001:f::/32' - >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') - '2a00:c98::/32' - - :param str ip: A string representing an IPv4 or IPv6 address. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - :param int proxySubnets: Place Tor/proxy users into this number of - "subnet" groups. This means that no matter how many different Tor - Exits or proxies a client uses, the most they can ever get is - **proxySubnets** different sets of bridge lines (per interval). - This parameter only has any effect when **usingProxy** is ``True``. - :rtype: str - :returns: The appropriately sized CIDR subnet representation of the **ip**. - """ - if not usingProxy: - # We aren't using bridgedb.parse.addr.isIPAddress(ip, - # compressed=False) here because adding the string "False" into - # the map would land any and all clients whose IP address appeared - # to be invalid at the same position in a hashring. - address = ipaddr.IPAddress(ip) - if address.version == 6: - truncated = ':'.join(address.exploded.split(':')[:2]) - subnet = str(ipaddr.IPv6Network(truncated + "::/32")) - else: - truncated = '.'.join(address.exploded.split('.')[:2]) - subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) - else: - group = (int(ipaddr.IPAddress(ip)) % 4) + 1 - subnet = "proxy-group-%d" % group - - logging.debug("Client IP was within area: %s" % subnet) - return subnet - - def mapSubnetToSubring(self, subnet, usingProxy=False): - """Determine the correct subhashring for a client, based upon the - **subnet**. - - :param str subnet: The subnet which contains the client's IP. See - :staticmethod:`getSubnet`. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - """ - # If the client wasn't using a proxy, select the client's subring - # based upon the client's subnet (modulo the total subrings): - if not usingProxy: - mod = self.totalSubrings - # If there is a proxy subring, don't count it for the modulus: - if self.proxySubring: - mod -= 1 - return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 - else: - return self.proxySubring - - def mapClientToHashringPosition(self, interval, subnet): - """Map the client to a position on a (sub)hashring, based upon the - **interval** which the client's request occurred within, as well as - the **subnet** of the client's IP address. - - .. note:: For an explanation of how **subnet** is determined, see - :staticmethod:`getSubnet`. - - :param str interval: The interval which this client's request for - bridges took place within. - :param str subnet: A string representing the subnet containing the - client's IP address. - :rtype: int - :returns: The results of keyed HMAC, which should determine the - client's position in a (sub)hashring of bridges (and thus - determine which bridges they receive). - """ - position = "<%s>%s" % (interval, subnet) - mapping = self._clientToPositionHMAC(position) - return mapping - - def prepopulateRings(self): - """Prepopulate this distributor's hashrings and subhashrings with - bridges. - - The hashring structure for this distributor is influenced by the - ``N_IP_CLUSTERS`` configuration option, as well as the number of - ``PROXY_LIST_FILES``. - - Essentially, :data:`totalSubrings` is set to the specified - ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of - Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), - are stored in :data:`proxies`, and the latter is added as an - additional cluster (such that :data:`totalSubrings` becomes - ``N_IP_CLUSTERS + 1``). The number of subhashrings which this - :class:`Distributor` has active in its hashring is then - :data:`totalSubrings`, where the last cluster is reserved for all - :data:`proxies`. - - As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and - ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number - of subhashrings is five — four for the "clusters", and one for the - :data:`proxies`. Thus, the resulting hashring-subhashring structure - would look like: - - +------------------+---------------------------------------------------+-------------- - | | Directly connecting users | Tor / known | - | | | proxy users | - +------------------+------------+------------+------------+------------+-------------+ - | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | - +==================+============+============+============+============+=============+ - | Subhashrings | | | | | | - | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | - +------------------+------------+------------+------------+------------+-------------+ - | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | - | Subhashrings | | | | | | - | bBy requested +------------+------------+------------+------------+-------------+ - | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | - | | | | | | | - +------------------+------------+------------+------------+------------+-------------+ - - The "filtered subhashrings" are essentially filtered copies of their - respective subhashring, such that they only contain bridges which - support IPv4 or IPv6, respectively. Additionally, the contents of - ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. - - Thus, in this example, we end up with **10 total subhashrings**. - """ - logging.info("Prepopulating %s distributor hashrings..." % self.name) - - for filterFn in [byIPv4, byIPv6]: - for subring in range(1, self.totalSubrings + 1): - filters = self._buildHashringFilters([filterFn,], subring) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - # For consistency with previous implementation of this method, - # only set the "name" for "clusters" which are for this - # distributor's proxies: - if subring == self.proxySubring: - ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - def insert(self, bridge): - """Assign a bridge to this distributor.""" - self.hashring.insert(bridge) - - def _buildHashringFilters(self, previousFilters, subring): - f = bySubring(self.hashring.hmac, subring, self.totalSubrings) - previousFilters.append(f) - return frozenset(previousFilters) - - def getBridges(self, bridgeRequest, interval): - """Return a list of bridges to give to a user. - - :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` - :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` - with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` - attribute set to a string containing the client's IP address. - :param str interval: The time period when we got this request. This - can be any string, so long as it changes with every period. - :rtype: list - :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in - the response. See - :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` - for an example of how this is used. - """ - logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) - - if not len(self.hashring): - logging.warn("Bailing! Hashring has zero bridges!") - return [] - - usingProxy = False - - # First, check if the client's IP is one of the known :data:`proxies`: - if bridgeRequest.client in self.proxies: - # The tag is a tag applied to a proxy IP address when it is added - # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default - # is 'exit_relay'. For other proxies loaded from the - # PROXY_LIST_FILES config option, the default tag is the full - # filename that the IP address originally came from. - usingProxy = True - tag = self.proxies.getTag(bridgeRequest.client) - logging.info("Client was from known proxy (tag: %s): %s" % - (tag, bridgeRequest.client)) - - subnet = self.getSubnet(bridgeRequest.client, usingProxy) - subring = self.mapSubnetToSubring(subnet, usingProxy) - position = self.mapClientToHashringPosition(interval, subnet) - filters = self._buildHashringFilters(bridgeRequest.filters, subring) - - logging.debug("Client request within time interval: %s" % interval) - logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) - logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) - logging.debug("Total bridges: %d" % len(self.hashring)) - logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) - - # Check wheth we have a cached copy of the hashring: - if filters in self.hashring.filterRings.keys(): - logging.debug("Cache hit %s" % filters) - _, ring = self.hashring.filterRings[filters] - # Otherwise, construct a new hashring and populate it: - else: - logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - # Determine the appropriate number of bridges to give to the client: - returnNum = self.bridgesPerResponse(ring) - answer = ring.getBridges(position, returnNum) - - return answer - - class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index ba74b66..3a15145 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -30,6 +30,7 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig +from bridgedb.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors

import bridgedb.Storage @@ -186,16 +187,17 @@ def createBridgeRings(cfg, proxyList, key): """Create the bridge distributors defined by the config file

:type cfg: :class:`Conf` - :param cfg: The current configuration, including any in-memory - settings (i.e. settings whose values were not obtained from the - config file, but were set via a function somewhere) + :param cfg: The current configuration, including any in-memory settings + (i.e. settings whose values were not obtained from the config file, + but were set via a function somewhere) :type proxyList: :class:`~bridgedb.proxy.ProxySet` :param proxyList: The container for the IP addresses of any currently - known open proxies. + known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an HTTPSDistributor or None, - and an EmailBasedDistributor or None. + :returns: A BridgeSplitter hashring, an + :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an + EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -210,7 +212,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") - ipDistributor = Dist.HTTPSDistributor( + ipDistributor = HTTPSDistributor( cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, @@ -328,8 +330,9 @@ def run(options, reactor=reactor): into their hashring assignments. :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently - known open proxies. - :ivar ipDistributor: A :class:`Dist.HTTPSDistributor`. + known open proxies. + :ivar ipDistributor: A + :class:`~bridgedb.https.distributor.HTTPSDistributor`. :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some diff --git a/lib/bridgedb/https/distributor.py b/lib/bridgedb/https/distributor.py new file mode 100644 index 0000000..f8cf09d --- /dev/null +++ b/lib/bridgedb/https/distributor.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_https_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# Matthew Finkel 0x017DD169EA793BE2 sysrqb@torproject.org +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""A Distributor that hands out bridges through a web interface.""" + +import ipaddr +import logging + +import bridgedb.Storage + +from bridgedb import proxy +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import FilteredBridgeSplitter +from bridgedb.crypto import getHMAC +from bridgedb.crypto import getHMACFunc +from bridgedb.distribute import Distributor +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import byFilters +from bridgedb.filters import bySubring + + +class HTTPSDistributor(Distributor): + """A Distributor that hands out bridges based on the IP address of an + incoming request and the current time period. + + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. + :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. + """ + + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ + super(HTTPSDistributor, self).__init__(key) + self.totalSubrings = totalSubrings + self.answerParameters = answerParameters + + if proxies: + logging.info("Added known proxies to HTTPS distributor...") + self.proxies = proxies + self.totalSubrings += 1 + self.proxySubring = self.totalSubrings + else: + logging.warn("No known proxies were added to HTTPS distributor!") + self.proxies = proxy.ProxySet() + self.proxySubring = 0 + + self.ringCacheSize = self.totalSubrings * 3 + + key2 = getHMAC(key, "Assign-Bridges-To-Rings") + key3 = getHMAC(key, "Order-Areas-In-Rings") + key4 = getHMAC(key, "Assign-Areas-To-Rings") + + self._clientToPositionHMAC = getHMACFunc(key3, hex=False) + self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) + self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) + self.name = 'HTTPS' + logging.debug("Added %s to %s distributor." % + (self.hashring.__class__.__name__, self.name)) + + def bridgesPerResponse(self, hashring=None): + return super(HTTPSDistributor, self).bridgesPerResponse(hashring) + + @classmethod + def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): + """Map all clients whose **ip**s are within the same subnet to the same + arbitrary string. + + .. hint:: For non-proxy IP addresses, any two IPv4 addresses within + the same ``/16`` subnet, or any two IPv6 addresses in the same + ``/32`` subnet, will get the same string. + + Subnets for this distributor are grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option, such that + Alice (with the address ``1.2.3.4`` and Bob (with the address + ``1.2.178.234``) are placed within the same cluster, but Carol (with + address ``1.3.11.33``) *might* end up in a different cluster. + + >>> from bridgedb.https.distributor import HTTPSDistributor + >>> HTTPSDistributor.getSubnet('1.2.3.4') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('1.2.211.154') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') + '2001:f::/32' + >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') + '2a00:c98::/32' + + :param str ip: A string representing an IPv4 or IPv6 address. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + :param int proxySubnets: Place Tor/proxy users into this number of + "subnet" groups. This means that no matter how many different Tor + Exits or proxies a client uses, the most they can ever get is + **proxySubnets** different sets of bridge lines (per interval). + This parameter only has any effect when **usingProxy** is ``True``. + :rtype: str + :returns: The appropriately sized CIDR subnet representation of the **ip**. + """ + if not usingProxy: + # We aren't using bridgedb.parse.addr.isIPAddress(ip, + # compressed=False) here because adding the string "False" into + # the map would land any and all clients whose IP address appeared + # to be invalid at the same position in a hashring. + address = ipaddr.IPAddress(ip) + if address.version == 6: + truncated = ':'.join(address.exploded.split(':')[:2]) + subnet = str(ipaddr.IPv6Network(truncated + "::/32")) + else: + truncated = '.'.join(address.exploded.split('.')[:2]) + subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) + else: + group = (int(ipaddr.IPAddress(ip)) % 4) + 1 + subnet = "proxy-group-%d" % group + + logging.debug("Client IP was within area: %s" % subnet) + return subnet + + def mapSubnetToSubring(self, subnet, usingProxy=False): + """Determine the correct subhashring for a client, based upon the + **subnet**. + + :param str subnet: The subnet which contains the client's IP. See + :staticmethod:`getSubnet`. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + """ + # If the client wasn't using a proxy, select the client's subring + # based upon the client's subnet (modulo the total subrings): + if not usingProxy: + mod = self.totalSubrings + # If there is a proxy subring, don't count it for the modulus: + if self.proxySubring: + mod -= 1 + return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 + else: + return self.proxySubring + + def mapClientToHashringPosition(self, interval, subnet): + """Map the client to a position on a (sub)hashring, based upon the + **interval** which the client's request occurred within, as well as + the **subnet** of the client's IP address. + + .. note:: For an explanation of how **subnet** is determined, see + :staticmethod:`getSubnet`. + + :param str interval: The interval which this client's request for + bridges took place within. + :param str subnet: A string representing the subnet containing the + client's IP address. + :rtype: int + :returns: The results of keyed HMAC, which should determine the + client's position in a (sub)hashring of bridges (and thus + determine which bridges they receive). + """ + position = "<%s>%s" % (interval, subnet) + mapping = self._clientToPositionHMAC(position) + return mapping + + def prepopulateRings(self): + """Prepopulate this distributor's hashrings and subhashrings with + bridges. + + The hashring structure for this distributor is influenced by the + ``N_IP_CLUSTERS`` configuration option, as well as the number of + ``PROXY_LIST_FILES``. + + Essentially, :data:`totalSubrings` is set to the specified + ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of + Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), + are stored in :data:`proxies`, and the latter is added as an + additional cluster (such that :data:`totalSubrings` becomes + ``N_IP_CLUSTERS + 1``). The number of subhashrings which this + :class:`Distributor` has active in its hashring is then + :data:`totalSubrings`, where the last cluster is reserved for all + :data:`proxies`. + + As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and + ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number + of subhashrings is five — four for the "clusters", and one for the + :data:`proxies`. Thus, the resulting hashring-subhashring structure + would look like: + + +------------------+---------------------------------------------------+-------------- + | | Directly connecting users | Tor / known | + | | | proxy users | + +------------------+------------+------------+------------+------------+-------------+ + | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | + +==================+============+============+============+============+=============+ + | Subhashrings | | | | | | + | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | + +------------------+------------+------------+------------+------------+-------------+ + | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | + | Subhashrings | | | | | | + | bBy requested +------------+------------+------------+------------+-------------+ + | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | + | | | | | | | + +------------------+------------+------------+------------+------------+-------------+ + + The "filtered subhashrings" are essentially filtered copies of their + respective subhashring, such that they only contain bridges which + support IPv4 or IPv6, respectively. Additionally, the contents of + ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. + + Thus, in this example, we end up with **10 total subhashrings**. + """ + logging.info("Prepopulating %s distributor hashrings..." % self.name) + + for filterFn in [byIPv4, byIPv6]: + for subring in range(1, self.totalSubrings + 1): + filters = self._buildHashringFilters([filterFn,], subring) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + # For consistency with previous implementation of this method, + # only set the "name" for "clusters" which are for this + # distributor's proxies: + if subring == self.proxySubring: + ring.setName('{0} Proxy Ring'.format(self.name)) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + def insert(self, bridge): + """Assign a bridge to this distributor.""" + self.hashring.insert(bridge) + + def _buildHashringFilters(self, previousFilters, subring): + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) + previousFilters.append(f) + return frozenset(previousFilters) + + def getBridges(self, bridgeRequest, interval): + """Return a list of bridges to give to a user. + + :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` + :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` + with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` + attribute set to a string containing the client's IP address. + :param str interval: The time period when we got this request. This + can be any string, so long as it changes with every period. + :rtype: list + :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in + the response. See + :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` + for an example of how this is used. + """ + logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) + + if not len(self.hashring): + logging.warn("Bailing! Hashring has zero bridges!") + return [] + + usingProxy = False + + # First, check if the client's IP is one of the known :data:`proxies`: + if bridgeRequest.client in self.proxies: + # The tag is a tag applied to a proxy IP address when it is added + # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default + # is 'exit_relay'. For other proxies loaded from the + # PROXY_LIST_FILES config option, the default tag is the full + # filename that the IP address originally came from. + usingProxy = True + tag = self.proxies.getTag(bridgeRequest.client) + logging.info("Client was from known proxy (tag: %s): %s" % + (tag, bridgeRequest.client)) + + subnet = self.getSubnet(bridgeRequest.client, usingProxy) + subring = self.mapSubnetToSubring(subnet, usingProxy) + position = self.mapClientToHashringPosition(interval, subnet) + filters = self._buildHashringFilters(bridgeRequest.filters, subring) + + logging.debug("Client request within time interval: %s" % interval) + logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) + logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Total bridges: %d" % len(self.hashring)) + logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) + + # Check wheth we have a cached copy of the hashring: + if filters in self.hashring.filterRings.keys(): + logging.debug("Cache hit %s" % filters) + _, ring = self.hashring.filterRings[filters] + # Otherwise, construct a new hashring and populate it: + else: + logging.debug("Cache miss %s" % filters) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + # Determine the appropriate number of bridges to give to the client: + returnNum = self.bridgesPerResponse(ring) + answer = ring.getBridges(position, returnNum) + + return answer diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py index 0ae52c7..b106a13 100644 --- a/lib/bridgedb/https/request.py +++ b/lib/bridgedb/https/request.py @@ -55,7 +55,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase):

def __init__(self, addClientCountryCode=True): """Process a new bridge request received through the - :class:`~bridgedb.Dist.HTTPSDistributor`. + :class:`~bridgedb.https.distributor.HTTPSDistributor`.

:param bool addClientCountryCode: If ``True``, then calling :meth:`withoutBlockInCountry` will attempt to add the client's own diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index 2a1d510..2106dbf 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -815,7 +815,7 @@ def addWebServer(config, distributor): GIMP_CAPTCHA_DIR GIMP_CAPTCHA_HMAC_KEYFILE GIMP_CAPTCHA_RSA_KEYFILE - :type distributor: :class:`bridgedb.Dist.HTTPSDistributor` + :type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor` :param distributor: A bridge distributor. :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 00726f6..0853c1e 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -26,6 +26,7 @@ from twisted.spread import jelly from bridgedb import Bridges from bridgedb import Dist from bridgedb import filters +from bridgedb.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet

@@ -34,7 +35,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist, httpsDistributor)

class MissingState(Exception): diff --git a/lib/bridgedb/test/https_helpers.py b/lib/bridgedb/test/https_helpers.py index 3d8ec19..e2c94ba 100644 --- a/lib/bridgedb/test/https_helpers.py +++ b/lib/bridgedb/test/https_helpers.py @@ -99,8 +99,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE):

class DummyHTTPSDistributor(object): - """A mocked :class:`bridgedb.Dist.HTTPSDistributor` which is used to test - :class:`bridgedb.https.server.BridgesResource`. + """A mocked :class:`bridgedb.https.distributor.HTTPSDistributor` which is + used to test :class:`bridgedb.https.server.BridgesResource`. """ _bridge_class = util.DummyBridge _bridgesPerResponseMin = 3 diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 3ec634c..c0a9ff0 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -35,15 +35,13 @@ from bridgedb.test.util import randomIPv6 from bridgedb.test.util import randomIPString from bridgedb.test.util import randomIPv4String from bridgedb.test.util import randomIPv6String +from bridgedb.test.util import randomPort +from bridgedb.test.util import randomValidIPv6

from math import log

+warnings.filterwarnings('ignore', '.*tmpnam.*')

-def suppressWarnings(): - warnings.filterwarnings('ignore', '.*tmpnam.*') - -def randomPort(): - return random.randint(1,65535)

def randomPortSpec(): """ @@ -57,8 +55,8 @@ def randomPortSpec():

def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv4()) + ip = randomIPv4() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -66,21 +64,7 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv4String() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6))

if transports: for i in xrange(0,8): @@ -91,8 +75,8 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False,

def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv6()) + ip = randomIPv6() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -100,47 +84,15 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv6() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - address = bracketIPv6(address) - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6))

if transports: for i in xrange(0,8): b.transports.append(bridgedb.Bridges.PluggableTransport(b, random.choice(["obfs", "obfs2", "pt1"]), randomIP(), randomPort())) - return b

-simpleDesc = "router Unnamed %s %s 0 9030\n"\ -"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ -"opt @purpose bridge\n" -orAddress = "or-address %s:%s\n" - - -class RhymesWith255ProxySet: - def __contains__(self, ip): - return ip.endswith(".255")

class EmailBridgeDistTests(unittest.TestCase): def setUp(self): @@ -412,6 +364,4 @@ def testSuite(): return suite

def main(): - suppressWarnings() - unittest.TextTestRunner(verbosity=1).run(testSuite()) diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py deleted file mode 100644 index 46a11b0..0000000 --- a/lib/bridgedb/test/test_Dist.py +++ /dev/null @@ -1,434 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org -# please also see AUTHORS file -# :copyright: (c) 2013-2015 Isis Lovecruft -# (c) 2007-2015, The Tor Project, Inc. -# (c) 2007-2015, all entities within the AUTHORS file -# :license: 3-clause BSD, see included LICENSE for information - -"""Tests for :mod:`bridgedb.Dist`.""" - -from __future__ import print_function - -import hashlib -import ipaddr -import random - -from twisted.trial import unittest - -from bridgedb import Dist -from bridgedb.bridges import Bridge -from bridgedb.bridges import PluggableTransport -from bridgedb.Bridges import BridgeRing -from bridgedb.Bridges import BridgeRingParameters -from bridgedb.filters import byIPv4 -from bridgedb.filters import byIPv6 -from bridgedb.https.request import HTTPSBridgeRequest -from bridgedb.proxy import ProxySet -from bridgedb.test.util import randomHighPort -from bridgedb.test.util import randomValidIPv4String -from bridgedb.test.util import randomValidIPv6 -from bridgedb.test.https_helpers import DummyRequest - - -def _generateFakeBridges(n=500): - bridges = [] - - for i in range(n): - addr = randomValidIPv4String() - nick = 'bridge-%d' % i - port = randomHighPort() - # Real tor currently only supports one extra ORAddress, and it can - # only be IPv6. - addrs = [(randomValidIPv6(), randomHighPort(), 6)] - fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) - - # We only support the ones without PT args, because they're easier to fake. - supported = ["obfs2", "obfs3", "fte"] - transports = [] - for j, method in zip(range(1, len(supported) + 1), supported): - pt = PluggableTransport(fpr, method, addr, port - j, {}) - transports.append(pt) - - bridge = Bridge(nick, addr, port, fpr) - bridge.flags.update("Running Stable") - bridge.transports = transports - bridge.orAddresses = addrs - bridges.append(bridge) - - return bridges - - -BRIDGES = _generateFakeBridges() - - -class HTTPSDistributorTests(unittest.TestCase): - """Tests for :class:`HTTPSDistributor`.""" - - def setUp(self): - self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' - self.bridges = BRIDGES - - def tearDown(self): - """Reset all bridge blocks in between test method runs.""" - for bridge in self.bridges: - bridge._blockedIn = {} - - def coinFlip(self): - return bool(random.getrandbits(1)) - - def randomClientRequest(self): - bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) - bridgeRequest.client = randomValidIPv4String() - bridgeRequest.isValid(True) - bridgeRequest.generateFilters() - return bridgeRequest - - def randomClientRequestForNotBlockedIn(self, cc): - httpRequest = DummyRequest(['']) - httpRequest.args.update({'unblocked': [cc]}) - bridgeRequest = self.randomClientRequest() - bridgeRequest.withoutBlockInCountry(httpRequest) - bridgeRequest.generateFilters() - return bridgeRequest - - def test_HTTPSDistributor_init_with_proxies(self): - """The HTTPSDistributor, when initialised with proxies, should add an - extra hashring for proxy users. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - self.assertIsNotNone(dist.proxies) - self.assertGreater(dist.proxySubring, 0) - self.assertEqual(dist.proxySubring, 4) - self.assertEqual(dist.totalSubrings, 4) - - def test_HTTPSDistributor_bridgesPerResponse_120(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:120]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_100(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_50(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:60]] - self.assertEqual(dist.bridgesPerResponse(), 2) - - def test_HTTPSDistributor_bridgesPerResponse_15(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:15]] - self.assertEqual(dist.bridgesPerResponse(), 1) - - def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): - dist = Dist.HTTPSDistributor(3, self.key) - dist._bridgesPerResponseMax = 5 - [dist.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 5) - - def test_HTTPSDistributor_getSubnet_usingProxy(self): - """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy - group number. - """ - clientRequest = self.randomClientRequest() - expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 - subnet = Dist.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) - self.assertTrue(subnet.startswith('proxy-group-')) - self.assertEqual(int(subnet[-1]), expectedGroup) - - def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): - """HTTPSDistributor.mapSubnetToSubring() when the client was using a - proxy should map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - subnet = 'proxy-group-3' - subring = dist.mapSubnetToSubring(subnet, usingProxy=True) - self.assertEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): - """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using - a proxy, but the distributor does have some known proxies and a - proxySubring, should not map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - # Note that if they were actually from a proxy, their subnet would be - # something like "proxy-group-3". - subnet = '15.1.0.0/16' - subring = dist.mapSubnetToSubring(subnet, usingProxy=False) - self.assertNotEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_prepopulateRings_with_proxies(self): - """An HTTPSDistributor with proxies should prepopulate two extra - subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 8) - - def test_HTTPSDistributor_prepopulateRings_without_proxies(self): - """An HTTPSDistributor without proxies should prepopulate - totalSubrings * 2 subrings. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 6) - - ipv4subrings = [] - ipv6subrings = [] - - for subringName, (filters, subring) in dist.hashring.filterRings.items(): - if 'IPv4' in subringName: - ipv6subrings.append(subring) - if 'IPv6' in subringName: - ipv6subrings.append(subring) - - self.assertEqual(len(ipv4subrings), len(ipv6subrings)) - - def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - bridge.setBlockedIn('cn') - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - b = dist.getBridges(clientRequest1, 1) - self.assertEqual(len(b), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - b = dist.getBridges(clientRequest2, 1) - self.assertEqual(len(b), 3) - - def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - blockedCN = [] - blockedIR = [] - - for bridge in bridges: - if self.coinFlip(): - bridge.setBlockedIn('cn') - blockedCN.append(bridge.fingerprint) - - if self.coinFlip(): - bridge.setBlockedIn('ir') - blockedIR.append(bridge.fingerprint) - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridges = dist.getBridges(clientRequest1, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('cn')) - self.assertNotIn(b.fingerprint, blockedCN) - # The client *should* have gotten some bridges still. - self.assertGreater(len(bridges), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - bridges = dist.getBridges(clientRequest2, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('ir')) - self.assertNotIn(b.fingerprint, blockedIR) - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - # Pretend that China blocks all vanilla bridges: - bridge.setBlockedIn('cn', methodname='vanilla') - # Pretend that China blocks all obfs2: - bridge.setBlockedIn('cn', methodname='obfs2') - # Pretend that China blocks some obfs3: - if self.coinFlip(): - bridge.setBlockedIn('cn', methodname='obfs3') - - [dist.insert(bridge) for bridge in bridges] - - for i in xrange(5): - bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest1.transports.append('obfs2') - bridgeRequest1.generateFilters() - # We shouldn't get any obfs2 bridges, since they're all blocked in - # China: - bridges = dist.getBridges(bridgeRequest1, "faketimestamp") - self.assertEqual(len(bridges), 0) - - bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest2.transports.append('obfs3') - bridgeRequest2.generateFilters() - # We probably will get at least one bridge back! It's pretty - # unlikely to lose a coin flip 500 times in a row. - bridges = dist.getBridges(bridgeRequest2, "faketimestamp") - self.assertGreater(len(bridges), 0) - - bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') - bridgeRequest3.transports.append('obfs3') - bridgeRequest3.generateFilters() - # We should get bridges, since obfs3 isn't blocked in netherlands: - bridges = dist.getBridges(bridgeRequest3, "faketimestamp") - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): - """An HTTPSDistributor should give separate bridges to proxy users.""" - proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) - dist = Dist.HTTPSDistributor(3, self.key, proxies) - [dist.insert(bridge) for bridge in self.bridges] - - for _ in range(10): - bridgeRequest1 = self.randomClientRequest() - bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) - - bridgeRequest2 = self.randomClientRequest() - bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) - - n1 = dist.getBridges(bridgeRequest1, 1) - n2 = dist.getBridges(bridgeRequest2, 1) - - self.assertGreater(len(n1), 0) - self.assertGreater(len(n2), 0) - - for b in n1: - self.assertNotIn(b, n2) - for b in n2: - self.assertNotIn(b, n1) - - def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): - """The same client asking for bridges from the HTTPSDistributor - multiple times in a row should get the same bridges in response each - time. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - responses = {} - for i in range(5): - responses[i] = dist.getBridges(bridgeRequest, 1) - for i in range(4): - self.assertItemsEqual(responses[i], responses[i+1]) - - def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): - param = BridgeRingParameters(needPorts=[(443, 1)]) - dist = Dist.HTTPSDistributor(3, self.key, answerParameters=param) - - bridges = self.bridges[:32] - for b in self.bridges: - b.orPort = 443 - - [dist.insert(bridge) for bridge in bridges] - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for _ in xrange(32): - bridgeRequest = self.randomClientRequest() - answer = dist.getBridges(bridgeRequest, 1) - count = 0 - fingerprints = {} - for bridge in answer: - fingerprints[bridge.identity] = 1 - if bridge.orPort == 443: - count += 1 - self.assertEquals(len(fingerprints), len(answer)) - self.assertGreater(len(fingerprints), 0) - self.assertTrue(count >= 1) - - def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): - """Asking for bridge addresses which are simultaneously IPv4 and IPv6 - (in that order) should return IPv4 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv4() - bridgeRequest.filters.append(byIPv6) - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): - """Asking for bridge addresses which are simultaneously IPv6 and IPv4 - (in that order) should return IPv6 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - bridgeRequest.filters.append(byIPv4) - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6(self): - """A request for IPv6 bridges should return IPv6 bridges.""" - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv4(self): - """A request for IPv4 bridges should return IPv4 bridges.""" - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_distributor.py b/lib/bridgedb/test/test_https_distributor.py new file mode 100644 index 0000000..83f2503 --- /dev/null +++ b/lib/bridgedb/test/test_https_distributor.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Tests for :mod:`bridgedb.https.distributor`.""" + +from __future__ import print_function + +import ipaddr +import logging +import random + +from twisted.trial import unittest + +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import BridgeRingParameters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.https import distributor +from bridgedb.https.request import HTTPSBridgeRequest +from bridgedb.proxy import ProxySet +from bridgedb.test.util import randomValidIPv4String +from bridgedb.test.util import generateFakeBridges +from bridgedb.test.https_helpers import DummyRequest + +logging.disable(50) + + +BRIDGES = generateFakeBridges() + + +class HTTPSDistributorTests(unittest.TestCase): + """Tests for :class:`HTTPSDistributor`.""" + + def setUp(self): + self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' + self.bridges = BRIDGES + + def tearDown(self): + """Reset all bridge blocks in between test method runs.""" + for bridge in self.bridges: + bridge._blockedIn = {} + + def coinFlip(self): + return bool(random.getrandbits(1)) + + def randomClientRequest(self): + bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) + bridgeRequest.client = randomValidIPv4String() + bridgeRequest.isValid(True) + bridgeRequest.generateFilters() + return bridgeRequest + + def randomClientRequestForNotBlockedIn(self, cc): + httpRequest = DummyRequest(['']) + httpRequest.args.update({'unblocked': [cc]}) + bridgeRequest = self.randomClientRequest() + bridgeRequest.withoutBlockInCountry(httpRequest) + bridgeRequest.generateFilters() + return bridgeRequest + + def test_HTTPSDistributor_init_with_proxies(self): + """The HTTPSDistributor, when initialised with proxies, should add an + extra hashring for proxy users. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + self.assertIsNotNone(dist.proxies) + self.assertGreater(dist.proxySubring, 0) + self.assertEqual(dist.proxySubring, 4) + self.assertEqual(dist.totalSubrings, 4) + + def test_HTTPSDistributor_bridgesPerResponse_120(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:120]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_100(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_50(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:60]] + self.assertEqual(dist.bridgesPerResponse(), 2) + + def test_HTTPSDistributor_bridgesPerResponse_15(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:15]] + self.assertEqual(dist.bridgesPerResponse(), 1) + + def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): + dist = distributor.HTTPSDistributor(3, self.key) + dist._bridgesPerResponseMax = 5 + [dist.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 5) + + def test_HTTPSDistributor_getSubnet_usingProxy(self): + """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy + group number. + """ + clientRequest = self.randomClientRequest() + expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 + subnet = distributor.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) + self.assertTrue(subnet.startswith('proxy-group-')) + self.assertEqual(int(subnet[-1]), expectedGroup) + + def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): + """HTTPSDistributor.mapSubnetToSubring() when the client was using a + proxy should map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + subnet = 'proxy-group-3' + subring = dist.mapSubnetToSubring(subnet, usingProxy=True) + self.assertEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): + """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using + a proxy, but the distributor does have some known proxies and a + proxySubring, should not map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + # Note that if they were actually from a proxy, their subnet would be + # something like "proxy-group-3". + subnet = '15.1.0.0/16' + subring = dist.mapSubnetToSubring(subnet, usingProxy=False) + self.assertNotEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_prepopulateRings_with_proxies(self): + """An HTTPSDistributor with proxies should prepopulate two extra + subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 8) + + def test_HTTPSDistributor_prepopulateRings_without_proxies(self): + """An HTTPSDistributor without proxies should prepopulate + totalSubrings * 2 subrings. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 6) + + ipv4subrings = [] + ipv6subrings = [] + + for subringName, (filters, subring) in dist.hashring.filterRings.items(): + if 'IPv4' in subringName: + ipv6subrings.append(subring) + if 'IPv6' in subringName: + ipv6subrings.append(subring) + + self.assertEqual(len(ipv4subrings), len(ipv6subrings)) + + def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + bridge.setBlockedIn('cn') + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + b = dist.getBridges(clientRequest1, 1) + self.assertEqual(len(b), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + b = dist.getBridges(clientRequest2, 1) + self.assertEqual(len(b), 3) + + def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + blockedCN = [] + blockedIR = [] + + for bridge in bridges: + if self.coinFlip(): + bridge.setBlockedIn('cn') + blockedCN.append(bridge.fingerprint) + + if self.coinFlip(): + bridge.setBlockedIn('ir') + blockedIR.append(bridge.fingerprint) + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridges = dist.getBridges(clientRequest1, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('cn')) + self.assertNotIn(b.fingerprint, blockedCN) + # The client *should* have gotten some bridges still. + self.assertGreater(len(bridges), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + bridges = dist.getBridges(clientRequest2, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('ir')) + self.assertNotIn(b.fingerprint, blockedIR) + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + # Pretend that China blocks all vanilla bridges: + bridge.setBlockedIn('cn', methodname='vanilla') + # Pretend that China blocks all obfs2: + bridge.setBlockedIn('cn', methodname='obfs2') + # Pretend that China blocks some obfs3: + if self.coinFlip(): + bridge.setBlockedIn('cn', methodname='obfs3') + + [dist.insert(bridge) for bridge in bridges] + + for i in xrange(5): + bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest1.transports.append('obfs2') + bridgeRequest1.generateFilters() + # We shouldn't get any obfs2 bridges, since they're all blocked in + # China: + bridges = dist.getBridges(bridgeRequest1, "faketimestamp") + self.assertEqual(len(bridges), 0) + + bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest2.transports.append('obfs3') + bridgeRequest2.generateFilters() + # We probably will get at least one bridge back! It's pretty + # unlikely to lose a coin flip 500 times in a row. + bridges = dist.getBridges(bridgeRequest2, "faketimestamp") + self.assertGreater(len(bridges), 0) + + bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') + bridgeRequest3.transports.append('obfs3') + bridgeRequest3.generateFilters() + # We should get bridges, since obfs3 isn't blocked in netherlands: + bridges = dist.getBridges(bridgeRequest3, "faketimestamp") + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): + """An HTTPSDistributor should give separate bridges to proxy users.""" + proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) + dist = distributor.HTTPSDistributor(3, self.key, proxies) + [dist.insert(bridge) for bridge in self.bridges] + + for _ in range(10): + bridgeRequest1 = self.randomClientRequest() + bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) + + bridgeRequest2 = self.randomClientRequest() + bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) + + n1 = dist.getBridges(bridgeRequest1, 1) + n2 = dist.getBridges(bridgeRequest2, 1) + + self.assertGreater(len(n1), 0) + self.assertGreater(len(n2), 0) + + for b in n1: + self.assertNotIn(b, n2) + for b in n2: + self.assertNotIn(b, n1) + + def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): + """The same client asking for bridges from the HTTPSDistributor + multiple times in a row should get the same bridges in response each + time. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + responses = {} + for i in range(5): + responses[i] = dist.getBridges(bridgeRequest, 1) + for i in range(4): + self.assertItemsEqual(responses[i], responses[i+1]) + + def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): + param = BridgeRingParameters(needPorts=[(443, 1)]) + dist = distributor.HTTPSDistributor(3, self.key, answerParameters=param) + + bridges = self.bridges[:32] + for b in self.bridges: + b.orPort = 443 + + [dist.insert(bridge) for bridge in bridges] + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for _ in xrange(32): + bridgeRequest = self.randomClientRequest() + answer = dist.getBridges(bridgeRequest, 1) + count = 0 + fingerprints = {} + for bridge in answer: + fingerprints[bridge.identity] = 1 + if bridge.orPort == 443: + count += 1 + self.assertEquals(len(fingerprints), len(answer)) + self.assertGreater(len(fingerprints), 0) + self.assertTrue(count >= 1) + + def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): + """Asking for bridge addresses which are simultaneously IPv4 and IPv6 + (in that order) should return IPv4 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv4() + bridgeRequest.filters.append(byIPv6) + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): + """Asking for bridge addresses which are simultaneously IPv6 and IPv4 + (in that order) should return IPv6 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + bridgeRequest.filters.append(byIPv4) + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6(self): + """A request for IPv6 bridges should return IPv6 bridges.""" + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv4(self): + """A request for IPv4 bridges should return IPv4 bridges.""" + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_server.py b/lib/bridgedb/test/test_https_server.py index e4f56f0..e782135 100644 --- a/lib/bridgedb/test/test_https_server.py +++ b/lib/bridgedb/test/test_https_server.py @@ -794,7 +794,7 @@ class HTTPSServerServiceTests(unittest.TestCase): """Unittests for :func:`bridgedb.email.server.addWebServer`."""

def setUp(self): - """Create a server.MailServerContext and EmailBasedDistributor.""" + """Create a config and an HTTPSDistributor.""" self.config = _createConfig() self.distributor = DummyHTTPSDistributor()

diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 2d0c020..f164aeb 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -168,6 +168,39 @@ randomValidIPv4String = valid(randomIPv4String) randomValidIPv6String = valid(randomIPv6String) randomValidIPString = valid(randomIPString)

+def generateFakeBridges(n=500): + """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random + data. + """ + from bridgedb.bridges import Bridge + from bridgedb.bridges import PluggableTransport + + bridges = [] + + for i in range(n): + addr = randomValidIPv4String() + nick = 'bridge-%d' % i + port = randomHighPort() + # Real tor currently only supports one extra ORAddress, and it can + # only be IPv6. + addrs = [(randomValidIPv6(), randomHighPort(), 6)] + fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) + + # We only support the ones without PT args, because they're easier to fake. + supported = ["obfs2", "obfs3", "fte"] + transports = [] + for j, method in zip(range(1, len(supported) + 1), supported): + pt = PluggableTransport(fpr, method, addr, port - j, {}) + transports.append(pt) + + bridge = Bridge(nick, addr, port, fpr) + bridge.flags.update("Running Stable") + bridge.transports = transports + bridge.orAddresses = addrs + bridges.append(bridge) + + return bridges +

#: Mixin class for use with :api:`~twisted.trial.unittest.TestCase`. A #: ``TestCaseMixin`` can be used to add additional methods, which should be