[bridgedb/master] Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.
isis at torproject.org
isis at torproject.org
Sat Jul 25 19:26:22 UTC 2015
commit dc91ea443e66223999c5b7a70c4b42025c0ac25c
Author: Isis Lovecruft <isis at torproject.org>
Date: Tue Apr 21 07:32:49 2015 +0000
Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.
* FIXES part of #12506: https://bugs.torproject.org/12506
---
doc/sphinx/source/bridgedb.https.rst | 1 +
doc/sphinx/source/conf.py | 1 +
lib/bridgedb/Bridges.py | 2 +-
lib/bridgedb/Dist.py | 302 -------------------
lib/bridgedb/Main.py | 21 +-
lib/bridgedb/https/distributor.py | 328 ++++++++++++++++++++
lib/bridgedb/https/request.py | 2 +-
lib/bridgedb/https/server.py | 2 +-
lib/bridgedb/persistent.py | 3 +-
lib/bridgedb/test/https_helpers.py | 4 +-
lib/bridgedb/test/legacy_Tests.py | 68 +----
lib/bridgedb/test/test_Dist.py | 434 ---------------------------
lib/bridgedb/test/test_https_distributor.py | 405 +++++++++++++++++++++++++
lib/bridgedb/test/test_https_server.py | 2 +-
lib/bridgedb/test/util.py | 33 ++
15 files changed, 797 insertions(+), 811 deletions(-)
diff --git a/doc/sphinx/source/bridgedb.https.rst b/doc/sphinx/source/bridgedb.https.rst
index 1512065..36fa6c6 100644
--- a/doc/sphinx/source/bridgedb.https.rst
+++ b/doc/sphinx/source/bridgedb.https.rst
@@ -7,5 +7,6 @@ bridgedb.https
:depth: 3
.. automodule:: bridgedb.https.__init__
+.. automodule:: bridgedb.https.distributor
.. automodule:: bridgedb.https.request
.. automodule:: bridgedb.https.server
diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py
index 9b43d0f..5038abb 100644
--- a/doc/sphinx/source/conf.py
+++ b/doc/sphinx/source/conf.py
@@ -43,6 +43,7 @@ import bridgedb.email.templates
import bridgedb.filters
import bridgedb.geo
import bridgedb.https
+import bridgedb.https.distributor
import bridgedb.https.request
import bridgedb.https.server
import bridgedb.Main
diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py
index 2b56884..7a237fe 100644
--- a/lib/bridgedb/Bridges.py
+++ b/lib/bridgedb/Bridges.py
@@ -510,7 +510,7 @@ class FilteredBridgeSplitter(object):
:ivar bridges: DOCDOC
:type distributorName: str
:ivar distributorName: The name of this splitter's distributor. See
- :meth:`bridgedb.Dist.HTTPSDistributor.setDistributorName`.
+ :meth:`~bridgedb.https.distributor.HTTPSDistributor.setDistributorName`.
"""
self.key = key
self.filterRings = {}
diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py
index 9b9e35c..1be705f 100644
--- a/lib/bridgedb/Dist.py
+++ b/lib/bridgedb/Dist.py
@@ -12,14 +12,11 @@
"""This module has functions to decide which bridges to hand out to whom."""
-import ipaddr
import logging
-import re
import time
import bridgedb.Storage
-from bridgedb import proxy
from bridgedb.Bridges import BridgeRing
from bridgedb.Bridges import FilteredBridgeSplitter
from bridgedb.crypto import getHMAC
@@ -47,305 +44,6 @@ class EmailRequestedKey(Exception):
"""Raised when an incoming email requested a copy of our GnuPG keys."""
-class HTTPSDistributor(Distributor):
- """A Distributor that hands out bridges based on the IP address of an
- incoming request and the current time period.
-
- :type proxies: :class:`~bridgedb.proxies.ProxySet`
- :ivar proxies: All known proxies, which we treat differently. See
- :param:`proxies`.
- :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter`
- :ivar hashring: A hashring that assigns bridges to subrings with fixed
- proportions. Used to assign bridges into the subrings of this
- distributor.
- """
-
- def __init__(self, totalSubrings, key, proxies=None, answerParameters=None):
- """Create a Distributor that decides which bridges to distribute based
- upon the client's IP address and the current time.
-
- :param int totalSubrings: The number of subhashrings to group clients
- into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf,
- then the actual number of clusters is one higher than
- ``totalSubrings``, because the set of all known open proxies is
- given its own subhashring.
- :param bytes key: The master HMAC key for this distributor. All added
- bridges are HMACed with this key in order to place them into the
- hashrings.
- :type proxies: :class:`~bridgedb.proxy.ProxySet`
- :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known
- Tor Exit relays and other known proxies. These will constitute
- the extra cluster, and any client requesting bridges from one of
- these **proxies** will be distributed bridges from a separate
- subhashring that is specific to Tor/proxy users.
- :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters`
- :param answerParameters: A mechanism for ensuring that the set of
- bridges that this distributor answers a client with fit certain
- parameters, i.e. that an answer has "at least two obfsproxy
- bridges" or "at least one bridge on port 443", etc.
- """
- super(HTTPSDistributor, self).__init__(key)
- self.totalSubrings = totalSubrings
- self.answerParameters = answerParameters
-
- if proxies:
- logging.info("Added known proxies to HTTPS distributor...")
- self.proxies = proxies
- self.totalSubrings += 1
- self.proxySubring = self.totalSubrings
- else:
- logging.warn("No known proxies were added to HTTPS distributor!")
- self.proxies = proxy.ProxySet()
- self.proxySubring = 0
-
- self.ringCacheSize = self.totalSubrings * 3
-
- key2 = getHMAC(key, "Assign-Bridges-To-Rings")
- key3 = getHMAC(key, "Order-Areas-In-Rings")
- key4 = getHMAC(key, "Assign-Areas-To-Rings")
-
- self._clientToPositionHMAC = getHMACFunc(key3, hex=False)
- self._subnetToSubringHMAC = getHMACFunc(key4, hex=True)
- self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize)
- self.name = 'HTTPS'
- logging.debug("Added %s to %s distributor." %
- (self.hashring.__class__.__name__, self.name))
-
- def bridgesPerResponse(self, hashring=None):
- return super(HTTPSDistributor, self).bridgesPerResponse(hashring)
-
- @classmethod
- def getSubnet(cls, ip, usingProxy=False, proxySubnets=4):
- """Map all clients whose **ip**s are within the same subnet to the same
- arbitrary string.
-
- .. hint:: For non-proxy IP addresses, any two IPv4 addresses within
- the same ``/16`` subnet, or any two IPv6 addresses in the same
- ``/32`` subnet, will get the same string.
-
- Subnets for this distributor are grouped into the number of rings
- specified by the ``N_IP_CLUSTERS`` configuration option, such that
- Alice (with the address ``1.2.3.4`` and Bob (with the address
- ``1.2.178.234``) are placed within the same cluster, but Carol (with
- address ``1.3.11.33``) *might* end up in a different cluster.
-
- >>> from bridgedb.Dist import HTTPSDistributor
- >>> HTTPSDistributor.getSubnet('1.2.3.4')
- '1.2.0.0/16'
- >>> HTTPSDistributor.getSubnet('1.2.211.154')
- '1.2.0.0/16'
- >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808')
- '2001:f::/32'
- >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42')
- '2a00:c98::/32'
-
- :param str ip: A string representing an IPv4 or IPv6 address.
- :param bool usingProxy: Set to ``True`` if the client was using one of
- the known :data:`proxies`.
- :param int proxySubnets: Place Tor/proxy users into this number of
- "subnet" groups. This means that no matter how many different Tor
- Exits or proxies a client uses, the most they can ever get is
- **proxySubnets** different sets of bridge lines (per interval).
- This parameter only has any effect when **usingProxy** is ``True``.
- :rtype: str
- :returns: The appropriately sized CIDR subnet representation of the **ip**.
- """
- if not usingProxy:
- # We aren't using bridgedb.parse.addr.isIPAddress(ip,
- # compressed=False) here because adding the string "False" into
- # the map would land any and all clients whose IP address appeared
- # to be invalid at the same position in a hashring.
- address = ipaddr.IPAddress(ip)
- if address.version == 6:
- truncated = ':'.join(address.exploded.split(':')[:2])
- subnet = str(ipaddr.IPv6Network(truncated + "::/32"))
- else:
- truncated = '.'.join(address.exploded.split('.')[:2])
- subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16'))
- else:
- group = (int(ipaddr.IPAddress(ip)) % 4) + 1
- subnet = "proxy-group-%d" % group
-
- logging.debug("Client IP was within area: %s" % subnet)
- return subnet
-
- def mapSubnetToSubring(self, subnet, usingProxy=False):
- """Determine the correct subhashring for a client, based upon the
- **subnet**.
-
- :param str subnet: The subnet which contains the client's IP. See
- :staticmethod:`getSubnet`.
- :param bool usingProxy: Set to ``True`` if the client was using one of
- the known :data:`proxies`.
- """
- # If the client wasn't using a proxy, select the client's subring
- # based upon the client's subnet (modulo the total subrings):
- if not usingProxy:
- mod = self.totalSubrings
- # If there is a proxy subring, don't count it for the modulus:
- if self.proxySubring:
- mod -= 1
- return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1
- else:
- return self.proxySubring
-
- def mapClientToHashringPosition(self, interval, subnet):
- """Map the client to a position on a (sub)hashring, based upon the
- **interval** which the client's request occurred within, as well as
- the **subnet** of the client's IP address.
-
- .. note:: For an explanation of how **subnet** is determined, see
- :staticmethod:`getSubnet`.
-
- :param str interval: The interval which this client's request for
- bridges took place within.
- :param str subnet: A string representing the subnet containing the
- client's IP address.
- :rtype: int
- :returns: The results of keyed HMAC, which should determine the
- client's position in a (sub)hashring of bridges (and thus
- determine which bridges they receive).
- """
- position = "<%s>%s" % (interval, subnet)
- mapping = self._clientToPositionHMAC(position)
- return mapping
-
- def prepopulateRings(self):
- """Prepopulate this distributor's hashrings and subhashrings with
- bridges.
-
- The hashring structure for this distributor is influenced by the
- ``N_IP_CLUSTERS`` configuration option, as well as the number of
- ``PROXY_LIST_FILES``.
-
- Essentially, :data:`totalSubrings` is set to the specified
- ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of
- Tor Exit relays (downloaded into memory with :script:`get-tor-exits`),
- are stored in :data:`proxies`, and the latter is added as an
- additional cluster (such that :data:`totalSubrings` becomes
- ``N_IP_CLUSTERS + 1``). The number of subhashrings which this
- :class:`Distributor` has active in its hashring is then
- :data:`totalSubrings`, where the last cluster is reserved for all
- :data:`proxies`.
-
- As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and
- ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number
- of subhashrings is five — four for the "clusters", and one for the
- :data:`proxies`. Thus, the resulting hashring-subhashring structure
- would look like:
-
- +------------------+---------------------------------------------------+--------------
- | | Directly connecting users | Tor / known |
- | | | proxy users |
- +------------------+------------+------------+------------+------------+-------------+
- | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 |
- +==================+============+============+============+============+=============+
- | Subhashrings | | | | | |
- | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) |
- +------------------+------------+------------+------------+------------+-------------+
- | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 |
- | Subhashrings | | | | | |
- | bBy requested +------------+------------+------------+------------+-------------+
- | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 |
- | | | | | | |
- +------------------+------------+------------+------------+------------+-------------+
-
- The "filtered subhashrings" are essentially filtered copies of their
- respective subhashring, such that they only contain bridges which
- support IPv4 or IPv6, respectively. Additionally, the contents of
- ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint.
-
- Thus, in this example, we end up with **10 total subhashrings**.
- """
- logging.info("Prepopulating %s distributor hashrings..." % self.name)
-
- for filterFn in [byIPv4, byIPv6]:
- for subring in range(1, self.totalSubrings + 1):
- filters = self._buildHashringFilters([filterFn,], subring)
- key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
- ring = BridgeRing(key1, self.answerParameters)
- # For consistency with previous implementation of this method,
- # only set the "name" for "clusters" which are for this
- # distributor's proxies:
- if subring == self.proxySubring:
- ring.setName('{0} Proxy Ring'.format(self.name))
- self.hashring.addRing(ring, filters, byFilters(filters),
- populate_from=self.hashring.bridges)
-
- def insert(self, bridge):
- """Assign a bridge to this distributor."""
- self.hashring.insert(bridge)
-
- def _buildHashringFilters(self, previousFilters, subring):
- f = bySubring(self.hashring.hmac, subring, self.totalSubrings)
- previousFilters.append(f)
- return frozenset(previousFilters)
-
- def getBridges(self, bridgeRequest, interval):
- """Return a list of bridges to give to a user.
-
- :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest`
- :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase`
- with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client`
- attribute set to a string containing the client's IP address.
- :param str interval: The time period when we got this request. This
- can be any string, so long as it changes with every period.
- :rtype: list
- :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in
- the response. See
- :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer`
- for an example of how this is used.
- """
- logging.info("Attempting to get bridges for %s..." % bridgeRequest.client)
-
- if not len(self.hashring):
- logging.warn("Bailing! Hashring has zero bridges!")
- return []
-
- usingProxy = False
-
- # First, check if the client's IP is one of the known :data:`proxies`:
- if bridgeRequest.client in self.proxies:
- # The tag is a tag applied to a proxy IP address when it is added
- # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default
- # is 'exit_relay'. For other proxies loaded from the
- # PROXY_LIST_FILES config option, the default tag is the full
- # filename that the IP address originally came from.
- usingProxy = True
- tag = self.proxies.getTag(bridgeRequest.client)
- logging.info("Client was from known proxy (tag: %s): %s" %
- (tag, bridgeRequest.client))
-
- subnet = self.getSubnet(bridgeRequest.client, usingProxy)
- subring = self.mapSubnetToSubring(subnet, usingProxy)
- position = self.mapClientToHashringPosition(interval, subnet)
- filters = self._buildHashringFilters(bridgeRequest.filters, subring)
-
- logging.debug("Client request within time interval: %s" % interval)
- logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings))
- logging.debug("Assigned client to subhashring position: %s" % position.encode('hex'))
- logging.debug("Total bridges: %d" % len(self.hashring))
- logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters]))
-
- # Check wheth we have a cached copy of the hashring:
- if filters in self.hashring.filterRings.keys():
- logging.debug("Cache hit %s" % filters)
- _, ring = self.hashring.filterRings[filters]
- # Otherwise, construct a new hashring and populate it:
- else:
- logging.debug("Cache miss %s" % filters)
- key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
- ring = BridgeRing(key1, self.answerParameters)
- self.hashring.addRing(ring, filters, byFilters(filters),
- populate_from=self.hashring.bridges)
-
- # Determine the appropriate number of bridges to give to the client:
- returnNum = self.bridgesPerResponse(ring)
- answer = ring.getBridges(position, returnNum)
-
- return answer
-
-
class EmailBasedDistributor(Distributor):
"""Object that hands out bridges based on the email address of an incoming
request and the current time period.
diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py
index ba74b66..3a15145 100644
--- a/lib/bridgedb/Main.py
+++ b/lib/bridgedb/Main.py
@@ -30,6 +30,7 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch
from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus
from bridgedb.bridges import Bridge
from bridgedb.configure import loadConfig
+from bridgedb.https.distributor import HTTPSDistributor
from bridgedb.parse import descriptors
import bridgedb.Storage
@@ -186,16 +187,17 @@ def createBridgeRings(cfg, proxyList, key):
"""Create the bridge distributors defined by the config file
:type cfg: :class:`Conf`
- :param cfg: The current configuration, including any in-memory
- settings (i.e. settings whose values were not obtained from the
- config file, but were set via a function somewhere)
+ :param cfg: The current configuration, including any in-memory settings
+ (i.e. settings whose values were not obtained from the config file,
+ but were set via a function somewhere)
:type proxyList: :class:`~bridgedb.proxy.ProxySet`
:param proxyList: The container for the IP addresses of any currently
- known open proxies.
+ known open proxies.
:param bytes key: Hashring master key
:rtype: tuple
- :returns: A BridgeSplitter hashring, an HTTPSDistributor or None,
- and an EmailBasedDistributor or None.
+ :returns: A BridgeSplitter hashring, an
+ :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an
+ EmailBasedDistributor or None.
"""
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
@@ -210,7 +212,7 @@ def createBridgeRings(cfg, proxyList, key):
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
logging.debug("Setting up HTTPS Distributor...")
- ipDistributor = Dist.HTTPSDistributor(
+ ipDistributor = HTTPSDistributor(
cfg.N_IP_CLUSTERS,
crypto.getHMAC(key, "HTTPS-IP-Dist-Key"),
proxyList,
@@ -328,8 +330,9 @@ def run(options, reactor=reactor):
into their hashring assignments.
:type proxyList: :class:`~bridgedb.proxy.ProxySet`
:ivar proxyList: The container for the IP addresses of any currently
- known open proxies.
- :ivar ipDistributor: A :class:`Dist.HTTPSDistributor`.
+ known open proxies.
+ :ivar ipDistributor: A
+ :class:`~bridgedb.https.distributor.HTTPSDistributor`.
:ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`.
:ivar dict tasks: A dictionary of ``{name: task}``, where name is a
string to associate with the ``task``, and ``task`` is some
diff --git a/lib/bridgedb/https/distributor.py b/lib/bridgedb/https/distributor.py
new file mode 100644
index 0000000..f8cf09d
--- /dev/null
+++ b/lib/bridgedb/https/distributor.py
@@ -0,0 +1,328 @@
+# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_https_distributor -*-
+#
+# This file is part of BridgeDB, a Tor bridge distribution system.
+#
+# :authors: Nick Mathewson
+# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis at torproject.org>
+# Matthew Finkel 0x017DD169EA793BE2 <sysrqb at torproject.org>
+# :copyright: (c) 2013-2015, Isis Lovecruft
+# (c) 2013-2015, Matthew Finkel
+# (c) 2007-2015, The Tor Project, Inc.
+# :license: see LICENSE for licensing information
+
+"""A Distributor that hands out bridges through a web interface."""
+
+import ipaddr
+import logging
+
+import bridgedb.Storage
+
+from bridgedb import proxy
+from bridgedb.Bridges import BridgeRing
+from bridgedb.Bridges import FilteredBridgeSplitter
+from bridgedb.crypto import getHMAC
+from bridgedb.crypto import getHMACFunc
+from bridgedb.distribute import Distributor
+from bridgedb.filters import byIPv4
+from bridgedb.filters import byIPv6
+from bridgedb.filters import byFilters
+from bridgedb.filters import bySubring
+
+
+class HTTPSDistributor(Distributor):
+ """A Distributor that hands out bridges based on the IP address of an
+ incoming request and the current time period.
+
+ :type proxies: :class:`~bridgedb.proxies.ProxySet`
+ :ivar proxies: All known proxies, which we treat differently. See
+ :param:`proxies`.
+ :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter`
+ :ivar hashring: A hashring that assigns bridges to subrings with fixed
+ proportions. Used to assign bridges into the subrings of this
+ distributor.
+ """
+
+ def __init__(self, totalSubrings, key, proxies=None, answerParameters=None):
+ """Create a Distributor that decides which bridges to distribute based
+ upon the client's IP address and the current time.
+
+ :param int totalSubrings: The number of subhashrings to group clients
+ into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf,
+ then the actual number of clusters is one higher than
+ ``totalSubrings``, because the set of all known open proxies is
+ given its own subhashring.
+ :param bytes key: The master HMAC key for this distributor. All added
+ bridges are HMACed with this key in order to place them into the
+ hashrings.
+ :type proxies: :class:`~bridgedb.proxy.ProxySet`
+ :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known
+ Tor Exit relays and other known proxies. These will constitute
+ the extra cluster, and any client requesting bridges from one of
+ these **proxies** will be distributed bridges from a separate
+ subhashring that is specific to Tor/proxy users.
+ :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters`
+ :param answerParameters: A mechanism for ensuring that the set of
+ bridges that this distributor answers a client with fit certain
+ parameters, i.e. that an answer has "at least two obfsproxy
+ bridges" or "at least one bridge on port 443", etc.
+ """
+ super(HTTPSDistributor, self).__init__(key)
+ self.totalSubrings = totalSubrings
+ self.answerParameters = answerParameters
+
+ if proxies:
+ logging.info("Added known proxies to HTTPS distributor...")
+ self.proxies = proxies
+ self.totalSubrings += 1
+ self.proxySubring = self.totalSubrings
+ else:
+ logging.warn("No known proxies were added to HTTPS distributor!")
+ self.proxies = proxy.ProxySet()
+ self.proxySubring = 0
+
+ self.ringCacheSize = self.totalSubrings * 3
+
+ key2 = getHMAC(key, "Assign-Bridges-To-Rings")
+ key3 = getHMAC(key, "Order-Areas-In-Rings")
+ key4 = getHMAC(key, "Assign-Areas-To-Rings")
+
+ self._clientToPositionHMAC = getHMACFunc(key3, hex=False)
+ self._subnetToSubringHMAC = getHMACFunc(key4, hex=True)
+ self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize)
+ self.name = 'HTTPS'
+ logging.debug("Added %s to %s distributor." %
+ (self.hashring.__class__.__name__, self.name))
+
+ def bridgesPerResponse(self, hashring=None):
+ return super(HTTPSDistributor, self).bridgesPerResponse(hashring)
+
+ @classmethod
+ def getSubnet(cls, ip, usingProxy=False, proxySubnets=4):
+ """Map all clients whose **ip**s are within the same subnet to the same
+ arbitrary string.
+
+ .. hint:: For non-proxy IP addresses, any two IPv4 addresses within
+ the same ``/16`` subnet, or any two IPv6 addresses in the same
+ ``/32`` subnet, will get the same string.
+
+ Subnets for this distributor are grouped into the number of rings
+ specified by the ``N_IP_CLUSTERS`` configuration option, such that
+ Alice (with the address ``1.2.3.4`` and Bob (with the address
+ ``1.2.178.234``) are placed within the same cluster, but Carol (with
+ address ``1.3.11.33``) *might* end up in a different cluster.
+
+ >>> from bridgedb.https.distributor import HTTPSDistributor
+ >>> HTTPSDistributor.getSubnet('1.2.3.4')
+ '1.2.0.0/16'
+ >>> HTTPSDistributor.getSubnet('1.2.211.154')
+ '1.2.0.0/16'
+ >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808')
+ '2001:f::/32'
+ >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42')
+ '2a00:c98::/32'
+
+ :param str ip: A string representing an IPv4 or IPv6 address.
+ :param bool usingProxy: Set to ``True`` if the client was using one of
+ the known :data:`proxies`.
+ :param int proxySubnets: Place Tor/proxy users into this number of
+ "subnet" groups. This means that no matter how many different Tor
+ Exits or proxies a client uses, the most they can ever get is
+ **proxySubnets** different sets of bridge lines (per interval).
+ This parameter only has any effect when **usingProxy** is ``True``.
+ :rtype: str
+ :returns: The appropriately sized CIDR subnet representation of the **ip**.
+ """
+ if not usingProxy:
+ # We aren't using bridgedb.parse.addr.isIPAddress(ip,
+ # compressed=False) here because adding the string "False" into
+ # the map would land any and all clients whose IP address appeared
+ # to be invalid at the same position in a hashring.
+ address = ipaddr.IPAddress(ip)
+ if address.version == 6:
+ truncated = ':'.join(address.exploded.split(':')[:2])
+ subnet = str(ipaddr.IPv6Network(truncated + "::/32"))
+ else:
+ truncated = '.'.join(address.exploded.split('.')[:2])
+ subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16'))
+ else:
+ group = (int(ipaddr.IPAddress(ip)) % 4) + 1
+ subnet = "proxy-group-%d" % group
+
+ logging.debug("Client IP was within area: %s" % subnet)
+ return subnet
+
+ def mapSubnetToSubring(self, subnet, usingProxy=False):
+ """Determine the correct subhashring for a client, based upon the
+ **subnet**.
+
+ :param str subnet: The subnet which contains the client's IP. See
+ :staticmethod:`getSubnet`.
+ :param bool usingProxy: Set to ``True`` if the client was using one of
+ the known :data:`proxies`.
+ """
+ # If the client wasn't using a proxy, select the client's subring
+ # based upon the client's subnet (modulo the total subrings):
+ if not usingProxy:
+ mod = self.totalSubrings
+ # If there is a proxy subring, don't count it for the modulus:
+ if self.proxySubring:
+ mod -= 1
+ return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1
+ else:
+ return self.proxySubring
+
+ def mapClientToHashringPosition(self, interval, subnet):
+ """Map the client to a position on a (sub)hashring, based upon the
+ **interval** which the client's request occurred within, as well as
+ the **subnet** of the client's IP address.
+
+ .. note:: For an explanation of how **subnet** is determined, see
+ :staticmethod:`getSubnet`.
+
+ :param str interval: The interval which this client's request for
+ bridges took place within.
+ :param str subnet: A string representing the subnet containing the
+ client's IP address.
+ :rtype: int
+ :returns: The results of keyed HMAC, which should determine the
+ client's position in a (sub)hashring of bridges (and thus
+ determine which bridges they receive).
+ """
+ position = "<%s>%s" % (interval, subnet)
+ mapping = self._clientToPositionHMAC(position)
+ return mapping
+
+ def prepopulateRings(self):
+ """Prepopulate this distributor's hashrings and subhashrings with
+ bridges.
+
+ The hashring structure for this distributor is influenced by the
+ ``N_IP_CLUSTERS`` configuration option, as well as the number of
+ ``PROXY_LIST_FILES``.
+
+ Essentially, :data:`totalSubrings` is set to the specified
+ ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of
+ Tor Exit relays (downloaded into memory with :script:`get-tor-exits`),
+ are stored in :data:`proxies`, and the latter is added as an
+ additional cluster (such that :data:`totalSubrings` becomes
+ ``N_IP_CLUSTERS + 1``). The number of subhashrings which this
+ :class:`Distributor` has active in its hashring is then
+ :data:`totalSubrings`, where the last cluster is reserved for all
+ :data:`proxies`.
+
+ As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and
+ ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number
+ of subhashrings is five — four for the "clusters", and one for the
+ :data:`proxies`. Thus, the resulting hashring-subhashring structure
+ would look like:
+
+ +------------------+---------------------------------------------------+--------------
+ | | Directly connecting users | Tor / known |
+ | | | proxy users |
+ +------------------+------------+------------+------------+------------+-------------+
+ | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 |
+ +==================+============+============+============+============+=============+
+ | Subhashrings | | | | | |
+ | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) |
+ +------------------+------------+------------+------------+------------+-------------+
+ | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 |
+ | Subhashrings | | | | | |
+ | bBy requested +------------+------------+------------+------------+-------------+
+ | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 |
+ | | | | | | |
+ +------------------+------------+------------+------------+------------+-------------+
+
+ The "filtered subhashrings" are essentially filtered copies of their
+ respective subhashring, such that they only contain bridges which
+ support IPv4 or IPv6, respectively. Additionally, the contents of
+ ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint.
+
+ Thus, in this example, we end up with **10 total subhashrings**.
+ """
+ logging.info("Prepopulating %s distributor hashrings..." % self.name)
+
+ for filterFn in [byIPv4, byIPv6]:
+ for subring in range(1, self.totalSubrings + 1):
+ filters = self._buildHashringFilters([filterFn,], subring)
+ key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
+ ring = BridgeRing(key1, self.answerParameters)
+ # For consistency with previous implementation of this method,
+ # only set the "name" for "clusters" which are for this
+ # distributor's proxies:
+ if subring == self.proxySubring:
+ ring.setName('{0} Proxy Ring'.format(self.name))
+ self.hashring.addRing(ring, filters, byFilters(filters),
+ populate_from=self.hashring.bridges)
+
+ def insert(self, bridge):
+ """Assign a bridge to this distributor."""
+ self.hashring.insert(bridge)
+
+ def _buildHashringFilters(self, previousFilters, subring):
+ f = bySubring(self.hashring.hmac, subring, self.totalSubrings)
+ previousFilters.append(f)
+ return frozenset(previousFilters)
+
+ def getBridges(self, bridgeRequest, interval):
+ """Return a list of bridges to give to a user.
+
+ :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest`
+ :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase`
+ with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client`
+ attribute set to a string containing the client's IP address.
+ :param str interval: The time period when we got this request. This
+ can be any string, so long as it changes with every period.
+ :rtype: list
+ :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in
+ the response. See
+ :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer`
+ for an example of how this is used.
+ """
+ logging.info("Attempting to get bridges for %s..." % bridgeRequest.client)
+
+ if not len(self.hashring):
+ logging.warn("Bailing! Hashring has zero bridges!")
+ return []
+
+ usingProxy = False
+
+ # First, check if the client's IP is one of the known :data:`proxies`:
+ if bridgeRequest.client in self.proxies:
+ # The tag is a tag applied to a proxy IP address when it is added
+ # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default
+ # is 'exit_relay'. For other proxies loaded from the
+ # PROXY_LIST_FILES config option, the default tag is the full
+ # filename that the IP address originally came from.
+ usingProxy = True
+ tag = self.proxies.getTag(bridgeRequest.client)
+ logging.info("Client was from known proxy (tag: %s): %s" %
+ (tag, bridgeRequest.client))
+
+ subnet = self.getSubnet(bridgeRequest.client, usingProxy)
+ subring = self.mapSubnetToSubring(subnet, usingProxy)
+ position = self.mapClientToHashringPosition(interval, subnet)
+ filters = self._buildHashringFilters(bridgeRequest.filters, subring)
+
+ logging.debug("Client request within time interval: %s" % interval)
+ logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings))
+ logging.debug("Assigned client to subhashring position: %s" % position.encode('hex'))
+ logging.debug("Total bridges: %d" % len(self.hashring))
+ logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters]))
+
+ # Check wheth we have a cached copy of the hashring:
+ if filters in self.hashring.filterRings.keys():
+ logging.debug("Cache hit %s" % filters)
+ _, ring = self.hashring.filterRings[filters]
+ # Otherwise, construct a new hashring and populate it:
+ else:
+ logging.debug("Cache miss %s" % filters)
+ key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
+ ring = BridgeRing(key1, self.answerParameters)
+ self.hashring.addRing(ring, filters, byFilters(filters),
+ populate_from=self.hashring.bridges)
+
+ # Determine the appropriate number of bridges to give to the client:
+ returnNum = self.bridgesPerResponse(ring)
+ answer = ring.getBridges(position, returnNum)
+
+ return answer
diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py
index 0ae52c7..b106a13 100644
--- a/lib/bridgedb/https/request.py
+++ b/lib/bridgedb/https/request.py
@@ -55,7 +55,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase):
def __init__(self, addClientCountryCode=True):
"""Process a new bridge request received through the
- :class:`~bridgedb.Dist.HTTPSDistributor`.
+ :class:`~bridgedb.https.distributor.HTTPSDistributor`.
:param bool addClientCountryCode: If ``True``, then calling
:meth:`withoutBlockInCountry` will attempt to add the client's own
diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py
index 2a1d510..2106dbf 100644
--- a/lib/bridgedb/https/server.py
+++ b/lib/bridgedb/https/server.py
@@ -815,7 +815,7 @@ def addWebServer(config, distributor):
GIMP_CAPTCHA_DIR
GIMP_CAPTCHA_HMAC_KEYFILE
GIMP_CAPTCHA_RSA_KEYFILE
- :type distributor: :class:`bridgedb.Dist.HTTPSDistributor`
+ :type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor`
:param distributor: A bridge distributor.
:raises SystemExit: if the servers cannot be started.
:rtype: :api:`twisted.web.server.Site`
diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py
index 00726f6..0853c1e 100644
--- a/lib/bridgedb/persistent.py
+++ b/lib/bridgedb/persistent.py
@@ -26,6 +26,7 @@ from twisted.spread import jelly
from bridgedb import Bridges
from bridgedb import Dist
from bridgedb import filters
+from bridgedb.https import distributor as httpsDistributor
from bridgedb.configure import Conf
#from bridgedb.proxy import ProxySet
@@ -34,7 +35,7 @@ _state = None
#: Types and classes which are allowed to be jellied:
_security = jelly.SecurityOptions()
#_security.allowInstancesOf(ProxySet)
-_security.allowModules(filters, Bridges, Dist)
+_security.allowModules(filters, Bridges, Dist, httpsDistributor)
class MissingState(Exception):
diff --git a/lib/bridgedb/test/https_helpers.py b/lib/bridgedb/test/https_helpers.py
index 3d8ec19..e2c94ba 100644
--- a/lib/bridgedb/test/https_helpers.py
+++ b/lib/bridgedb/test/https_helpers.py
@@ -99,8 +99,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE):
class DummyHTTPSDistributor(object):
- """A mocked :class:`bridgedb.Dist.HTTPSDistributor` which is used to test
- :class:`bridgedb.https.server.BridgesResource`.
+ """A mocked :class:`bridgedb.https.distributor.HTTPSDistributor` which is
+ used to test :class:`bridgedb.https.server.BridgesResource`.
"""
_bridge_class = util.DummyBridge
_bridgesPerResponseMin = 3
diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py
index 3ec634c..c0a9ff0 100644
--- a/lib/bridgedb/test/legacy_Tests.py
+++ b/lib/bridgedb/test/legacy_Tests.py
@@ -35,15 +35,13 @@ from bridgedb.test.util import randomIPv6
from bridgedb.test.util import randomIPString
from bridgedb.test.util import randomIPv4String
from bridgedb.test.util import randomIPv6String
+from bridgedb.test.util import randomPort
+from bridgedb.test.util import randomValidIPv6
from math import log
+warnings.filterwarnings('ignore', '.*tmpnam.*')
-def suppressWarnings():
- warnings.filterwarnings('ignore', '.*tmpnam.*')
-
-def randomPort():
- return random.randint(1,65535)
def randomPortSpec():
"""
@@ -57,8 +55,8 @@ def randomPortSpec():
def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False,
transports=False):
- nn = "bridge-%s"%random.randrange(0,1000000)
- ip = ipaddr.IPAddress(randomIPv4())
+ ip = randomIPv4()
+ nn = "bridge-%s" % int(ip)
fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)])
b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp)
b.setStatus(running, stable)
@@ -66,21 +64,7 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False,
oraddrs = []
if or_addresses:
for i in xrange(8):
- # Only add or_addresses if they are valid. Otherwise, the test
- # will randomly fail if an invalid address is chosen:
- address = randomIPv4String()
- portlist = addr.PortList(randomPortSpec())
- if addr.isValidIP(address):
- oraddrs.append((address, portlist,))
-
- for address, portlist in oraddrs:
- networkstatus.parseALine("{0}:{1}".format(address, portlist))
- try:
- portlist.add(b.or_addresses[address])
- except KeyError:
- pass
- finally:
- b.or_addresses[address] = portlist
+ b.orAddresses.append((randomValidIPv6(), randomPort(), 6))
if transports:
for i in xrange(0,8):
@@ -91,8 +75,8 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False,
def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False,
transports=False):
- nn = "bridge-%s"%random.randrange(0,1000000)
- ip = ipaddr.IPAddress(randomIPv6())
+ ip = randomIPv6()
+ nn = "bridge-%s" % int(ip)
fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)])
b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp)
b.setStatus(running, stable)
@@ -100,47 +84,15 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False,
oraddrs = []
if or_addresses:
for i in xrange(8):
- # Only add or_addresses if they are valid. Otherwise, the test
- # will randomly fail if an invalid address is chosen:
- address = randomIPv6()
- portlist = addr.PortList(randomPortSpec())
- if addr.isValidIP(address):
- address = bracketIPv6(address)
- oraddrs.append((address, portlist,))
-
- for address, portlist in oraddrs:
- networkstatus.parseALine("{0}:{1}".format(address, portlist))
- try:
- portlist.add(b.or_addresses[address])
- except KeyError:
- pass
- finally:
- b.or_addresses[address] = portlist
-
- try:
- portlist.add(b.or_addresses[address])
- except KeyError:
- pass
- finally:
- b.or_addresses[address] = portlist
+ b.orAddresses.append((randomValidIPv6(), randomPort(), 6))
if transports:
for i in xrange(0,8):
b.transports.append(bridgedb.Bridges.PluggableTransport(b,
random.choice(["obfs", "obfs2", "pt1"]),
randomIP(), randomPort()))
-
return b
-simpleDesc = "router Unnamed %s %s 0 9030\n"\
-"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\
-"opt @purpose bridge\n"
-orAddress = "or-address %s:%s\n"
-
-
-class RhymesWith255ProxySet:
- def __contains__(self, ip):
- return ip.endswith(".255")
class EmailBridgeDistTests(unittest.TestCase):
def setUp(self):
@@ -412,6 +364,4 @@ def testSuite():
return suite
def main():
- suppressWarnings()
-
unittest.TextTestRunner(verbosity=1).run(testSuite())
diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py
deleted file mode 100644
index 46a11b0..0000000
--- a/lib/bridgedb/test/test_Dist.py
+++ /dev/null
@@ -1,434 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of BridgeDB, a Tor bridge distribution system.
-#
-# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis at torproject.org>
-# please also see AUTHORS file
-# :copyright: (c) 2013-2015 Isis Lovecruft
-# (c) 2007-2015, The Tor Project, Inc.
-# (c) 2007-2015, all entities within the AUTHORS file
-# :license: 3-clause BSD, see included LICENSE for information
-
-"""Tests for :mod:`bridgedb.Dist`."""
-
-from __future__ import print_function
-
-import hashlib
-import ipaddr
-import random
-
-from twisted.trial import unittest
-
-from bridgedb import Dist
-from bridgedb.bridges import Bridge
-from bridgedb.bridges import PluggableTransport
-from bridgedb.Bridges import BridgeRing
-from bridgedb.Bridges import BridgeRingParameters
-from bridgedb.filters import byIPv4
-from bridgedb.filters import byIPv6
-from bridgedb.https.request import HTTPSBridgeRequest
-from bridgedb.proxy import ProxySet
-from bridgedb.test.util import randomHighPort
-from bridgedb.test.util import randomValidIPv4String
-from bridgedb.test.util import randomValidIPv6
-from bridgedb.test.https_helpers import DummyRequest
-
-
-def _generateFakeBridges(n=500):
- bridges = []
-
- for i in range(n):
- addr = randomValidIPv4String()
- nick = 'bridge-%d' % i
- port = randomHighPort()
- # Real tor currently only supports one extra ORAddress, and it can
- # only be IPv6.
- addrs = [(randomValidIPv6(), randomHighPort(), 6)]
- fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40))
-
- # We only support the ones without PT args, because they're easier to fake.
- supported = ["obfs2", "obfs3", "fte"]
- transports = []
- for j, method in zip(range(1, len(supported) + 1), supported):
- pt = PluggableTransport(fpr, method, addr, port - j, {})
- transports.append(pt)
-
- bridge = Bridge(nick, addr, port, fpr)
- bridge.flags.update("Running Stable")
- bridge.transports = transports
- bridge.orAddresses = addrs
- bridges.append(bridge)
-
- return bridges
-
-
-BRIDGES = _generateFakeBridges()
-
-
-class HTTPSDistributorTests(unittest.TestCase):
- """Tests for :class:`HTTPSDistributor`."""
-
- def setUp(self):
- self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X'
- self.bridges = BRIDGES
-
- def tearDown(self):
- """Reset all bridge blocks in between test method runs."""
- for bridge in self.bridges:
- bridge._blockedIn = {}
-
- def coinFlip(self):
- return bool(random.getrandbits(1))
-
- def randomClientRequest(self):
- bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False)
- bridgeRequest.client = randomValidIPv4String()
- bridgeRequest.isValid(True)
- bridgeRequest.generateFilters()
- return bridgeRequest
-
- def randomClientRequestForNotBlockedIn(self, cc):
- httpRequest = DummyRequest([''])
- httpRequest.args.update({'unblocked': [cc]})
- bridgeRequest = self.randomClientRequest()
- bridgeRequest.withoutBlockInCountry(httpRequest)
- bridgeRequest.generateFilters()
- return bridgeRequest
-
- def test_HTTPSDistributor_init_with_proxies(self):
- """The HTTPSDistributor, when initialised with proxies, should add an
- extra hashring for proxy users.
- """
- dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
- self.assertIsNotNone(dist.proxies)
- self.assertGreater(dist.proxySubring, 0)
- self.assertEqual(dist.proxySubring, 4)
- self.assertEqual(dist.totalSubrings, 4)
-
- def test_HTTPSDistributor_bridgesPerResponse_120(self):
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:120]]
- self.assertEqual(dist.bridgesPerResponse(), 3)
-
- def test_HTTPSDistributor_bridgesPerResponse_100(self):
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.hashring.insert(bridge) for bridge in self.bridges[:100]]
- self.assertEqual(dist.bridgesPerResponse(), 3)
-
- def test_HTTPSDistributor_bridgesPerResponse_50(self):
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:60]]
- self.assertEqual(dist.bridgesPerResponse(), 2)
-
- def test_HTTPSDistributor_bridgesPerResponse_15(self):
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:15]]
- self.assertEqual(dist.bridgesPerResponse(), 1)
-
- def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self):
- dist = Dist.HTTPSDistributor(3, self.key)
- dist._bridgesPerResponseMax = 5
- [dist.insert(bridge) for bridge in self.bridges[:100]]
- self.assertEqual(dist.bridgesPerResponse(), 5)
-
- def test_HTTPSDistributor_getSubnet_usingProxy(self):
- """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy
- group number.
- """
- clientRequest = self.randomClientRequest()
- expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1
- subnet = Dist.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True)
- self.assertTrue(subnet.startswith('proxy-group-'))
- self.assertEqual(int(subnet[-1]), expectedGroup)
-
- def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self):
- """HTTPSDistributor.mapSubnetToSubring() when the client was using a
- proxy should map the client to the proxy subhashring.
- """
- dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
- subnet = 'proxy-group-3'
- subring = dist.mapSubnetToSubring(subnet, usingProxy=True)
- self.assertEqual(subring, dist.proxySubring)
-
- def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self):
- """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using
- a proxy, but the distributor does have some known proxies and a
- proxySubring, should not map the client to the proxy subhashring.
- """
- dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
- # Note that if they were actually from a proxy, their subnet would be
- # something like "proxy-group-3".
- subnet = '15.1.0.0/16'
- subring = dist.mapSubnetToSubring(subnet, usingProxy=False)
- self.assertNotEqual(subring, dist.proxySubring)
-
- def test_HTTPSDistributor_prepopulateRings_with_proxies(self):
- """An HTTPSDistributor with proxies should prepopulate two extra
- subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6).
- """
- dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
- [dist.insert(bridge) for bridge in self.bridges]
- dist.prepopulateRings()
- self.assertEqual(len(dist.hashring.filterRings), 8)
-
- def test_HTTPSDistributor_prepopulateRings_without_proxies(self):
- """An HTTPSDistributor without proxies should prepopulate
- totalSubrings * 2 subrings.
- """
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges]
- dist.prepopulateRings()
- self.assertEqual(len(dist.hashring.filterRings), 6)
-
- ipv4subrings = []
- ipv6subrings = []
-
- for subringName, (filters, subring) in dist.hashring.filterRings.items():
- if 'IPv4' in subringName:
- ipv6subrings.append(subring)
- if 'IPv6' in subringName:
- ipv6subrings.append(subring)
-
- self.assertEqual(len(ipv4subrings), len(ipv6subrings))
-
- def test_HTTPSDistributor_getBridges_with_blocked_bridges(self):
- dist = Dist.HTTPSDistributor(1, self.key)
- bridges = self.bridges[:]
-
- for bridge in bridges:
- bridge.setBlockedIn('cn')
-
- [dist.insert(bridge) for bridge in bridges]
-
- for _ in range(5):
- clientRequest1 = self.randomClientRequestForNotBlockedIn('cn')
- b = dist.getBridges(clientRequest1, 1)
- self.assertEqual(len(b), 0)
-
- clientRequest2 = self.randomClientRequestForNotBlockedIn('ir')
- b = dist.getBridges(clientRequest2, 1)
- self.assertEqual(len(b), 3)
-
- def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self):
- dist = Dist.HTTPSDistributor(1, self.key)
- bridges = self.bridges[:]
-
- blockedCN = []
- blockedIR = []
-
- for bridge in bridges:
- if self.coinFlip():
- bridge.setBlockedIn('cn')
- blockedCN.append(bridge.fingerprint)
-
- if self.coinFlip():
- bridge.setBlockedIn('ir')
- blockedIR.append(bridge.fingerprint)
-
- [dist.insert(bridge) for bridge in bridges]
-
- for _ in range(5):
- clientRequest1 = self.randomClientRequestForNotBlockedIn('cn')
- bridges = dist.getBridges(clientRequest1, 1)
- for b in bridges:
- self.assertFalse(b.isBlockedIn('cn'))
- self.assertNotIn(b.fingerprint, blockedCN)
- # The client *should* have gotten some bridges still.
- self.assertGreater(len(bridges), 0)
-
- clientRequest2 = self.randomClientRequestForNotBlockedIn('ir')
- bridges = dist.getBridges(clientRequest2, 1)
- for b in bridges:
- self.assertFalse(b.isBlockedIn('ir'))
- self.assertNotIn(b.fingerprint, blockedIR)
- self.assertGreater(len(bridges), 0)
-
- def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self):
- dist = Dist.HTTPSDistributor(1, self.key)
- bridges = self.bridges[:]
-
- for bridge in bridges:
- # Pretend that China blocks all vanilla bridges:
- bridge.setBlockedIn('cn', methodname='vanilla')
- # Pretend that China blocks all obfs2:
- bridge.setBlockedIn('cn', methodname='obfs2')
- # Pretend that China blocks some obfs3:
- if self.coinFlip():
- bridge.setBlockedIn('cn', methodname='obfs3')
-
- [dist.insert(bridge) for bridge in bridges]
-
- for i in xrange(5):
- bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn')
- bridgeRequest1.transports.append('obfs2')
- bridgeRequest1.generateFilters()
- # We shouldn't get any obfs2 bridges, since they're all blocked in
- # China:
- bridges = dist.getBridges(bridgeRequest1, "faketimestamp")
- self.assertEqual(len(bridges), 0)
-
- bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn')
- bridgeRequest2.transports.append('obfs3')
- bridgeRequest2.generateFilters()
- # We probably will get at least one bridge back! It's pretty
- # unlikely to lose a coin flip 500 times in a row.
- bridges = dist.getBridges(bridgeRequest2, "faketimestamp")
- self.assertGreater(len(bridges), 0)
-
- bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl')
- bridgeRequest3.transports.append('obfs3')
- bridgeRequest3.generateFilters()
- # We should get bridges, since obfs3 isn't blocked in netherlands:
- bridges = dist.getBridges(bridgeRequest3, "faketimestamp")
- self.assertGreater(len(bridges), 0)
-
- def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self):
- """An HTTPSDistributor should give separate bridges to proxy users."""
- proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)])
- dist = Dist.HTTPSDistributor(3, self.key, proxies)
- [dist.insert(bridge) for bridge in self.bridges]
-
- for _ in range(10):
- bridgeRequest1 = self.randomClientRequest()
- bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))])
-
- bridgeRequest2 = self.randomClientRequest()
- bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))])
-
- n1 = dist.getBridges(bridgeRequest1, 1)
- n2 = dist.getBridges(bridgeRequest2, 1)
-
- self.assertGreater(len(n1), 0)
- self.assertGreater(len(n2), 0)
-
- for b in n1:
- self.assertNotIn(b, n2)
- for b in n2:
- self.assertNotIn(b, n1)
-
- def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self):
- """The same client asking for bridges from the HTTPSDistributor
- multiple times in a row should get the same bridges in response each
- time.
- """
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- bridgeRequest = self.randomClientRequest()
- responses = {}
- for i in range(5):
- responses[i] = dist.getBridges(bridgeRequest, 1)
- for i in range(4):
- self.assertItemsEqual(responses[i], responses[i+1])
-
- def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self):
- param = BridgeRingParameters(needPorts=[(443, 1)])
- dist = Dist.HTTPSDistributor(3, self.key, answerParameters=param)
-
- bridges = self.bridges[:32]
- for b in self.bridges:
- b.orPort = 443
-
- [dist.insert(bridge) for bridge in bridges]
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- for _ in xrange(32):
- bridgeRequest = self.randomClientRequest()
- answer = dist.getBridges(bridgeRequest, 1)
- count = 0
- fingerprints = {}
- for bridge in answer:
- fingerprints[bridge.identity] = 1
- if bridge.orPort == 443:
- count += 1
- self.assertEquals(len(fingerprints), len(answer))
- self.assertGreater(len(fingerprints), 0)
- self.assertTrue(count >= 1)
-
- def test_HTTPSDistributor_getBridges_ipv4_ipv6(self):
- """Asking for bridge addresses which are simultaneously IPv4 and IPv6
- (in that order) should return IPv4 bridges.
- """
- dist = Dist.HTTPSDistributor(1, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- bridgeRequest = self.randomClientRequest()
- bridgeRequest.withIPv4()
- bridgeRequest.filters.append(byIPv6)
- bridgeRequest.generateFilters()
-
- bridges = dist.getBridges(bridgeRequest, 1)
- self.assertEqual(len(bridges), 3)
-
- bridge = random.choice(bridges)
- bridgeLine = bridge.getBridgeLine(bridgeRequest)
- addrport, fingerprint = bridgeLine.split()
- address, port = addrport.rsplit(':', 1)
- address = address.strip('[]')
- self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address)
- self.assertIsNotNone(byIPv4(random.choice(bridges)))
-
- def test_HTTPSDistributor_getBridges_ipv6_ipv4(self):
- """Asking for bridge addresses which are simultaneously IPv6 and IPv4
- (in that order) should return IPv6 bridges.
- """
- dist = Dist.HTTPSDistributor(1, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- bridgeRequest = self.randomClientRequest()
- bridgeRequest.withIPv6()
- bridgeRequest.generateFilters()
- bridgeRequest.filters.append(byIPv4)
-
- bridges = dist.getBridges(bridgeRequest, 1)
- self.assertEqual(len(bridges), 3)
-
- bridge = random.choice(bridges)
- bridgeLine = bridge.getBridgeLine(bridgeRequest)
- addrport, fingerprint = bridgeLine.split()
- address, port = addrport.rsplit(':', 1)
- address = address.strip('[]')
- self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address)
- self.assertIsNotNone(byIPv6(random.choice(bridges)))
-
- def test_HTTPSDistributor_getBridges_ipv6(self):
- """A request for IPv6 bridges should return IPv6 bridges."""
- dist = Dist.HTTPSDistributor(3, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- for i in xrange(500):
- bridgeRequest = self.randomClientRequest()
- bridgeRequest.withIPv6()
- bridgeRequest.generateFilters()
-
- bridges = dist.getBridges(bridgeRequest, "faketimestamp")
- self.assertTrue(type(bridges) is list)
- self.assertGreater(len(bridges), 0)
-
- bridge = random.choice(bridges)
- bridgeLine = bridge.getBridgeLine(bridgeRequest)
- addrport, fingerprint = bridgeLine.split()
- address, port = addrport.rsplit(':', 1)
- address = address.strip('[]')
- self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address)
- self.assertIsNotNone(byIPv6(random.choice(bridges)))
-
- def test_HTTPSDistributor_getBridges_ipv4(self):
- """A request for IPv4 bridges should return IPv4 bridges."""
- dist = Dist.HTTPSDistributor(1, self.key)
- [dist.insert(bridge) for bridge in self.bridges[:250]]
-
- for i in xrange(500):
- bridgeRequest = self.randomClientRequest()
- bridgeRequest.generateFilters()
-
- bridges = dist.getBridges(bridgeRequest, "faketimestamp")
- self.assertTrue(type(bridges) is list)
- self.assertGreater(len(bridges), 0)
-
- bridge = random.choice(bridges)
- bridgeLine = bridge.getBridgeLine(bridgeRequest)
- addrport, fingerprint = bridgeLine.split()
- address, port = addrport.rsplit(':', 1)
- self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address)
- self.assertIsNotNone(byIPv4(random.choice(bridges)))
diff --git a/lib/bridgedb/test/test_https_distributor.py b/lib/bridgedb/test/test_https_distributor.py
new file mode 100644
index 0000000..83f2503
--- /dev/null
+++ b/lib/bridgedb/test/test_https_distributor.py
@@ -0,0 +1,405 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of BridgeDB, a Tor bridge distribution system.
+#
+# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis at torproject.org>
+# please also see AUTHORS file
+# :copyright: (c) 2013-2015 Isis Lovecruft
+# (c) 2007-2015, The Tor Project, Inc.
+# (c) 2007-2015, all entities within the AUTHORS file
+# :license: 3-clause BSD, see included LICENSE for information
+
+"""Tests for :mod:`bridgedb.https.distributor`."""
+
+from __future__ import print_function
+
+import ipaddr
+import logging
+import random
+
+from twisted.trial import unittest
+
+from bridgedb.Bridges import BridgeRing
+from bridgedb.Bridges import BridgeRingParameters
+from bridgedb.filters import byIPv4
+from bridgedb.filters import byIPv6
+from bridgedb.https import distributor
+from bridgedb.https.request import HTTPSBridgeRequest
+from bridgedb.proxy import ProxySet
+from bridgedb.test.util import randomValidIPv4String
+from bridgedb.test.util import generateFakeBridges
+from bridgedb.test.https_helpers import DummyRequest
+
+logging.disable(50)
+
+
+BRIDGES = generateFakeBridges()
+
+
+class HTTPSDistributorTests(unittest.TestCase):
+ """Tests for :class:`HTTPSDistributor`."""
+
+ def setUp(self):
+ self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X'
+ self.bridges = BRIDGES
+
+ def tearDown(self):
+ """Reset all bridge blocks in between test method runs."""
+ for bridge in self.bridges:
+ bridge._blockedIn = {}
+
+ def coinFlip(self):
+ return bool(random.getrandbits(1))
+
+ def randomClientRequest(self):
+ bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False)
+ bridgeRequest.client = randomValidIPv4String()
+ bridgeRequest.isValid(True)
+ bridgeRequest.generateFilters()
+ return bridgeRequest
+
+ def randomClientRequestForNotBlockedIn(self, cc):
+ httpRequest = DummyRequest([''])
+ httpRequest.args.update({'unblocked': [cc]})
+ bridgeRequest = self.randomClientRequest()
+ bridgeRequest.withoutBlockInCountry(httpRequest)
+ bridgeRequest.generateFilters()
+ return bridgeRequest
+
+ def test_HTTPSDistributor_init_with_proxies(self):
+ """The HTTPSDistributor, when initialised with proxies, should add an
+ extra hashring for proxy users.
+ """
+ dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
+ self.assertIsNotNone(dist.proxies)
+ self.assertGreater(dist.proxySubring, 0)
+ self.assertEqual(dist.proxySubring, 4)
+ self.assertEqual(dist.totalSubrings, 4)
+
+ def test_HTTPSDistributor_bridgesPerResponse_120(self):
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:120]]
+ self.assertEqual(dist.bridgesPerResponse(), 3)
+
+ def test_HTTPSDistributor_bridgesPerResponse_100(self):
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.hashring.insert(bridge) for bridge in self.bridges[:100]]
+ self.assertEqual(dist.bridgesPerResponse(), 3)
+
+ def test_HTTPSDistributor_bridgesPerResponse_50(self):
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:60]]
+ self.assertEqual(dist.bridgesPerResponse(), 2)
+
+ def test_HTTPSDistributor_bridgesPerResponse_15(self):
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:15]]
+ self.assertEqual(dist.bridgesPerResponse(), 1)
+
+ def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self):
+ dist = distributor.HTTPSDistributor(3, self.key)
+ dist._bridgesPerResponseMax = 5
+ [dist.insert(bridge) for bridge in self.bridges[:100]]
+ self.assertEqual(dist.bridgesPerResponse(), 5)
+
+ def test_HTTPSDistributor_getSubnet_usingProxy(self):
+ """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy
+ group number.
+ """
+ clientRequest = self.randomClientRequest()
+ expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1
+ subnet = distributor.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True)
+ self.assertTrue(subnet.startswith('proxy-group-'))
+ self.assertEqual(int(subnet[-1]), expectedGroup)
+
+ def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self):
+ """HTTPSDistributor.mapSubnetToSubring() when the client was using a
+ proxy should map the client to the proxy subhashring.
+ """
+ dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
+ subnet = 'proxy-group-3'
+ subring = dist.mapSubnetToSubring(subnet, usingProxy=True)
+ self.assertEqual(subring, dist.proxySubring)
+
+ def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self):
+ """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using
+ a proxy, but the distributor does have some known proxies and a
+ proxySubring, should not map the client to the proxy subhashring.
+ """
+ dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
+ # Note that if they were actually from a proxy, their subnet would be
+ # something like "proxy-group-3".
+ subnet = '15.1.0.0/16'
+ subring = dist.mapSubnetToSubring(subnet, usingProxy=False)
+ self.assertNotEqual(subring, dist.proxySubring)
+
+ def test_HTTPSDistributor_prepopulateRings_with_proxies(self):
+ """An HTTPSDistributor with proxies should prepopulate two extra
+ subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6).
+ """
+ dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2']))
+ [dist.insert(bridge) for bridge in self.bridges]
+ dist.prepopulateRings()
+ self.assertEqual(len(dist.hashring.filterRings), 8)
+
+ def test_HTTPSDistributor_prepopulateRings_without_proxies(self):
+ """An HTTPSDistributor without proxies should prepopulate
+ totalSubrings * 2 subrings.
+ """
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges]
+ dist.prepopulateRings()
+ self.assertEqual(len(dist.hashring.filterRings), 6)
+
+ ipv4subrings = []
+ ipv6subrings = []
+
+ for subringName, (filters, subring) in dist.hashring.filterRings.items():
+ if 'IPv4' in subringName:
+ ipv6subrings.append(subring)
+ if 'IPv6' in subringName:
+ ipv6subrings.append(subring)
+
+ self.assertEqual(len(ipv4subrings), len(ipv6subrings))
+
+ def test_HTTPSDistributor_getBridges_with_blocked_bridges(self):
+ dist = distributor.HTTPSDistributor(1, self.key)
+ bridges = self.bridges[:]
+
+ for bridge in bridges:
+ bridge.setBlockedIn('cn')
+
+ [dist.insert(bridge) for bridge in bridges]
+
+ for _ in range(5):
+ clientRequest1 = self.randomClientRequestForNotBlockedIn('cn')
+ b = dist.getBridges(clientRequest1, 1)
+ self.assertEqual(len(b), 0)
+
+ clientRequest2 = self.randomClientRequestForNotBlockedIn('ir')
+ b = dist.getBridges(clientRequest2, 1)
+ self.assertEqual(len(b), 3)
+
+ def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self):
+ dist = distributor.HTTPSDistributor(1, self.key)
+ bridges = self.bridges[:]
+
+ blockedCN = []
+ blockedIR = []
+
+ for bridge in bridges:
+ if self.coinFlip():
+ bridge.setBlockedIn('cn')
+ blockedCN.append(bridge.fingerprint)
+
+ if self.coinFlip():
+ bridge.setBlockedIn('ir')
+ blockedIR.append(bridge.fingerprint)
+
+ [dist.insert(bridge) for bridge in bridges]
+
+ for _ in range(5):
+ clientRequest1 = self.randomClientRequestForNotBlockedIn('cn')
+ bridges = dist.getBridges(clientRequest1, 1)
+ for b in bridges:
+ self.assertFalse(b.isBlockedIn('cn'))
+ self.assertNotIn(b.fingerprint, blockedCN)
+ # The client *should* have gotten some bridges still.
+ self.assertGreater(len(bridges), 0)
+
+ clientRequest2 = self.randomClientRequestForNotBlockedIn('ir')
+ bridges = dist.getBridges(clientRequest2, 1)
+ for b in bridges:
+ self.assertFalse(b.isBlockedIn('ir'))
+ self.assertNotIn(b.fingerprint, blockedIR)
+ self.assertGreater(len(bridges), 0)
+
+ def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self):
+ dist = distributor.HTTPSDistributor(1, self.key)
+ bridges = self.bridges[:]
+
+ for bridge in bridges:
+ # Pretend that China blocks all vanilla bridges:
+ bridge.setBlockedIn('cn', methodname='vanilla')
+ # Pretend that China blocks all obfs2:
+ bridge.setBlockedIn('cn', methodname='obfs2')
+ # Pretend that China blocks some obfs3:
+ if self.coinFlip():
+ bridge.setBlockedIn('cn', methodname='obfs3')
+
+ [dist.insert(bridge) for bridge in bridges]
+
+ for i in xrange(5):
+ bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn')
+ bridgeRequest1.transports.append('obfs2')
+ bridgeRequest1.generateFilters()
+ # We shouldn't get any obfs2 bridges, since they're all blocked in
+ # China:
+ bridges = dist.getBridges(bridgeRequest1, "faketimestamp")
+ self.assertEqual(len(bridges), 0)
+
+ bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn')
+ bridgeRequest2.transports.append('obfs3')
+ bridgeRequest2.generateFilters()
+ # We probably will get at least one bridge back! It's pretty
+ # unlikely to lose a coin flip 500 times in a row.
+ bridges = dist.getBridges(bridgeRequest2, "faketimestamp")
+ self.assertGreater(len(bridges), 0)
+
+ bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl')
+ bridgeRequest3.transports.append('obfs3')
+ bridgeRequest3.generateFilters()
+ # We should get bridges, since obfs3 isn't blocked in netherlands:
+ bridges = dist.getBridges(bridgeRequest3, "faketimestamp")
+ self.assertGreater(len(bridges), 0)
+
+ def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self):
+ """An HTTPSDistributor should give separate bridges to proxy users."""
+ proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)])
+ dist = distributor.HTTPSDistributor(3, self.key, proxies)
+ [dist.insert(bridge) for bridge in self.bridges]
+
+ for _ in range(10):
+ bridgeRequest1 = self.randomClientRequest()
+ bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))])
+
+ bridgeRequest2 = self.randomClientRequest()
+ bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))])
+
+ n1 = dist.getBridges(bridgeRequest1, 1)
+ n2 = dist.getBridges(bridgeRequest2, 1)
+
+ self.assertGreater(len(n1), 0)
+ self.assertGreater(len(n2), 0)
+
+ for b in n1:
+ self.assertNotIn(b, n2)
+ for b in n2:
+ self.assertNotIn(b, n1)
+
+ def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self):
+ """The same client asking for bridges from the HTTPSDistributor
+ multiple times in a row should get the same bridges in response each
+ time.
+ """
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ bridgeRequest = self.randomClientRequest()
+ responses = {}
+ for i in range(5):
+ responses[i] = dist.getBridges(bridgeRequest, 1)
+ for i in range(4):
+ self.assertItemsEqual(responses[i], responses[i+1])
+
+ def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self):
+ param = BridgeRingParameters(needPorts=[(443, 1)])
+ dist = distributor.HTTPSDistributor(3, self.key, answerParameters=param)
+
+ bridges = self.bridges[:32]
+ for b in self.bridges:
+ b.orPort = 443
+
+ [dist.insert(bridge) for bridge in bridges]
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ for _ in xrange(32):
+ bridgeRequest = self.randomClientRequest()
+ answer = dist.getBridges(bridgeRequest, 1)
+ count = 0
+ fingerprints = {}
+ for bridge in answer:
+ fingerprints[bridge.identity] = 1
+ if bridge.orPort == 443:
+ count += 1
+ self.assertEquals(len(fingerprints), len(answer))
+ self.assertGreater(len(fingerprints), 0)
+ self.assertTrue(count >= 1)
+
+ def test_HTTPSDistributor_getBridges_ipv4_ipv6(self):
+ """Asking for bridge addresses which are simultaneously IPv4 and IPv6
+ (in that order) should return IPv4 bridges.
+ """
+ dist = distributor.HTTPSDistributor(1, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ bridgeRequest = self.randomClientRequest()
+ bridgeRequest.withIPv4()
+ bridgeRequest.filters.append(byIPv6)
+ bridgeRequest.generateFilters()
+
+ bridges = dist.getBridges(bridgeRequest, 1)
+ self.assertEqual(len(bridges), 3)
+
+ bridge = random.choice(bridges)
+ bridgeLine = bridge.getBridgeLine(bridgeRequest)
+ addrport, fingerprint = bridgeLine.split()
+ address, port = addrport.rsplit(':', 1)
+ address = address.strip('[]')
+ self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address)
+ self.assertIsNotNone(byIPv4(random.choice(bridges)))
+
+ def test_HTTPSDistributor_getBridges_ipv6_ipv4(self):
+ """Asking for bridge addresses which are simultaneously IPv6 and IPv4
+ (in that order) should return IPv6 bridges.
+ """
+ dist = distributor.HTTPSDistributor(1, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ bridgeRequest = self.randomClientRequest()
+ bridgeRequest.withIPv6()
+ bridgeRequest.generateFilters()
+ bridgeRequest.filters.append(byIPv4)
+
+ bridges = dist.getBridges(bridgeRequest, 1)
+ self.assertEqual(len(bridges), 3)
+
+ bridge = random.choice(bridges)
+ bridgeLine = bridge.getBridgeLine(bridgeRequest)
+ addrport, fingerprint = bridgeLine.split()
+ address, port = addrport.rsplit(':', 1)
+ address = address.strip('[]')
+ self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address)
+ self.assertIsNotNone(byIPv6(random.choice(bridges)))
+
+ def test_HTTPSDistributor_getBridges_ipv6(self):
+ """A request for IPv6 bridges should return IPv6 bridges."""
+ dist = distributor.HTTPSDistributor(3, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ for i in xrange(500):
+ bridgeRequest = self.randomClientRequest()
+ bridgeRequest.withIPv6()
+ bridgeRequest.generateFilters()
+
+ bridges = dist.getBridges(bridgeRequest, "faketimestamp")
+ self.assertTrue(type(bridges) is list)
+ self.assertGreater(len(bridges), 0)
+
+ bridge = random.choice(bridges)
+ bridgeLine = bridge.getBridgeLine(bridgeRequest)
+ addrport, fingerprint = bridgeLine.split()
+ address, port = addrport.rsplit(':', 1)
+ address = address.strip('[]')
+ self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address)
+ self.assertIsNotNone(byIPv6(random.choice(bridges)))
+
+ def test_HTTPSDistributor_getBridges_ipv4(self):
+ """A request for IPv4 bridges should return IPv4 bridges."""
+ dist = distributor.HTTPSDistributor(1, self.key)
+ [dist.insert(bridge) for bridge in self.bridges[:250]]
+
+ for i in xrange(500):
+ bridgeRequest = self.randomClientRequest()
+ bridgeRequest.generateFilters()
+
+ bridges = dist.getBridges(bridgeRequest, "faketimestamp")
+ self.assertTrue(type(bridges) is list)
+ self.assertGreater(len(bridges), 0)
+
+ bridge = random.choice(bridges)
+ bridgeLine = bridge.getBridgeLine(bridgeRequest)
+ addrport, fingerprint = bridgeLine.split()
+ address, port = addrport.rsplit(':', 1)
+ self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address)
+ self.assertIsNotNone(byIPv4(random.choice(bridges)))
diff --git a/lib/bridgedb/test/test_https_server.py b/lib/bridgedb/test/test_https_server.py
index e4f56f0..e782135 100644
--- a/lib/bridgedb/test/test_https_server.py
+++ b/lib/bridgedb/test/test_https_server.py
@@ -794,7 +794,7 @@ class HTTPSServerServiceTests(unittest.TestCase):
"""Unittests for :func:`bridgedb.email.server.addWebServer`."""
def setUp(self):
- """Create a server.MailServerContext and EmailBasedDistributor."""
+ """Create a config and an HTTPSDistributor."""
self.config = _createConfig()
self.distributor = DummyHTTPSDistributor()
diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py
index 2d0c020..f164aeb 100644
--- a/lib/bridgedb/test/util.py
+++ b/lib/bridgedb/test/util.py
@@ -168,6 +168,39 @@ randomValidIPv4String = valid(randomIPv4String)
randomValidIPv6String = valid(randomIPv6String)
randomValidIPString = valid(randomIPString)
+def generateFakeBridges(n=500):
+ """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random
+ data.
+ """
+ from bridgedb.bridges import Bridge
+ from bridgedb.bridges import PluggableTransport
+
+ bridges = []
+
+ for i in range(n):
+ addr = randomValidIPv4String()
+ nick = 'bridge-%d' % i
+ port = randomHighPort()
+ # Real tor currently only supports one extra ORAddress, and it can
+ # only be IPv6.
+ addrs = [(randomValidIPv6(), randomHighPort(), 6)]
+ fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40))
+
+ # We only support the ones without PT args, because they're easier to fake.
+ supported = ["obfs2", "obfs3", "fte"]
+ transports = []
+ for j, method in zip(range(1, len(supported) + 1), supported):
+ pt = PluggableTransport(fpr, method, addr, port - j, {})
+ transports.append(pt)
+
+ bridge = Bridge(nick, addr, port, fpr)
+ bridge.flags.update("Running Stable")
+ bridge.transports = transports
+ bridge.orAddresses = addrs
+ bridges.append(bridge)
+
+ return bridges
+
#: Mixin class for use with :api:`~twisted.trial.unittest.TestCase`. A
#: ``TestCaseMixin`` can be used to add additional methods, which should be
More information about the tor-commits
mailing list