June 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Remove unnecessary use of logSafely() in bridgedb.Bridges.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 68434ca69f5a5b38fc877abf03064eddbe879956 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 20 03:45:24 2015 +0000 Remove unnecessary use of logSafely() in bridgedb.Bridges. Logging str(bridge) will produce a hashed fingerprint if safelogging is enabled, so there's no use in calling logSafely(bridge.fingerprint). --- lib/bridgedb/Bridges.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 1a06dd8..4b77cdc 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -539,14 +539,12 @@ class FilteredBridgeSplitter(object): """ # The bridge must be running to insert it: if not bridge.flags.running: - logging.warn( - "Skipping hashring insertion for non-running bridge: '%s'" - % logSafely(bridge.fingerprint)) + logging.warn(("Skipping hashring insertion for non-running " + "bridge: %s") % bridge) return index = 0 - logging.debug("Inserting %s into hashring" - % (logSafely(bridge.fingerprint))) + logging.debug("Inserting %s into hashring..." % bridge) for old_bridge in self.bridges[:]: if bridge.fingerprint == old_bridge.fingerprint: self.bridges[index] = bridge @@ -557,8 +555,8 @@ class FilteredBridgeSplitter(object): for ringname, (filterFn, subring) in self.filterRings.items(): if filterFn(bridge): subring.insert(bridge) - logging.debug("Inserted bridge '%s' into '%s' sub hashring" - % (logSafely(bridge.fingerprint), ringname)) + logging.debug("Inserted bridge %s into %s subhashring." % + (bridge, ringname)) def extractFilterNames(self, ringname): """Get the names of the filters applied to a particular sub hashring.

1 0

[bridgedb/develop] Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit dc91ea443e66223999c5b7a70c4b42025c0ac25c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 07:32:49 2015 +0000 Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor. * FIXES part of #12506: https://bugs.torproject.org/12506 --- doc/sphinx/source/bridgedb.https.rst | 1 + doc/sphinx/source/conf.py | 1 + lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 302 ------------------- lib/bridgedb/Main.py | 21 +- lib/bridgedb/https/distributor.py | 328 ++++++++++++++++++++ lib/bridgedb/https/request.py | 2 +- lib/bridgedb/https/server.py | 2 +- lib/bridgedb/persistent.py | 3 +- lib/bridgedb/test/https_helpers.py | 4 +- lib/bridgedb/test/legacy_Tests.py | 68 +---- lib/bridgedb/test/test_Dist.py | 434 --------------------------- lib/bridgedb/test/test_https_distributor.py | 405 +++++++++++++++++++++++++ lib/bridgedb/test/test_https_server.py | 2 +- lib/bridgedb/test/util.py | 33 ++ 15 files changed, 797 insertions(+), 811 deletions(-) diff --git a/doc/sphinx/source/bridgedb.https.rst b/doc/sphinx/source/bridgedb.https.rst index 1512065..36fa6c6 100644 --- a/doc/sphinx/source/bridgedb.https.rst +++ b/doc/sphinx/source/bridgedb.https.rst @@ -7,5 +7,6 @@ bridgedb.https :depth: 3 .. automodule:: bridgedb.https.__init__ +.. automodule:: bridgedb.https.distributor .. automodule:: bridgedb.https.request .. automodule:: bridgedb.https.server diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 9b43d0f..5038abb 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -43,6 +43,7 @@ import bridgedb.email.templates import bridgedb.filters import bridgedb.geo import bridgedb.https +import bridgedb.https.distributor import bridgedb.https.request import bridgedb.https.server import bridgedb.Main diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 2b56884..7a237fe 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -510,7 +510,7 @@ class FilteredBridgeSplitter(object): :ivar bridges: DOCDOC :type distributorName: str :ivar distributorName: The name of this splitter's distributor. See - :meth:`bridgedb.Dist.HTTPSDistributor.setDistributorName`. + :meth:`~bridgedb.https.distributor.HTTPSDistributor.setDistributorName`. """ self.key = key self.filterRings = {} diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 9b9e35c..1be705f 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -12,14 +12,11 @@ """This module has functions to decide which bridges to hand out to whom.""" -import ipaddr import logging -import re import time import bridgedb.Storage -from bridgedb import proxy from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC @@ -47,305 +44,6 @@ class EmailRequestedKey(Exception): """Raised when an incoming email requested a copy of our GnuPG keys.""" -class HTTPSDistributor(Distributor): - """A Distributor that hands out bridges based on the IP address of an - incoming request and the current time period. - - :type proxies: :class:`~bridgedb.proxies.ProxySet` - :ivar proxies: All known proxies, which we treat differently. See - :param:`proxies`. - :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter` - :ivar hashring: A hashring that assigns bridges to subrings with fixed - proportions. Used to assign bridges into the subrings of this - distributor. - """ - - def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): - """Create a Distributor that decides which bridges to distribute based - upon the client's IP address and the current time. - - :param int totalSubrings: The number of subhashrings to group clients - into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, - then the actual number of clusters is one higher than - ``totalSubrings``, because the set of all known open proxies is - given its own subhashring. - :param bytes key: The master HMAC key for this distributor. All added - bridges are HMACed with this key in order to place them into the - hashrings. - :type proxies: :class:`~bridgedb.proxy.ProxySet` - :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known - Tor Exit relays and other known proxies. These will constitute - the extra cluster, and any client requesting bridges from one of - these **proxies** will be distributed bridges from a separate - subhashring that is specific to Tor/proxy users. - :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` - :param answerParameters: A mechanism for ensuring that the set of - bridges that this distributor answers a client with fit certain - parameters, i.e. that an answer has "at least two obfsproxy - bridges" or "at least one bridge on port 443", etc. - """ - super(HTTPSDistributor, self).__init__(key) - self.totalSubrings = totalSubrings - self.answerParameters = answerParameters - - if proxies: - logging.info("Added known proxies to HTTPS distributor...") - self.proxies = proxies - self.totalSubrings += 1 - self.proxySubring = self.totalSubrings - else: - logging.warn("No known proxies were added to HTTPS distributor!") - self.proxies = proxy.ProxySet() - self.proxySubring = 0 - - self.ringCacheSize = self.totalSubrings * 3 - - key2 = getHMAC(key, "Assign-Bridges-To-Rings") - key3 = getHMAC(key, "Order-Areas-In-Rings") - key4 = getHMAC(key, "Assign-Areas-To-Rings") - - self._clientToPositionHMAC = getHMACFunc(key3, hex=False) - self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) - self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) - self.name = 'HTTPS' - logging.debug("Added %s to %s distributor." % - (self.hashring.__class__.__name__, self.name)) - - def bridgesPerResponse(self, hashring=None): - return super(HTTPSDistributor, self).bridgesPerResponse(hashring) - - @classmethod - def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): - """Map all clients whose **ip**s are within the same subnet to the same - arbitrary string. - - .. hint:: For non-proxy IP addresses, any two IPv4 addresses within - the same ``/16`` subnet, or any two IPv6 addresses in the same - ``/32`` subnet, will get the same string. - - Subnets for this distributor are grouped into the number of rings - specified by the ``N_IP_CLUSTERS`` configuration option, such that - Alice (with the address ``1.2.3.4`` and Bob (with the address - ``1.2.178.234``) are placed within the same cluster, but Carol (with - address ``1.3.11.33``) *might* end up in a different cluster. - - >>> from bridgedb.Dist import HTTPSDistributor - >>> HTTPSDistributor.getSubnet('1.2.3.4') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('1.2.211.154') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') - '2001:f::/32' - >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') - '2a00:c98::/32' - - :param str ip: A string representing an IPv4 or IPv6 address. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - :param int proxySubnets: Place Tor/proxy users into this number of - "subnet" groups. This means that no matter how many different Tor - Exits or proxies a client uses, the most they can ever get is - **proxySubnets** different sets of bridge lines (per interval). - This parameter only has any effect when **usingProxy** is ``True``. - :rtype: str - :returns: The appropriately sized CIDR subnet representation of the **ip**. - """ - if not usingProxy: - # We aren't using bridgedb.parse.addr.isIPAddress(ip, - # compressed=False) here because adding the string "False" into - # the map would land any and all clients whose IP address appeared - # to be invalid at the same position in a hashring. - address = ipaddr.IPAddress(ip) - if address.version == 6: - truncated = ':'.join(address.exploded.split(':')[:2]) - subnet = str(ipaddr.IPv6Network(truncated + "::/32")) - else: - truncated = '.'.join(address.exploded.split('.')[:2]) - subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) - else: - group = (int(ipaddr.IPAddress(ip)) % 4) + 1 - subnet = "proxy-group-%d" % group - - logging.debug("Client IP was within area: %s" % subnet) - return subnet - - def mapSubnetToSubring(self, subnet, usingProxy=False): - """Determine the correct subhashring for a client, based upon the - **subnet**. - - :param str subnet: The subnet which contains the client's IP. See - :staticmethod:`getSubnet`. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - """ - # If the client wasn't using a proxy, select the client's subring - # based upon the client's subnet (modulo the total subrings): - if not usingProxy: - mod = self.totalSubrings - # If there is a proxy subring, don't count it for the modulus: - if self.proxySubring: - mod -= 1 - return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 - else: - return self.proxySubring - - def mapClientToHashringPosition(self, interval, subnet): - """Map the client to a position on a (sub)hashring, based upon the - **interval** which the client's request occurred within, as well as - the **subnet** of the client's IP address. - - .. note:: For an explanation of how **subnet** is determined, see - :staticmethod:`getSubnet`. - - :param str interval: The interval which this client's request for - bridges took place within. - :param str subnet: A string representing the subnet containing the - client's IP address. - :rtype: int - :returns: The results of keyed HMAC, which should determine the - client's position in a (sub)hashring of bridges (and thus - determine which bridges they receive). - """ - position = "<%s>%s" % (interval, subnet) - mapping = self._clientToPositionHMAC(position) - return mapping - - def prepopulateRings(self): - """Prepopulate this distributor's hashrings and subhashrings with - bridges. - - The hashring structure for this distributor is influenced by the - ``N_IP_CLUSTERS`` configuration option, as well as the number of - ``PROXY_LIST_FILES``. - - Essentially, :data:`totalSubrings` is set to the specified - ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of - Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), - are stored in :data:`proxies`, and the latter is added as an - additional cluster (such that :data:`totalSubrings` becomes - ``N_IP_CLUSTERS + 1``). The number of subhashrings which this - :class:`Distributor` has active in its hashring is then - :data:`totalSubrings`, where the last cluster is reserved for all - :data:`proxies`. - - As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and - ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number - of subhashrings is five — four for the "clusters", and one for the - :data:`proxies`. Thus, the resulting hashring-subhashring structure - would look like: - - +------------------+---------------------------------------------------+-------------- - | | Directly connecting users | Tor / known | - | | | proxy users | - +------------------+------------+------------+------------+------------+-------------+ - | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | - +==================+============+============+============+============+=============+ - | Subhashrings | | | | | | - | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | - +------------------+------------+------------+------------+------------+-------------+ - | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | - | Subhashrings | | | | | | - | bBy requested +------------+------------+------------+------------+-------------+ - | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | - | | | | | | | - +------------------+------------+------------+------------+------------+-------------+ - - The "filtered subhashrings" are essentially filtered copies of their - respective subhashring, such that they only contain bridges which - support IPv4 or IPv6, respectively. Additionally, the contents of - ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. - - Thus, in this example, we end up with **10 total subhashrings**. - """ - logging.info("Prepopulating %s distributor hashrings..." % self.name) - - for filterFn in [byIPv4, byIPv6]: - for subring in range(1, self.totalSubrings + 1): - filters = self._buildHashringFilters([filterFn,], subring) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - # For consistency with previous implementation of this method, - # only set the "name" for "clusters" which are for this - # distributor's proxies: - if subring == self.proxySubring: - ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - def insert(self, bridge): - """Assign a bridge to this distributor.""" - self.hashring.insert(bridge) - - def _buildHashringFilters(self, previousFilters, subring): - f = bySubring(self.hashring.hmac, subring, self.totalSubrings) - previousFilters.append(f) - return frozenset(previousFilters) - - def getBridges(self, bridgeRequest, interval): - """Return a list of bridges to give to a user. - - :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` - :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` - with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` - attribute set to a string containing the client's IP address. - :param str interval: The time period when we got this request. This - can be any string, so long as it changes with every period. - :rtype: list - :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in - the response. See - :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` - for an example of how this is used. - """ - logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) - - if not len(self.hashring): - logging.warn("Bailing! Hashring has zero bridges!") - return [] - - usingProxy = False - - # First, check if the client's IP is one of the known :data:`proxies`: - if bridgeRequest.client in self.proxies: - # The tag is a tag applied to a proxy IP address when it is added - # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default - # is 'exit_relay'. For other proxies loaded from the - # PROXY_LIST_FILES config option, the default tag is the full - # filename that the IP address originally came from. - usingProxy = True - tag = self.proxies.getTag(bridgeRequest.client) - logging.info("Client was from known proxy (tag: %s): %s" % - (tag, bridgeRequest.client)) - - subnet = self.getSubnet(bridgeRequest.client, usingProxy) - subring = self.mapSubnetToSubring(subnet, usingProxy) - position = self.mapClientToHashringPosition(interval, subnet) - filters = self._buildHashringFilters(bridgeRequest.filters, subring) - - logging.debug("Client request within time interval: %s" % interval) - logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) - logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) - logging.debug("Total bridges: %d" % len(self.hashring)) - logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) - - # Check wheth we have a cached copy of the hashring: - if filters in self.hashring.filterRings.keys(): - logging.debug("Cache hit %s" % filters) - _, ring = self.hashring.filterRings[filters] - # Otherwise, construct a new hashring and populate it: - else: - logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - # Determine the appropriate number of bridges to give to the client: - returnNum = self.bridgesPerResponse(ring) - answer = ring.getBridges(position, returnNum) - - return answer - - class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index ba74b66..3a15145 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -30,6 +30,7 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig +from bridgedb.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors import bridgedb.Storage @@ -186,16 +187,17 @@ def createBridgeRings(cfg, proxyList, key): """Create the bridge distributors defined by the config file :type cfg: :class:`Conf` - :param cfg: The current configuration, including any in-memory - settings (i.e. settings whose values were not obtained from the - config file, but were set via a function somewhere) + :param cfg: The current configuration, including any in-memory settings + (i.e. settings whose values were not obtained from the config file, + but were set via a function somewhere) :type proxyList: :class:`~bridgedb.proxy.ProxySet` :param proxyList: The container for the IP addresses of any currently - known open proxies. + known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an HTTPSDistributor or None, - and an EmailBasedDistributor or None. + :returns: A BridgeSplitter hashring, an + :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an + EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -210,7 +212,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") - ipDistributor = Dist.HTTPSDistributor( + ipDistributor = HTTPSDistributor( cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, @@ -328,8 +330,9 @@ def run(options, reactor=reactor): into their hashring assignments. :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently - known open proxies. - :ivar ipDistributor: A :class:`Dist.HTTPSDistributor`. + known open proxies. + :ivar ipDistributor: A + :class:`~bridgedb.https.distributor.HTTPSDistributor`. :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some diff --git a/lib/bridgedb/https/distributor.py b/lib/bridgedb/https/distributor.py new file mode 100644 index 0000000..f8cf09d --- /dev/null +++ b/lib/bridgedb/https/distributor.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_https_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""A Distributor that hands out bridges through a web interface.""" + +import ipaddr +import logging + +import bridgedb.Storage + +from bridgedb import proxy +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import FilteredBridgeSplitter +from bridgedb.crypto import getHMAC +from bridgedb.crypto import getHMACFunc +from bridgedb.distribute import Distributor +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import byFilters +from bridgedb.filters import bySubring + + +class HTTPSDistributor(Distributor): + """A Distributor that hands out bridges based on the IP address of an + incoming request and the current time period. + + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. + :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. + """ + + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ + super(HTTPSDistributor, self).__init__(key) + self.totalSubrings = totalSubrings + self.answerParameters = answerParameters + + if proxies: + logging.info("Added known proxies to HTTPS distributor...") + self.proxies = proxies + self.totalSubrings += 1 + self.proxySubring = self.totalSubrings + else: + logging.warn("No known proxies were added to HTTPS distributor!") + self.proxies = proxy.ProxySet() + self.proxySubring = 0 + + self.ringCacheSize = self.totalSubrings * 3 + + key2 = getHMAC(key, "Assign-Bridges-To-Rings") + key3 = getHMAC(key, "Order-Areas-In-Rings") + key4 = getHMAC(key, "Assign-Areas-To-Rings") + + self._clientToPositionHMAC = getHMACFunc(key3, hex=False) + self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) + self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) + self.name = 'HTTPS' + logging.debug("Added %s to %s distributor." % + (self.hashring.__class__.__name__, self.name)) + + def bridgesPerResponse(self, hashring=None): + return super(HTTPSDistributor, self).bridgesPerResponse(hashring) + + @classmethod + def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): + """Map all clients whose **ip**s are within the same subnet to the same + arbitrary string. + + .. hint:: For non-proxy IP addresses, any two IPv4 addresses within + the same ``/16`` subnet, or any two IPv6 addresses in the same + ``/32`` subnet, will get the same string. + + Subnets for this distributor are grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option, such that + Alice (with the address ``1.2.3.4`` and Bob (with the address + ``1.2.178.234``) are placed within the same cluster, but Carol (with + address ``1.3.11.33``) *might* end up in a different cluster. + + >>> from bridgedb.https.distributor import HTTPSDistributor + >>> HTTPSDistributor.getSubnet('1.2.3.4') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('1.2.211.154') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') + '2001:f::/32' + >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') + '2a00:c98::/32' + + :param str ip: A string representing an IPv4 or IPv6 address. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + :param int proxySubnets: Place Tor/proxy users into this number of + "subnet" groups. This means that no matter how many different Tor + Exits or proxies a client uses, the most they can ever get is + **proxySubnets** different sets of bridge lines (per interval). + This parameter only has any effect when **usingProxy** is ``True``. + :rtype: str + :returns: The appropriately sized CIDR subnet representation of the **ip**. + """ + if not usingProxy: + # We aren't using bridgedb.parse.addr.isIPAddress(ip, + # compressed=False) here because adding the string "False" into + # the map would land any and all clients whose IP address appeared + # to be invalid at the same position in a hashring. + address = ipaddr.IPAddress(ip) + if address.version == 6: + truncated = ':'.join(address.exploded.split(':')[:2]) + subnet = str(ipaddr.IPv6Network(truncated + "::/32")) + else: + truncated = '.'.join(address.exploded.split('.')[:2]) + subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) + else: + group = (int(ipaddr.IPAddress(ip)) % 4) + 1 + subnet = "proxy-group-%d" % group + + logging.debug("Client IP was within area: %s" % subnet) + return subnet + + def mapSubnetToSubring(self, subnet, usingProxy=False): + """Determine the correct subhashring for a client, based upon the + **subnet**. + + :param str subnet: The subnet which contains the client's IP. See + :staticmethod:`getSubnet`. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + """ + # If the client wasn't using a proxy, select the client's subring + # based upon the client's subnet (modulo the total subrings): + if not usingProxy: + mod = self.totalSubrings + # If there is a proxy subring, don't count it for the modulus: + if self.proxySubring: + mod -= 1 + return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 + else: + return self.proxySubring + + def mapClientToHashringPosition(self, interval, subnet): + """Map the client to a position on a (sub)hashring, based upon the + **interval** which the client's request occurred within, as well as + the **subnet** of the client's IP address. + + .. note:: For an explanation of how **subnet** is determined, see + :staticmethod:`getSubnet`. + + :param str interval: The interval which this client's request for + bridges took place within. + :param str subnet: A string representing the subnet containing the + client's IP address. + :rtype: int + :returns: The results of keyed HMAC, which should determine the + client's position in a (sub)hashring of bridges (and thus + determine which bridges they receive). + """ + position = "<%s>%s" % (interval, subnet) + mapping = self._clientToPositionHMAC(position) + return mapping + + def prepopulateRings(self): + """Prepopulate this distributor's hashrings and subhashrings with + bridges. + + The hashring structure for this distributor is influenced by the + ``N_IP_CLUSTERS`` configuration option, as well as the number of + ``PROXY_LIST_FILES``. + + Essentially, :data:`totalSubrings` is set to the specified + ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of + Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), + are stored in :data:`proxies`, and the latter is added as an + additional cluster (such that :data:`totalSubrings` becomes + ``N_IP_CLUSTERS + 1``). The number of subhashrings which this + :class:`Distributor` has active in its hashring is then + :data:`totalSubrings`, where the last cluster is reserved for all + :data:`proxies`. + + As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and + ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number + of subhashrings is five — four for the "clusters", and one for the + :data:`proxies`. Thus, the resulting hashring-subhashring structure + would look like: + + +------------------+---------------------------------------------------+-------------- + | | Directly connecting users | Tor / known | + | | | proxy users | + +------------------+------------+------------+------------+------------+-------------+ + | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | + +==================+============+============+============+============+=============+ + | Subhashrings | | | | | | + | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | + +------------------+------------+------------+------------+------------+-------------+ + | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | + | Subhashrings | | | | | | + | bBy requested +------------+------------+------------+------------+-------------+ + | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | + | | | | | | | + +------------------+------------+------------+------------+------------+-------------+ + + The "filtered subhashrings" are essentially filtered copies of their + respective subhashring, such that they only contain bridges which + support IPv4 or IPv6, respectively. Additionally, the contents of + ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. + + Thus, in this example, we end up with **10 total subhashrings**. + """ + logging.info("Prepopulating %s distributor hashrings..." % self.name) + + for filterFn in [byIPv4, byIPv6]: + for subring in range(1, self.totalSubrings + 1): + filters = self._buildHashringFilters([filterFn,], subring) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + # For consistency with previous implementation of this method, + # only set the "name" for "clusters" which are for this + # distributor's proxies: + if subring == self.proxySubring: + ring.setName('{0} Proxy Ring'.format(self.name)) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + def insert(self, bridge): + """Assign a bridge to this distributor.""" + self.hashring.insert(bridge) + + def _buildHashringFilters(self, previousFilters, subring): + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) + previousFilters.append(f) + return frozenset(previousFilters) + + def getBridges(self, bridgeRequest, interval): + """Return a list of bridges to give to a user. + + :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` + :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` + with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` + attribute set to a string containing the client's IP address. + :param str interval: The time period when we got this request. This + can be any string, so long as it changes with every period. + :rtype: list + :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in + the response. See + :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` + for an example of how this is used. + """ + logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) + + if not len(self.hashring): + logging.warn("Bailing! Hashring has zero bridges!") + return [] + + usingProxy = False + + # First, check if the client's IP is one of the known :data:`proxies`: + if bridgeRequest.client in self.proxies: + # The tag is a tag applied to a proxy IP address when it is added + # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default + # is 'exit_relay'. For other proxies loaded from the + # PROXY_LIST_FILES config option, the default tag is the full + # filename that the IP address originally came from. + usingProxy = True + tag = self.proxies.getTag(bridgeRequest.client) + logging.info("Client was from known proxy (tag: %s): %s" % + (tag, bridgeRequest.client)) + + subnet = self.getSubnet(bridgeRequest.client, usingProxy) + subring = self.mapSubnetToSubring(subnet, usingProxy) + position = self.mapClientToHashringPosition(interval, subnet) + filters = self._buildHashringFilters(bridgeRequest.filters, subring) + + logging.debug("Client request within time interval: %s" % interval) + logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) + logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Total bridges: %d" % len(self.hashring)) + logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) + + # Check wheth we have a cached copy of the hashring: + if filters in self.hashring.filterRings.keys(): + logging.debug("Cache hit %s" % filters) + _, ring = self.hashring.filterRings[filters] + # Otherwise, construct a new hashring and populate it: + else: + logging.debug("Cache miss %s" % filters) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + # Determine the appropriate number of bridges to give to the client: + returnNum = self.bridgesPerResponse(ring) + answer = ring.getBridges(position, returnNum) + + return answer diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py index 0ae52c7..b106a13 100644 --- a/lib/bridgedb/https/request.py +++ b/lib/bridgedb/https/request.py @@ -55,7 +55,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self, addClientCountryCode=True): """Process a new bridge request received through the - :class:`~bridgedb.Dist.HTTPSDistributor`. + :class:`~bridgedb.https.distributor.HTTPSDistributor`. :param bool addClientCountryCode: If ``True``, then calling :meth:`withoutBlockInCountry` will attempt to add the client's own diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index 2a1d510..2106dbf 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -815,7 +815,7 @@ def addWebServer(config, distributor): GIMP_CAPTCHA_DIR GIMP_CAPTCHA_HMAC_KEYFILE GIMP_CAPTCHA_RSA_KEYFILE - :type distributor: :class:`bridgedb.Dist.HTTPSDistributor` + :type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor` :param distributor: A bridge distributor. :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 00726f6..0853c1e 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -26,6 +26,7 @@ from twisted.spread import jelly from bridgedb import Bridges from bridgedb import Dist from bridgedb import filters +from bridgedb.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -34,7 +35,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist, httpsDistributor) class MissingState(Exception): diff --git a/lib/bridgedb/test/https_helpers.py b/lib/bridgedb/test/https_helpers.py index 3d8ec19..e2c94ba 100644 --- a/lib/bridgedb/test/https_helpers.py +++ b/lib/bridgedb/test/https_helpers.py @@ -99,8 +99,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE): class DummyHTTPSDistributor(object): - """A mocked :class:`bridgedb.Dist.HTTPSDistributor` which is used to test - :class:`bridgedb.https.server.BridgesResource`. + """A mocked :class:`bridgedb.https.distributor.HTTPSDistributor` which is + used to test :class:`bridgedb.https.server.BridgesResource`. """ _bridge_class = util.DummyBridge _bridgesPerResponseMin = 3 diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 3ec634c..c0a9ff0 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -35,15 +35,13 @@ from bridgedb.test.util import randomIPv6 from bridgedb.test.util import randomIPString from bridgedb.test.util import randomIPv4String from bridgedb.test.util import randomIPv6String +from bridgedb.test.util import randomPort +from bridgedb.test.util import randomValidIPv6 from math import log +warnings.filterwarnings('ignore', '.*tmpnam.*') -def suppressWarnings(): - warnings.filterwarnings('ignore', '.*tmpnam.*') - -def randomPort(): - return random.randint(1,65535) def randomPortSpec(): """ @@ -57,8 +55,8 @@ def randomPortSpec(): def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv4()) + ip = randomIPv4() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -66,21 +64,7 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv4String() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: for i in xrange(0,8): @@ -91,8 +75,8 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv6()) + ip = randomIPv6() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -100,47 +84,15 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv6() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - address = bracketIPv6(address) - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: for i in xrange(0,8): b.transports.append(bridgedb.Bridges.PluggableTransport(b, random.choice(["obfs", "obfs2", "pt1"]), randomIP(), randomPort())) - return b -simpleDesc = "router Unnamed %s %s 0 9030\n"\ -"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ -"opt @purpose bridge\n" -orAddress = "or-address %s:%s\n" - - -class RhymesWith255ProxySet: - def __contains__(self, ip): - return ip.endswith(".255") class EmailBridgeDistTests(unittest.TestCase): def setUp(self): @@ -412,6 +364,4 @@ def testSuite(): return suite def main(): - suppressWarnings() - unittest.TextTestRunner(verbosity=1).run(testSuite()) diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py deleted file mode 100644 index 46a11b0..0000000 --- a/lib/bridgedb/test/test_Dist.py +++ /dev/null @@ -1,434 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> -# please also see AUTHORS file -# :copyright: (c) 2013-2015 Isis Lovecruft -# (c) 2007-2015, The Tor Project, Inc. -# (c) 2007-2015, all entities within the AUTHORS file -# :license: 3-clause BSD, see included LICENSE for information - -"""Tests for :mod:`bridgedb.Dist`.""" - -from __future__ import print_function - -import hashlib -import ipaddr -import random - -from twisted.trial import unittest - -from bridgedb import Dist -from bridgedb.bridges import Bridge -from bridgedb.bridges import PluggableTransport -from bridgedb.Bridges import BridgeRing -from bridgedb.Bridges import BridgeRingParameters -from bridgedb.filters import byIPv4 -from bridgedb.filters import byIPv6 -from bridgedb.https.request import HTTPSBridgeRequest -from bridgedb.proxy import ProxySet -from bridgedb.test.util import randomHighPort -from bridgedb.test.util import randomValidIPv4String -from bridgedb.test.util import randomValidIPv6 -from bridgedb.test.https_helpers import DummyRequest - - -def _generateFakeBridges(n=500): - bridges = [] - - for i in range(n): - addr = randomValidIPv4String() - nick = 'bridge-%d' % i - port = randomHighPort() - # Real tor currently only supports one extra ORAddress, and it can - # only be IPv6. - addrs = [(randomValidIPv6(), randomHighPort(), 6)] - fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) - - # We only support the ones without PT args, because they're easier to fake. - supported = ["obfs2", "obfs3", "fte"] - transports = [] - for j, method in zip(range(1, len(supported) + 1), supported): - pt = PluggableTransport(fpr, method, addr, port - j, {}) - transports.append(pt) - - bridge = Bridge(nick, addr, port, fpr) - bridge.flags.update("Running Stable") - bridge.transports = transports - bridge.orAddresses = addrs - bridges.append(bridge) - - return bridges - - -BRIDGES = _generateFakeBridges() - - -class HTTPSDistributorTests(unittest.TestCase): - """Tests for :class:`HTTPSDistributor`.""" - - def setUp(self): - self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' - self.bridges = BRIDGES - - def tearDown(self): - """Reset all bridge blocks in between test method runs.""" - for bridge in self.bridges: - bridge._blockedIn = {} - - def coinFlip(self): - return bool(random.getrandbits(1)) - - def randomClientRequest(self): - bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) - bridgeRequest.client = randomValidIPv4String() - bridgeRequest.isValid(True) - bridgeRequest.generateFilters() - return bridgeRequest - - def randomClientRequestForNotBlockedIn(self, cc): - httpRequest = DummyRequest(['']) - httpRequest.args.update({'unblocked': [cc]}) - bridgeRequest = self.randomClientRequest() - bridgeRequest.withoutBlockInCountry(httpRequest) - bridgeRequest.generateFilters() - return bridgeRequest - - def test_HTTPSDistributor_init_with_proxies(self): - """The HTTPSDistributor, when initialised with proxies, should add an - extra hashring for proxy users. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - self.assertIsNotNone(dist.proxies) - self.assertGreater(dist.proxySubring, 0) - self.assertEqual(dist.proxySubring, 4) - self.assertEqual(dist.totalSubrings, 4) - - def test_HTTPSDistributor_bridgesPerResponse_120(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:120]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_100(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_50(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:60]] - self.assertEqual(dist.bridgesPerResponse(), 2) - - def test_HTTPSDistributor_bridgesPerResponse_15(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:15]] - self.assertEqual(dist.bridgesPerResponse(), 1) - - def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): - dist = Dist.HTTPSDistributor(3, self.key) - dist._bridgesPerResponseMax = 5 - [dist.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 5) - - def test_HTTPSDistributor_getSubnet_usingProxy(self): - """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy - group number. - """ - clientRequest = self.randomClientRequest() - expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 - subnet = Dist.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) - self.assertTrue(subnet.startswith('proxy-group-')) - self.assertEqual(int(subnet[-1]), expectedGroup) - - def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): - """HTTPSDistributor.mapSubnetToSubring() when the client was using a - proxy should map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - subnet = 'proxy-group-3' - subring = dist.mapSubnetToSubring(subnet, usingProxy=True) - self.assertEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): - """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using - a proxy, but the distributor does have some known proxies and a - proxySubring, should not map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - # Note that if they were actually from a proxy, their subnet would be - # something like "proxy-group-3". - subnet = '15.1.0.0/16' - subring = dist.mapSubnetToSubring(subnet, usingProxy=False) - self.assertNotEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_prepopulateRings_with_proxies(self): - """An HTTPSDistributor with proxies should prepopulate two extra - subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 8) - - def test_HTTPSDistributor_prepopulateRings_without_proxies(self): - """An HTTPSDistributor without proxies should prepopulate - totalSubrings * 2 subrings. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 6) - - ipv4subrings = [] - ipv6subrings = [] - - for subringName, (filters, subring) in dist.hashring.filterRings.items(): - if 'IPv4' in subringName: - ipv6subrings.append(subring) - if 'IPv6' in subringName: - ipv6subrings.append(subring) - - self.assertEqual(len(ipv4subrings), len(ipv6subrings)) - - def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - bridge.setBlockedIn('cn') - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - b = dist.getBridges(clientRequest1, 1) - self.assertEqual(len(b), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - b = dist.getBridges(clientRequest2, 1) - self.assertEqual(len(b), 3) - - def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - blockedCN = [] - blockedIR = [] - - for bridge in bridges: - if self.coinFlip(): - bridge.setBlockedIn('cn') - blockedCN.append(bridge.fingerprint) - - if self.coinFlip(): - bridge.setBlockedIn('ir') - blockedIR.append(bridge.fingerprint) - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridges = dist.getBridges(clientRequest1, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('cn')) - self.assertNotIn(b.fingerprint, blockedCN) - # The client *should* have gotten some bridges still. - self.assertGreater(len(bridges), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - bridges = dist.getBridges(clientRequest2, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('ir')) - self.assertNotIn(b.fingerprint, blockedIR) - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - # Pretend that China blocks all vanilla bridges: - bridge.setBlockedIn('cn', methodname='vanilla') - # Pretend that China blocks all obfs2: - bridge.setBlockedIn('cn', methodname='obfs2') - # Pretend that China blocks some obfs3: - if self.coinFlip(): - bridge.setBlockedIn('cn', methodname='obfs3') - - [dist.insert(bridge) for bridge in bridges] - - for i in xrange(5): - bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest1.transports.append('obfs2') - bridgeRequest1.generateFilters() - # We shouldn't get any obfs2 bridges, since they're all blocked in - # China: - bridges = dist.getBridges(bridgeRequest1, "faketimestamp") - self.assertEqual(len(bridges), 0) - - bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest2.transports.append('obfs3') - bridgeRequest2.generateFilters() - # We probably will get at least one bridge back! It's pretty - # unlikely to lose a coin flip 500 times in a row. - bridges = dist.getBridges(bridgeRequest2, "faketimestamp") - self.assertGreater(len(bridges), 0) - - bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') - bridgeRequest3.transports.append('obfs3') - bridgeRequest3.generateFilters() - # We should get bridges, since obfs3 isn't blocked in netherlands: - bridges = dist.getBridges(bridgeRequest3, "faketimestamp") - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): - """An HTTPSDistributor should give separate bridges to proxy users.""" - proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) - dist = Dist.HTTPSDistributor(3, self.key, proxies) - [dist.insert(bridge) for bridge in self.bridges] - - for _ in range(10): - bridgeRequest1 = self.randomClientRequest() - bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) - - bridgeRequest2 = self.randomClientRequest() - bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) - - n1 = dist.getBridges(bridgeRequest1, 1) - n2 = dist.getBridges(bridgeRequest2, 1) - - self.assertGreater(len(n1), 0) - self.assertGreater(len(n2), 0) - - for b in n1: - self.assertNotIn(b, n2) - for b in n2: - self.assertNotIn(b, n1) - - def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): - """The same client asking for bridges from the HTTPSDistributor - multiple times in a row should get the same bridges in response each - time. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - responses = {} - for i in range(5): - responses[i] = dist.getBridges(bridgeRequest, 1) - for i in range(4): - self.assertItemsEqual(responses[i], responses[i+1]) - - def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): - param = BridgeRingParameters(needPorts=[(443, 1)]) - dist = Dist.HTTPSDistributor(3, self.key, answerParameters=param) - - bridges = self.bridges[:32] - for b in self.bridges: - b.orPort = 443 - - [dist.insert(bridge) for bridge in bridges] - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for _ in xrange(32): - bridgeRequest = self.randomClientRequest() - answer = dist.getBridges(bridgeRequest, 1) - count = 0 - fingerprints = {} - for bridge in answer: - fingerprints[bridge.identity] = 1 - if bridge.orPort == 443: - count += 1 - self.assertEquals(len(fingerprints), len(answer)) - self.assertGreater(len(fingerprints), 0) - self.assertTrue(count >= 1) - - def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): - """Asking for bridge addresses which are simultaneously IPv4 and IPv6 - (in that order) should return IPv4 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv4() - bridgeRequest.filters.append(byIPv6) - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): - """Asking for bridge addresses which are simultaneously IPv6 and IPv4 - (in that order) should return IPv6 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - bridgeRequest.filters.append(byIPv4) - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6(self): - """A request for IPv6 bridges should return IPv6 bridges.""" - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv4(self): - """A request for IPv4 bridges should return IPv4 bridges.""" - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_distributor.py b/lib/bridgedb/test/test_https_distributor.py new file mode 100644 index 0000000..83f2503 --- /dev/null +++ b/lib/bridgedb/test/test_https_distributor.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Tests for :mod:`bridgedb.https.distributor`.""" + +from __future__ import print_function + +import ipaddr +import logging +import random + +from twisted.trial import unittest + +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import BridgeRingParameters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.https import distributor +from bridgedb.https.request import HTTPSBridgeRequest +from bridgedb.proxy import ProxySet +from bridgedb.test.util import randomValidIPv4String +from bridgedb.test.util import generateFakeBridges +from bridgedb.test.https_helpers import DummyRequest + +logging.disable(50) + + +BRIDGES = generateFakeBridges() + + +class HTTPSDistributorTests(unittest.TestCase): + """Tests for :class:`HTTPSDistributor`.""" + + def setUp(self): + self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' + self.bridges = BRIDGES + + def tearDown(self): + """Reset all bridge blocks in between test method runs.""" + for bridge in self.bridges: + bridge._blockedIn = {} + + def coinFlip(self): + return bool(random.getrandbits(1)) + + def randomClientRequest(self): + bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) + bridgeRequest.client = randomValidIPv4String() + bridgeRequest.isValid(True) + bridgeRequest.generateFilters() + return bridgeRequest + + def randomClientRequestForNotBlockedIn(self, cc): + httpRequest = DummyRequest(['']) + httpRequest.args.update({'unblocked': [cc]}) + bridgeRequest = self.randomClientRequest() + bridgeRequest.withoutBlockInCountry(httpRequest) + bridgeRequest.generateFilters() + return bridgeRequest + + def test_HTTPSDistributor_init_with_proxies(self): + """The HTTPSDistributor, when initialised with proxies, should add an + extra hashring for proxy users. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + self.assertIsNotNone(dist.proxies) + self.assertGreater(dist.proxySubring, 0) + self.assertEqual(dist.proxySubring, 4) + self.assertEqual(dist.totalSubrings, 4) + + def test_HTTPSDistributor_bridgesPerResponse_120(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:120]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_100(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_50(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:60]] + self.assertEqual(dist.bridgesPerResponse(), 2) + + def test_HTTPSDistributor_bridgesPerResponse_15(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:15]] + self.assertEqual(dist.bridgesPerResponse(), 1) + + def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): + dist = distributor.HTTPSDistributor(3, self.key) + dist._bridgesPerResponseMax = 5 + [dist.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 5) + + def test_HTTPSDistributor_getSubnet_usingProxy(self): + """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy + group number. + """ + clientRequest = self.randomClientRequest() + expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 + subnet = distributor.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) + self.assertTrue(subnet.startswith('proxy-group-')) + self.assertEqual(int(subnet[-1]), expectedGroup) + + def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): + """HTTPSDistributor.mapSubnetToSubring() when the client was using a + proxy should map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + subnet = 'proxy-group-3' + subring = dist.mapSubnetToSubring(subnet, usingProxy=True) + self.assertEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): + """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using + a proxy, but the distributor does have some known proxies and a + proxySubring, should not map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + # Note that if they were actually from a proxy, their subnet would be + # something like "proxy-group-3". + subnet = '15.1.0.0/16' + subring = dist.mapSubnetToSubring(subnet, usingProxy=False) + self.assertNotEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_prepopulateRings_with_proxies(self): + """An HTTPSDistributor with proxies should prepopulate two extra + subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 8) + + def test_HTTPSDistributor_prepopulateRings_without_proxies(self): + """An HTTPSDistributor without proxies should prepopulate + totalSubrings * 2 subrings. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 6) + + ipv4subrings = [] + ipv6subrings = [] + + for subringName, (filters, subring) in dist.hashring.filterRings.items(): + if 'IPv4' in subringName: + ipv6subrings.append(subring) + if 'IPv6' in subringName: + ipv6subrings.append(subring) + + self.assertEqual(len(ipv4subrings), len(ipv6subrings)) + + def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + bridge.setBlockedIn('cn') + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + b = dist.getBridges(clientRequest1, 1) + self.assertEqual(len(b), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + b = dist.getBridges(clientRequest2, 1) + self.assertEqual(len(b), 3) + + def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + blockedCN = [] + blockedIR = [] + + for bridge in bridges: + if self.coinFlip(): + bridge.setBlockedIn('cn') + blockedCN.append(bridge.fingerprint) + + if self.coinFlip(): + bridge.setBlockedIn('ir') + blockedIR.append(bridge.fingerprint) + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridges = dist.getBridges(clientRequest1, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('cn')) + self.assertNotIn(b.fingerprint, blockedCN) + # The client *should* have gotten some bridges still. + self.assertGreater(len(bridges), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + bridges = dist.getBridges(clientRequest2, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('ir')) + self.assertNotIn(b.fingerprint, blockedIR) + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + # Pretend that China blocks all vanilla bridges: + bridge.setBlockedIn('cn', methodname='vanilla') + # Pretend that China blocks all obfs2: + bridge.setBlockedIn('cn', methodname='obfs2') + # Pretend that China blocks some obfs3: + if self.coinFlip(): + bridge.setBlockedIn('cn', methodname='obfs3') + + [dist.insert(bridge) for bridge in bridges] + + for i in xrange(5): + bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest1.transports.append('obfs2') + bridgeRequest1.generateFilters() + # We shouldn't get any obfs2 bridges, since they're all blocked in + # China: + bridges = dist.getBridges(bridgeRequest1, "faketimestamp") + self.assertEqual(len(bridges), 0) + + bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest2.transports.append('obfs3') + bridgeRequest2.generateFilters() + # We probably will get at least one bridge back! It's pretty + # unlikely to lose a coin flip 500 times in a row. + bridges = dist.getBridges(bridgeRequest2, "faketimestamp") + self.assertGreater(len(bridges), 0) + + bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') + bridgeRequest3.transports.append('obfs3') + bridgeRequest3.generateFilters() + # We should get bridges, since obfs3 isn't blocked in netherlands: + bridges = dist.getBridges(bridgeRequest3, "faketimestamp") + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): + """An HTTPSDistributor should give separate bridges to proxy users.""" + proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) + dist = distributor.HTTPSDistributor(3, self.key, proxies) + [dist.insert(bridge) for bridge in self.bridges] + + for _ in range(10): + bridgeRequest1 = self.randomClientRequest() + bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) + + bridgeRequest2 = self.randomClientRequest() + bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) + + n1 = dist.getBridges(bridgeRequest1, 1) + n2 = dist.getBridges(bridgeRequest2, 1) + + self.assertGreater(len(n1), 0) + self.assertGreater(len(n2), 0) + + for b in n1: + self.assertNotIn(b, n2) + for b in n2: + self.assertNotIn(b, n1) + + def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): + """The same client asking for bridges from the HTTPSDistributor + multiple times in a row should get the same bridges in response each + time. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + responses = {} + for i in range(5): + responses[i] = dist.getBridges(bridgeRequest, 1) + for i in range(4): + self.assertItemsEqual(responses[i], responses[i+1]) + + def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): + param = BridgeRingParameters(needPorts=[(443, 1)]) + dist = distributor.HTTPSDistributor(3, self.key, answerParameters=param) + + bridges = self.bridges[:32] + for b in self.bridges: + b.orPort = 443 + + [dist.insert(bridge) for bridge in bridges] + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for _ in xrange(32): + bridgeRequest = self.randomClientRequest() + answer = dist.getBridges(bridgeRequest, 1) + count = 0 + fingerprints = {} + for bridge in answer: + fingerprints[bridge.identity] = 1 + if bridge.orPort == 443: + count += 1 + self.assertEquals(len(fingerprints), len(answer)) + self.assertGreater(len(fingerprints), 0) + self.assertTrue(count >= 1) + + def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): + """Asking for bridge addresses which are simultaneously IPv4 and IPv6 + (in that order) should return IPv4 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv4() + bridgeRequest.filters.append(byIPv6) + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): + """Asking for bridge addresses which are simultaneously IPv6 and IPv4 + (in that order) should return IPv6 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + bridgeRequest.filters.append(byIPv4) + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6(self): + """A request for IPv6 bridges should return IPv6 bridges.""" + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv4(self): + """A request for IPv4 bridges should return IPv4 bridges.""" + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_server.py b/lib/bridgedb/test/test_https_server.py index e4f56f0..e782135 100644 --- a/lib/bridgedb/test/test_https_server.py +++ b/lib/bridgedb/test/test_https_server.py @@ -794,7 +794,7 @@ class HTTPSServerServiceTests(unittest.TestCase): """Unittests for :func:`bridgedb.email.server.addWebServer`.""" def setUp(self): - """Create a server.MailServerContext and EmailBasedDistributor.""" + """Create a config and an HTTPSDistributor.""" self.config = _createConfig() self.distributor = DummyHTTPSDistributor() diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 2d0c020..f164aeb 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -168,6 +168,39 @@ randomValidIPv4String = valid(randomIPv4String) randomValidIPv6String = valid(randomIPv6String) randomValidIPString = valid(randomIPString) +def generateFakeBridges(n=500): + """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random + data. + """ + from bridgedb.bridges import Bridge + from bridgedb.bridges import PluggableTransport + + bridges = [] + + for i in range(n): + addr = randomValidIPv4String() + nick = 'bridge-%d' % i + port = randomHighPort() + # Real tor currently only supports one extra ORAddress, and it can + # only be IPv6. + addrs = [(randomValidIPv6(), randomHighPort(), 6)] + fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) + + # We only support the ones without PT args, because they're easier to fake. + supported = ["obfs2", "obfs3", "fte"] + transports = [] + for j, method in zip(range(1, len(supported) + 1), supported): + pt = PluggableTransport(fpr, method, addr, port - j, {}) + transports.append(pt) + + bridge = Bridge(nick, addr, port, fpr) + bridge.flags.update("Running Stable") + bridge.transports = transports + bridge.orAddresses = addrs + bridges.append(bridge) + + return bridges + #: Mixin class for use with :api:`~twisted.trial.unittest.TestCase`. A #: ``TestCaseMixin`` can be used to add additional methods, which should be

1 0

[bridgedb/develop] Change BridgeRequestBase.addressClass → BridgeRequestBase.ipVersion.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 9c34acd384a57f23f451142978b0fdbc4694dc72 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 01:04:42 2015 +0000 Change BridgeRequestBase.addressClass → BridgeRequestBase.ipVersion. Passing an integer around both requires less memory, and makes the IP version filters in bridgedb.filters run faster (since they no longer need to use isinstance()). * CHANGE bridgedb.bridges.Bridge to use ipVersion when working with BridgeRequestBase. --- lib/bridgedb/bridgerequest.py | 57 ++++++++++++++++++------------- lib/bridgedb/bridges.py | 12 +++---- lib/bridgedb/filters.py | 4 +-- lib/bridgedb/test/test_email_request.py | 16 ++++----- lib/bridgedb/test/util.py | 2 +- 5 files changed, 51 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 2ded7b3..1266145 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -27,10 +27,10 @@ from bridgedb.filters import byTransport class IRequestBridges(Interface): """Interface specification of client options for requested bridges.""" - addressClass = Attribute( - "The IP version of bridge addresses to distribute to the client.") filters = Attribute( "A list of callables used to filter bridges from a hashring.") + ipVersion = Attribute( + "The IP version of bridge addresses to distribute to the client.") transports = Attribute( "A list of strings of Pluggable Transport types requested.") notBlockedIn = Attribute( @@ -54,7 +54,7 @@ class IRequestBridges(Interface): def generateFilters(): """Build the list of callables, ``filters``, according to the current contents of the lists of ``transports``, ``notBlockedIn``, and the - ``addressClass``. + ``ipVersion``. """ def getHashringPlacement(): @@ -68,10 +68,10 @@ class IRequestBridges(Interface): """Determine if the request is ``valid`` according to some parameters.""" def withIPv4(): - """Set the ``addressClass`` to IPv4.""" + """Set the ``ipVersion`` to IPv4.""" def withIPv6(): - """Set the ``addressClass`` to IPv6.""" + """Set the ``ipVersion`` to IPv6.""" def withPluggableTransportType(typeOfPT): """Add this **typeOfPT** to the list of requested ``transports``.""" @@ -87,13 +87,8 @@ class BridgeRequestBase(object): implements(IRequestBridges) - def __init__(self, addressClass=None): - #: (:class:`ipaddr.IPv4Address` or :class:`ipaddr.IPv6Address`) The IP - #: version of bridge addresses to distribute to the client. - self.addressClass = addressClass - if not ((self.addressClass is ipaddr.IPv4Address) or - (self.addressClass is ipaddr.IPv6Address)): - self.addressClass = ipaddr.IPv4Address + def __init__(self, ipVersion=None): + self.ipVersion = ipVersion #: (list) A list of callables used to filter bridges from a hashring. self.filters = list() #: (list) A list of strings of Pluggable Transport types requested. @@ -110,6 +105,26 @@ class BridgeRequestBase(object): #: (bool) Should be ``True`` if the client's request was valid. self.valid = False + @property + def ipVersion(self): + """The IP version of bridge addresses to distribute to the client. + + :rtype: int + :returns: Either ``4`` or ``6``. + """ + return self._ipVersion + + @ipVersion.setter + def ipVersion(self, ipVersion): + """The IP version of bridge addresses to distribute to the client. + + :param int ipVersion: The IP address version for the bridge lines we + should distribute in response to this client request. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + self._ipVersion = ipVersion + def getHashringPlacement(self, key, client=None): """Create an HMAC of some **client** info using a **key**. @@ -145,10 +160,10 @@ class BridgeRequestBase(object): return self.valid def withIPv4(self): - self.addressClass = ipaddr.IPv4Address + self.ipVersion = 4 def withIPv6(self): - self.addressClass = ipaddr.IPv6Address + self.ipVersion = 6 def withoutBlockInCountry(self, country): self.notBlockedIn.append(country.lower()) @@ -175,21 +190,17 @@ class BridgeRequestBase(object): self.clearFilters() pt = self.justOnePTType() - msg = ("Adding a filter to %s for %s for IPv%s" - % (self.__class__.__name__, self.client, self.addressClass)) - - self.ipVersion = 4 - if self.addressClass is ipaddr.IPv6Address: - self.ipVersion = 6 + msg = ("Adding a filter to %s for %s for IPv%d" + % (self.__class__.__name__, self.client, self.ipVersion)) if self.notBlockedIn: for country in self.notBlockedIn: logging.info("%s %s bridges not blocked in %s..." % (msg, pt or "vanilla", country)) - self.addFilter(byNotBlockedIn(country, pt, self.addressClass)) + self.addFilter(byNotBlockedIn(country, pt, self.ipVersion)) elif pt: logging.info("%s %s bridges..." % (msg, pt)) - self.addFilter(byTransport(pt, self.addressClass)) + self.addFilter(byTransport(pt, self.ipVersion)) else: logging.info("%s bridges..." % msg) - self.addFilter(byIPv(self.addressClass)) + self.addFilter(byIPv(self.ipVersion)) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index f6e1d6b..f56b0e7 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -785,7 +785,8 @@ class BridgeBackwardsCompatibility(BridgeBase): :data:`bridgerequest.BridgeRequestBase.client`. :param str transport: A pluggable transport method name. """ - bridgeRequest = bridgerequest.BridgeRequestBase(addressClass) + ipVersion = 6 if addressClass is ipaddr.IPv6Address else 4 + bridgeRequest = bridgerequest.BridgeRequestBase(ipVersion) bridgeRequest.client = request if request else bridgeRequest.client bridgeRequest.isValid(True) @@ -1087,7 +1088,7 @@ class Bridge(BridgeBackwardsCompatibility): type. """ desired = bridgeRequest.justOnePTType() - addressClass = bridgeRequest.addressClass + ipVersion = bridgeRequest.ipVersion logging.info("Bridge %s answering request for %s transport..." % (self, desired)) @@ -1096,8 +1097,7 @@ class Bridge(BridgeBackwardsCompatibility): # their ``methodname`` matches the requested transport: transports = filter(lambda pt: pt.methodname == desired, self.transports) # Filter again for whichever of IPv4 or IPv6 was requested: - transports = filter(lambda pt: isinstance(pt.address, addressClass), - transports) + transports = filter(lambda pt: pt.address.version == ipVersion, transports) if not transports: raise PluggableTransportUnavailable( @@ -1136,13 +1136,13 @@ class Bridge(BridgeBackwardsCompatibility): """ logging.info( "Bridge %s answering request for IPv%s vanilla address..." % - (self, "6" if bridgeRequest.addressClass is ipaddr.IPv6Address else "4")) + (self, bridgeRequest.ipVersion)) addresses = [] for address, port, version in self.allVanillaAddresses: # Filter ``allVanillaAddresses`` by whether IPv4 or IPv6 was requested: - if isinstance(address, bridgeRequest.addressClass): + if version == bridgeRequest.ipVersion: # Determine if the address is blocked in any of the country # codes. Because :meth:`addressIsBlockedIn` returns a bool, # we get a list like: ``[True, False, False, True]``, and diff --git a/lib/bridgedb/filters.py b/lib/bridgedb/filters.py index 8843937..ca9b673 100644 --- a/lib/bridgedb/filters.py +++ b/lib/bridgedb/filters.py @@ -126,8 +126,8 @@ def byTransport(methodname=None, ipVersion=None): 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches **methodname**, and - 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an - instance of **addressClass**. + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` version + matches the **ipVersion**. :param str methodname: A Pluggable Transport :data:`~bridge.bridges.PluggableTransport.methodname`. diff --git a/lib/bridgedb/test/test_email_request.py b/lib/bridgedb/test/test_email_request.py index 7e7f6ea..745ea71 100644 --- a/lib/bridgedb/test/test_email_request.py +++ b/lib/bridgedb/test/test_email_request.py @@ -55,7 +55,7 @@ class DetermineBridgeRequestOptionsTests(unittest.TestCase): self.assertEqual(reqvest.isValid(), False) self.assertFalse(reqvest.wantsKey()) # Though they did request IPv6, technically. - self.assertIs(reqvest.addressClass, ipaddr.IPv6Address) + self.assertIs(reqvest.ipVersion, 6) # And they did request a transport, technically. self.assertEqual(len(reqvest.transports), 1) self.assertEqual(reqvest.transports[0], 'obfs3') @@ -71,7 +71,7 @@ class DetermineBridgeRequestOptionsTests(unittest.TestCase): self.assertEqual(reqvest.isValid(), True) self.assertFalse(reqvest.wantsKey()) # Though they didn't request IPv6, so it should default to IPv4. - self.assertIs(reqvest.addressClass, ipaddr.IPv4Address) + self.assertIs(reqvest.ipVersion, 4) # And they requested two transports. self.assertEqual(len(reqvest.transports), 2) self.assertEqual(reqvest.transports[0], 'obfs3') @@ -93,7 +93,7 @@ class DetermineBridgeRequestOptionsTests(unittest.TestCase): self.assertEqual(reqvest.isValid(), True) self.assertFalse(reqvest.wantsKey()) # Though they didn't request IPv6, so it should default to IPv4. - self.assertIs(reqvest.addressClass, ipaddr.IPv4Address) + self.assertIs(reqvest.ipVersion, 4) # And they requested two transports. self.assertEqual(len(reqvest.transports), 2) self.assertEqual(reqvest.transports[0], 'obfs3') @@ -116,7 +116,7 @@ class DetermineBridgeRequestOptionsTests(unittest.TestCase): lines = ['', 'get ipv6'] reqvest = request.determineBridgeRequestOptions(lines) - self.assertIs(reqvest.addressClass, ipaddr.IPv6Address) + self.assertIs(reqvest.ipVersion, 6) self.assertEqual(reqvest.isValid(), True) @@ -128,7 +128,7 @@ class EmailBridgeRequestTests(unittest.TestCase): self.request = request.EmailBridgeRequest() def tearDown(self): - """Reset cached 'unblocked'/'transport' lists and addressClass between + """Reset cached 'unblocked'/'transport' lists and ipVersion between tests. """ self.request.withIPv4() @@ -174,10 +174,10 @@ class EmailBridgeRequestTests(unittest.TestCase): self.assertEqual(self.request.wantsKey(), False) def test_EmailBridgeRequest_withIPv6(self): - """IPv6 requests should have ``addressClass = ipaddr.IPv6Address``.""" - self.assertEqual(self.request.addressClass, ipaddr.IPv4Address) + """IPv6 requests should have ``ipVersion == 6``.""" + self.assertEqual(self.request.ipVersion, 4) self.request.withIPv6() - self.assertEqual(self.request.addressClass, ipaddr.IPv6Address) + self.assertEqual(self.request.ipVersion, 6) def test_EmailBridgeRequest_withoutBlockInCountry_CN(self): """Country codes that aren't lowercase should be ignored.""" diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 2cddc11..500bbb8 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -273,7 +273,7 @@ class DummyBridge(object): line = [] if bridgeRequest.transports: line.append(bridgeRequest.transports[-1]) # Just the last PT - if bridgeRequest.addressClass is ipaddr.IPv6Address: + if bridgeRequest.ipVersion is 6: line.append("[%s]:%s" % self.orAddresses[0][:2]) else: line.append("%s:%s" % (self.address, self.orPort))

1 0

[bridgedb/develop] Add a utility for registering interface adapter classes.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit ab5c6b01c2c0dc461d45899105a0e6892e682ef7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 11 02:01:45 2015 +0000 Add a utility for registering interface adapter classes. --- lib/bridgedb/util.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index 7f906d7..f15e08b 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -19,6 +19,8 @@ import logging.config import logging.handlers import os +from twisted.python import components + def _getLogHandlers(logToFile=True, logToStderr=True): """Get the appropriate list of log handlers. @@ -245,6 +247,20 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"): return str(escaped) +def registerAdapter(adapter, adapted, interface): + """Register a Zope interface adapter for global use. + + See :api:`twisted.python.components.registerAdapter` and the Twisted + Matrix Labs `howto documentation for components`_. + + .. howto documentation for components: + https://twistedmatrix.com/documents/current/core/howto/components.html + """ + try: + components.registerAdapter(adapter, adapted, interface) + except ValueError: # An adapter class was already registered + pass + class JustifiedLogFormatter(logging.Formatter): """A logging formatter which pretty prints thread and calling function

1 0

[bridgedb/develop] Move bridgedb.Dist.EmailBasedDistributor → bridgedb.email.distributor.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 55f36f523dae951e2c0e57cdfdc84e3a78ba7cbe Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 21:41:20 2015 +0000 Move bridgedb.Dist.EmailBasedDistributor → bridgedb.email.distributor. --- doc/sphinx/source/bridgedb.Dist.rst | 8 - doc/sphinx/source/bridgedb.email.rst | 1 + doc/sphinx/source/bridgedb.rst | 1 - doc/sphinx/source/conf.py | 2 +- lib/bridgedb/Dist.py | 186 ---------------------- lib/bridgedb/Main.py | 9 +- lib/bridgedb/email/autoresponder.py | 8 +- lib/bridgedb/email/distributor.py | 209 +++++++++++++++++++++++++ lib/bridgedb/email/request.py | 6 +- lib/bridgedb/email/server.py | 4 +- lib/bridgedb/email/templates.py | 2 +- lib/bridgedb/parse/addr.py | 7 +- lib/bridgedb/persistent.py | 4 +- lib/bridgedb/test/email_helpers.py | 14 +- lib/bridgedb/test/legacy_Tests.py | 43 +---- lib/bridgedb/test/test_email_autoresponder.py | 2 +- lib/bridgedb/test/test_email_distributor.py | 93 +++++++++++ lib/bridgedb/test/test_email_server.py | 6 +- 18 files changed, 340 insertions(+), 265 deletions(-) diff --git a/doc/sphinx/source/bridgedb.Dist.rst b/doc/sphinx/source/bridgedb.Dist.rst deleted file mode 100644 index 995a7ac..0000000 --- a/doc/sphinx/source/bridgedb.Dist.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.Dist ----------------- - -.. automodule:: bridgedb.Dist - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.rst b/doc/sphinx/source/bridgedb.email.rst index ba9b4a9..020b614 100644 --- a/doc/sphinx/source/bridgedb.email.rst +++ b/doc/sphinx/source/bridgedb.email.rst @@ -8,6 +8,7 @@ bridgedb.email .. automodule:: bridgedb.email.__init__ .. automodule:: bridgedb.email.autoresponder +.. automodule:: bridgedb.email.distributor .. automodule:: bridgedb.email.dkim .. automodule:: bridgedb.email.request .. automodule:: bridgedb.email.server diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 7b1fef7..207313c 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -13,7 +13,6 @@ BridgeDB Package and Module Documentation bridgedb.captcha bridgedb.configure bridgedb.crypto - bridgedb.Dist bridgedb.email bridgedb.filters bridgedb.geo diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 5038abb..cab614f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -33,9 +33,9 @@ import bridgedb.captcha import bridgedb.Bridges import bridgedb.Bucket import bridgedb.crypto -import bridgedb.Dist import bridgedb.email import bridgedb.email.autoresponder +import bridgedb.email.distributor import bridgedb.email.dkim import bridgedb.email.request import bridgedb.email.server diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py deleted file mode 100644 index 1be705f..0000000 --- a/lib/bridgedb/Dist.py +++ /dev/null @@ -1,186 +0,0 @@ -# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_Dist -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Nick Mathewson -# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> -# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> -# :copyright: (c) 2013-2015, Isis Lovecruft -# (c) 2013-2015, Matthew Finkel -# (c) 2007-2015, The Tor Project, Inc. -# :license: 3-Clause BSD, see LICENSE for licensing information - -"""This module has functions to decide which bridges to hand out to whom.""" - -import logging -import time - -import bridgedb.Storage - -from bridgedb.Bridges import BridgeRing -from bridgedb.Bridges import FilteredBridgeSplitter -from bridgedb.crypto import getHMAC -from bridgedb.crypto import getHMACFunc -from bridgedb.distribute import Distributor -from bridgedb.filters import byFilters -from bridgedb.filters import byIPv4 -from bridgedb.filters import byIPv6 -from bridgedb.filters import bySubring -from bridgedb.parse import addr - - -MAX_EMAIL_RATE = 3*3600 - -class IgnoreEmail(addr.BadEmail): - """Raised when we get requests from this address after rate warning.""" - -class TooSoonEmail(addr.BadEmail): - """Raised when we got a request from this address too recently.""" - -class EmailRequestedHelp(Exception): - """Raised when a client has emailed requesting help.""" - -class EmailRequestedKey(Exception): - """Raised when an incoming email requested a copy of our GnuPG keys.""" - - -class EmailBasedDistributor(Distributor): - """Object that hands out bridges based on the email address of an incoming - request and the current time period. - - :type hashring: :class:`~bridgedb.Bridges.BridgeRing` - :ivar hashring: A hashring to hold all the bridges we hand out. - """ - - def __init__(self, key, domainmap, domainrules, - answerParameters=None, whitelist=None): - """Create a bridge distributor which uses email. - - :type emailHmac: callable - :param emailHmac: An hmac function used to order email addresses - within a ring. See :func:`~bridgedb.crypto.getHMACFunc`. - :param dict domainmap: A map from lowercase domains that we support - mail from to their canonical forms. See `EMAIL_DOMAIN_MAP` option - in `bridgedb.conf`. - :param domainrules: DOCDOC - :param answerParameters: DOCDOC - :type whitelist: dict or ``None`` - :param whitelist: A dictionary that maps whitelisted email addresses - to GnuPG fingerprints. - """ - super(EmailBasedDistributor, self).__init__(key) - - key1 = getHMAC(key, "Map-Addresses-To-Ring") - self.emailHmac = getHMACFunc(key1, hex=False) - - key2 = getHMAC(key, "Order-Bridges-In-Ring") - # XXXX clear the store when the period rolls over! - self.domainmap = domainmap - self.domainrules = domainrules - self.whitelist = whitelist or dict() - self.answerParameters = answerParameters - - #XXX cache options not implemented - self.hashring = FilteredBridgeSplitter(key2, max_cached_rings=5) - self.name = "Email" - - def bridgesPerResponse(self, hashring=None): - return super(EmailBasedDistributor, self).bridgesPerResponse(hashring) - - def insert(self, bridge): - """Assign a bridge to this distributor.""" - self.hashring.insert(bridge) - - def getBridges(self, bridgeRequest, interval): - """Return a list of bridges to give to a user. - - :type bridgeRequest: :class:`~bridgedb.email.request.EmailBridgeRequest` - :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` - with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` - attribute set to a string containing the client's full, canonicalized - email address. - :param interval: The time period when we got this request. This can be - any string, so long as it changes with every period. - """ - # All checks on the email address, such as checks for whitelisting and - # canonicalization of domain name, are done in - # :meth:`bridgedb.email.autoresponder.getMailTo` and - # :meth:`bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. - if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): - raise addr.BadEmail( - ("%s distributor can't get bridges for invalid email email " - " address: %s") % (self.name, bridgeRequest.client)) - - logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) - - now = time.time() - - with bridgedb.Storage.getDB() as db: - wasWarned = db.getWarnedEmail(bridgeRequest.client) - lastSaw = db.getEmailTime(bridgeRequest.client) - if lastSaw is not None: - if bridgeRequest.client in self.whitelist.keys(): - logging.info(("Whitelisted email address %s was last seen " - "%d seconds ago.") - % (bridgeRequest.client, now - lastSaw)) - elif (lastSaw + MAX_EMAIL_RATE) >= now: - wait = (lastSaw + MAX_EMAIL_RATE) - now - logging.info("Client %s must wait another %d seconds." - % (bridgeRequest.client, wait)) - if wasWarned: - raise IgnoreEmail("Client was warned.", - bridgeRequest.client) - else: - logging.info("Sending duplicate request warning.") - db.setWarnedEmail(bridgeRequest.client, True, now) - db.commit() - raise TooSoonEmail("Must wait %d seconds" % wait, - bridgeRequest.client) - # warning period is over - elif wasWarned: - db.setWarnedEmail(bridgeRequest.client, False) - - pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) - - ring = None - ruleset = frozenset(bridgeRequest.filters) - if ruleset in self.hashring.filterRings.keys(): - logging.debug("Cache hit %s" % ruleset) - _, ring = self.hashring.filterRings[ruleset] - else: - # cache miss, add new ring - logging.debug("Cache miss %s" % ruleset) - - # add new ring - key1 = getHMAC(self.key, "Order-Bridges-In-Ring") - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, byFilters(ruleset), - populate_from=self.hashring.bridges) - - returnNum = self.bridgesPerResponse(ring) - result = ring.getBridges(pos, returnNum) - - db.setEmailTime(bridgeRequest.client, now) - db.commit() - - return result - - def cleanDatabase(self): - with bridgedb.Storage.getDB() as db: - try: - db.cleanEmailedBridges(time.time() - MAX_EMAIL_RATE) - db.cleanWarnedEmails(time.time() - MAX_EMAIL_RATE) - except: - db.rollback() - raise - else: - db.commit() - - def prepopulateRings(self): - # populate all rings (for dumping assignments and testing) - for filterFn in [byIPv4, byIPv6]: - ruleset = frozenset([filterFn]) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring") - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, byFilters([filterFn]), - populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 3a15145..b281d21 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -30,13 +30,13 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig +from bridgedb.email.distributor import EmailDistributor from bridgedb.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors import bridgedb.Storage from bridgedb import Bridges -from bridgedb import Dist from bridgedb.Stability import updateBridgeHistory @@ -197,7 +197,7 @@ def createBridgeRings(cfg, proxyList, key): :rtype: tuple :returns: A BridgeSplitter hashring, an :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an - EmailBasedDistributor or None. + :class:`~bridgedb.email.distributor.EmailDistributor` or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -222,7 +222,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an email-based distributor. if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: logging.debug("Setting up Email Distributor...") - emailDistributor = Dist.EmailBasedDistributor( + emailDistributor = EmailDistributor( crypto.getHMAC(key, "Email-Dist-Key"), cfg.EMAIL_DOMAIN_MAP.copy(), cfg.EMAIL_DOMAIN_RULES.copy(), @@ -333,7 +333,8 @@ def run(options, reactor=reactor): known open proxies. :ivar ipDistributor: A :class:`~bridgedb.https.distributor.HTTPSDistributor`. - :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. + :ivar emailDistributor: A + :class:`~bridgedb.email.distributor.EmailDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some scheduled event, repetitive or otherwise, for the :class:`reactor diff --git a/lib/bridgedb/email/autoresponder.py b/lib/bridgedb/email/autoresponder.py index b98717f..ad63bfd 100644 --- a/lib/bridgedb/email/autoresponder.py +++ b/lib/bridgedb/email/autoresponder.py @@ -47,13 +47,13 @@ from twisted.python import failure from bridgedb import safelog from bridgedb.crypto import NEW_BUFFER_INTERFACE -from bridgedb.Dist import EmailRequestedHelp -from bridgedb.Dist import EmailRequestedKey -from bridgedb.Dist import TooSoonEmail -from bridgedb.Dist import IgnoreEmail from bridgedb.email import dkim from bridgedb.email import request from bridgedb.email import templates +from bridgedb.email.distributor import EmailRequestedHelp +from bridgedb.email.distributor import EmailRequestedKey +from bridgedb.email.distributor import TooSoonEmail +from bridgedb.email.distributor import IgnoreEmail from bridgedb.parse import addr from bridgedb.parse.addr import canonicalizeEmailDomain from bridgedb.util import levenshteinDistance diff --git a/lib/bridgedb/email/distributor.py b/lib/bridgedb/email/distributor.py new file mode 100644 index 0000000..b73c082 --- /dev/null +++ b/lib/bridgedb/email/distributor.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""A :class:`~bridgedb.distribute.Distributor` which hands out +:class:`bridges <bridgedb.bridges.Bridge>` to clients via an email interface. +""" + +import logging +import time + +import bridgedb.Storage + +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import FilteredBridgeSplitter +from bridgedb.crypto import getHMAC +from bridgedb.crypto import getHMACFunc +from bridgedb.distribute import Distributor +from bridgedb.filters import byFilters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import bySubring +from bridgedb.parse import addr + + +#: The minimum amount of time (in seconds) which must pass before a client who +#: has previously been given an email response must wait before being eligible +#: to receive another response. +MAX_EMAIL_RATE = 3 * 3600 + + +class IgnoreEmail(addr.BadEmail): + """Raised when we get requests from this address after rate warning.""" + + +class TooSoonEmail(addr.BadEmail): + """Raised when we got a request from this address too recently.""" + + +class EmailRequestedHelp(Exception): + """Raised when a client has emailed requesting help.""" + + +class EmailRequestedKey(Exception): + """Raised when an incoming email requested a copy of our GnuPG keys.""" + + +class EmailDistributor(Distributor): + """Object that hands out bridges based on the email address of an incoming + request and the current time period. + + :type hashring: :class:`~bridgedb.Bridges.BridgeRing` + :ivar hashring: A hashring to hold all the bridges we hand out. + """ + + #: The minimum amount of time (in seconds) which must pass before a client + #: who has previously been given an email response must wait before being + #: eligible to receive another response. + emailRateMax = MAX_EMAIL_RATE + + def __init__(self, key, domainmap, domainrules, + answerParameters=None, whitelist=None): + """Create a bridge distributor which uses email. + + :type emailHmac: callable + :param emailHmac: An hmac function used to order email addresses + within a ring. See :func:`~bridgedb.crypto.getHMACFunc`. + :param dict domainmap: A map from lowercase domains that we support + mail from to their canonical forms. See `EMAIL_DOMAIN_MAP` option + in `bridgedb.conf`. + :param domainrules: DOCDOC + :param answerParameters: DOCDOC + :type whitelist: dict or ``None`` + :param whitelist: A dictionary that maps whitelisted email addresses + to GnuPG fingerprints. + """ + super(EmailDistributor, self).__init__(key) + + self.domainmap = domainmap + self.domainrules = domainrules + self.whitelist = whitelist or dict() + self.answerParameters = answerParameters + + key1 = getHMAC(key, "Map-Addresses-To-Ring") + key2 = getHMAC(key, "Order-Bridges-In-Ring") + + self.emailHmac = getHMACFunc(key1, hex=False) + #XXX cache options not implemented + self.hashring = FilteredBridgeSplitter(key2, max_cached_rings=5) + + self.name = "Email" + + def bridgesPerResponse(self, hashring=None): + return super(EmailDistributor, self).bridgesPerResponse(hashring) + + def getBridges(self, bridgeRequest, interval): + """Return a list of bridges to give to a user. + + .. hint:: All checks on the email address (which should be stored in + the ``bridgeRequest.client`` attribute), such as checks for + whitelisting and canonicalization of domain name, are done in + :meth:`bridgedb.email.autoresponder.getMailTo` and + :meth:`bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + + :type bridgeRequest: + :class:`~bridgedb.email.request.EmailBridgeRequest` + :param bridgeRequest: A + :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the + :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute + set to a string containing the client's full, canonicalized email + address. + :param interval: The time period when we got this request. This can be + any string, so long as it changes with every period. + """ + if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): + raise addr.BadEmail( + ("%s distributor can't get bridges for invalid email address: " + "%s") % (self.name, bridgeRequest.client), bridgeRequest.client) + + logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) + + now = time.time() + + with bridgedb.Storage.getDB() as db: + wasWarned = db.getWarnedEmail(bridgeRequest.client) + lastSaw = db.getEmailTime(bridgeRequest.client) + if lastSaw is not None: + if bridgeRequest.client in self.whitelist: + logging.info( + "Whitelisted address %s was last seen %d seconds ago." + % (bridgeRequest.client, now - lastSaw)) + elif (lastSaw + self.emailRateMax) >= now: + wait = (lastSaw + self.emailRateMax) - now + logging.info("Client %s must wait another %d seconds." + % (bridgeRequest.client, wait)) + if wasWarned: + raise IgnoreEmail( + "Client %s was warned." % bridgeRequest.client, + bridgeRequest.client) + else: + logging.info("Sending duplicate request warning.") + db.setWarnedEmail(bridgeRequest.client, True, now) + db.commit() + raise TooSoonEmail("Must wait %d seconds" % wait, + bridgeRequest.client) + # warning period is over + elif wasWarned: + db.setWarnedEmail(bridgeRequest.client, False) + + pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) + + ring = None + filtres = frozenset(bridgeRequest.filters) + if filtres in self.hashring.filterRings: + logging.debug("Cache hit %s" % filtres) + _, ring = self.hashring.filterRings[filtres] + else: + logging.debug("Cache miss %s" % filtres) + key = getHMAC(self.key, "Order-Bridges-In-Ring") + ring = BridgeRing(key, self.answerParameters) + self.hashring.addRing(ring, filtres, byFilters(filtres), + populate_from=self.hashring.bridges) + + returnNum = self.bridgesPerResponse(ring) + result = ring.getBridges(pos, returnNum) + + db.setEmailTime(bridgeRequest.client, now) + db.commit() + + return result + + def cleanDatabase(self): + """Clear all emailed response and warning times from the database.""" + logging.info(("Cleaning all response and warning times for the %s " + "distributor from the database...") % self.name) + with bridgedb.Storage.getDB() as db: + try: + db.cleanEmailedBridges(time.time() - self.emailRateMax) + db.cleanWarnedEmails(time.time() - self.emailRateMax) + except: + db.rollback() + raise + else: + db.commit() + + def prepopulateRings(self): + """Prepopulate this distributor's hashrings and subhashrings with + bridges. + """ + logging.info("Prepopulating %s distributor hashrings..." % self.name) + + for filterFn in [byIPv4, byIPv6]: + ruleset = frozenset([filterFn]) + key = getHMAC(self.key, "Order-Bridges-In-Ring") + ring = BridgeRing(key, self.answerParameters) + self.hashring.addRing(ring, ruleset, byFilters([filterFn]), + populate_from=self.hashring.bridges) + + # Since prepopulateRings is called every half hour when the bridge + # descriptors are re-parsed, we should clean the database then. + self.cleanDatabase() diff --git a/lib/bridgedb/email/request.py b/lib/bridgedb/email/request.py index 32446ac..50fd32c 100644 --- a/lib/bridgedb/email/request.py +++ b/lib/bridgedb/email/request.py @@ -40,8 +40,8 @@ import logging import re from bridgedb import bridgerequest -from bridgedb.Dist import EmailRequestedHelp -from bridgedb.Dist import EmailRequestedKey +from bridgedb.email.distributor import EmailRequestedHelp +from bridgedb.email.distributor import EmailRequestedKey #: A regular expression for matching the Pluggable Transport method TYPE in @@ -105,7 +105,7 @@ class EmailBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self): """Process a new bridge request received through the - :class:`~bridgedb.Dist.EmailBasedDistributor`. + :class:`~bridgedb.email.distributor.EmailDistributor`. """ super(EmailBridgeRequest, self).__init__() self._wantsKey = False diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index 8034d35..736c3f6 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -103,7 +103,7 @@ class MailServerContext(object): """Create a context for storing configs for email bridge distribution. :type config: :class:`bridgedb.persistent.Conf` - :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor` + :type distributor: :class:`~bridgedb.email.distributor.EmailDistributor` :param distributor: The distributor will handle getting the correct bridges (or none) for a client for us. :type schedule: :class:`bridgedb.schedule.ScheduledInterval` @@ -466,7 +466,7 @@ def addServer(config, distributor): :type config: :class:`bridgedb.configure.Conf` :param config: A configuration object. - :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor` + :type distributor: :class:`bridgedb.email.distributor.EmailDistributor` :param dist: A distributor which will handle database interactions, and will decide which bridges to give to who and when. """ diff --git a/lib/bridgedb/email/templates.py b/lib/bridgedb/email/templates.py index 4db4e3c..6998716 100644 --- a/lib/bridgedb/email/templates.py +++ b/lib/bridgedb/email/templates.py @@ -30,7 +30,7 @@ import os from datetime import datetime from bridgedb import strings -from bridgedb.Dist import MAX_EMAIL_RATE +from bridgedb.email.distributor import MAX_EMAIL_RATE def addCommands(template): diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index b3ad680..cd512d1 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -217,7 +217,7 @@ def canonicalizeEmailDomain(domain, domainmap): :param str domain: The domain portion of an email address to validate. It will be checked that it is one of the domains allowed to email requests for bridges to the - :class:`~bridgedb.Dist.EmailBasedDistributor`. + :class:`~bridgedb.email.distributor.EmailDistributor`. :param dict domainmap: A map of permitted alternate domains (in lowercase) to their canonical domain names (in lowercase). This can be configured with the ``EMAIL_DOMAIN_MAP`` option in ``bridgedb.conf``, for @@ -441,8 +441,9 @@ def normalizeEmail(emailaddr, domainmap, domainrules, ignorePlus=True): The email address, **emailaddr**, will be parsed and validated, and then checked that it originated from one of the domains allowed to email - requests for bridges to the :class:`~bridgedb.Dist.EmailBasedDistributor` - via the :func:`canonicaliseEmailDomain` function. + requests for bridges to the + :class:`~bridgedb.email.distributor.EmailDistributor` via the + :func:`canonicaliseEmailDomain` function. :param str emailaddr: An email address to normalise. :param dict domainmap: A map of permitted alternate domains (in lowercase) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 0853c1e..22673dd 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -24,8 +24,8 @@ from twisted.python.reflect import safe_repr from twisted.spread import jelly from bridgedb import Bridges -from bridgedb import Dist from bridgedb import filters +from bridgedb.email import distributor as emailDistributor from bridgedb.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -35,7 +35,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(filters, Bridges, Dist, httpsDistributor) +_security.allowModules(filters, Bridges, emailDistributor, httpsDistributor) class MissingState(Exception): diff --git a/lib/bridgedb/test/email_helpers.py b/lib/bridgedb/test/email_helpers.py index 20aee57..14c86f4 100644 --- a/lib/bridgedb/test/email_helpers.py +++ b/lib/bridgedb/test/email_helpers.py @@ -13,9 +13,9 @@ import io -from bridgedb.Dist import IgnoreEmail -from bridgedb.Dist import TooSoonEmail from bridgedb.persistent import Conf +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.email.server import MailServerContext from bridgedb.schedule import Unscheduled from bridgedb.test import util @@ -120,8 +120,8 @@ def _createMailServerContext(config=None, distributor=None): class DummyEmailDistributor(object): - """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which is used to - test :class:`bridgedb.EmailServer`. + """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which is used + to test :class:`bridgedb.EmailServer`. """ _bridgesPerResponseMin = 3 @@ -144,9 +144,9 @@ class DummyEmailDistributor(object): class DummyEmailDistributorWithState(DummyEmailDistributor): - """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which raises - :exc:`bridgedb.Dist.TooSoonEmail` on the second email and - :exc:`bridgedb.Dist.IgnoreEmail` on the third. + """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which raises + :exc:`bridgedb.email.distributor.TooSoonEmail` on the second email and + :exc:`bridgedb.email.distributor.IgnoreEmail` on the third. Note that the state tracking is done in a really dumb way. For example, we currently don't consider requests for help text or GnuPG keys to be a diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index c0a9ff0..40f7ae4 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -19,7 +19,6 @@ from datetime import datetime import bridgedb.Bridges import bridgedb.Main -import bridgedb.Dist import bridgedb.schedule import bridgedb.Storage import re @@ -27,6 +26,9 @@ import ipaddr from bridgedb.Stability import BridgeHistory +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse import addr from bridgedb.test.util import bracketIPv6 from bridgedb.test.util import randomIP @@ -94,42 +96,6 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, return b -class EmailBridgeDistTests(unittest.TestCase): - def setUp(self): - self.fd, self.fname = tempfile.mkstemp() - self.db = bridgedb.Storage.Database(self.fname) - bridgedb.Storage.setDB(self.db) - self.cur = self.db._conn.cursor() - - def tearDown(self): - self.db.close() - os.close(self.fd) - os.unlink(self.fname) - - def testEmailRateLimit(self): - db = self.db - EMAIL_DOMAIN_MAP = {'example.com':'example.com'} - d = bridgedb.Dist.EmailBasedDistributor( - "Foo", - {'example.com': 'example.com', - 'dkim.example.com': 'dkim.example.com'}, - {'example.com': [], 'dkim.example.com': ['dkim']}) - for _ in xrange(256): - d.insert(fakeBridge()) - d.getBridges('abc(a)example.com', 1) - self.assertRaises(bridgedb.Dist.TooSoonEmail, - d.getBridges, 'abc(a)example.com', 1) - self.assertRaises(bridgedb.Dist.IgnoreEmail, - d.getBridges, 'abc(a)example.com', 1) - - def testUnsupportedDomain(self): - db = self.db - self.assertRaises(bridgedb.parse.addr.UnsupportedDomain, - bridgedb.parse.addr.normalizeEmail, - 'bad(a)email.com', - {'example.com':'example.com'}, - {'example.com':[]}) - class SQLStorageTests(unittest.TestCase): def setUp(self): @@ -358,8 +324,7 @@ class BridgeStabilityTests(unittest.TestCase): def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - - for klass in [SQLStorageTests, EmailBridgeDistTests, BridgeStabilityTests]: + for klass in [SQLStorageTests, BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) return suite diff --git a/lib/bridgedb/test/test_email_autoresponder.py b/lib/bridgedb/test/test_email_autoresponder.py index 2895802..98302ca 100644 --- a/lib/bridgedb/test/test_email_autoresponder.py +++ b/lib/bridgedb/test/test_email_autoresponder.py @@ -25,7 +25,7 @@ from twisted.test import proto_helpers from bridgedb.email import autoresponder from bridgedb.email.server import SMTPMessage -from bridgedb.Dist import TooSoonEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.test.email_helpers import _createConfig from bridgedb.test.email_helpers import _createMailServerContext from bridgedb.test.email_helpers import DummyEmailDistributorWithState diff --git a/lib/bridgedb/test/test_email_distributor.py b/lib/bridgedb/test/test_email_distributor.py new file mode 100644 index 0000000..96eb306 --- /dev/null +++ b/lib/bridgedb/test/test_email_distributor.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# :license: see included LICENSE for information + +"""Tests for :mod:`bridgedb.email.distributor`.""" + +from __future__ import print_function + +import logging +import tempfile +import os + +from twisted.trial import unittest + +import bridgedb.Storage + +from bridgedb.bridges import Bridge +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail +from bridgedb.email.request import EmailBridgeRequest +from bridgedb.parse.addr import UnsupportedDomain +from bridgedb.parse.addr import normalizeEmail +from bridgedb.test.util import generateFakeBridges + +logging.disable(50) + + +BRIDGES = generateFakeBridges() + + +class EmailDistributorTests(unittest.TestCase): + """Tests for :class:`bridgedb.email.distributor.EmailDistributor`.""" + + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + bridgedb.Storage.initializeDBLock() + self.db = bridgedb.Storage.openDatabase(self.fname) + bridgedb.Storage.setDBFilename(self.fname) + self.cur = self.db.cursor() + self.db.close() + + self.bridges = BRIDGES + self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' + self.domainmap = { + 'example.com': 'example.com', + 'dkim.example.com': 'dkim.example.com', + } + self.domainrules = { + 'example.com': ['ignore_dots'], + 'dkim.example.com': ['dkim', 'ignore_dots'] + } + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def makeClientRequest(self, clientEmailAddress): + bridgeRequest = EmailBridgeRequest() + bridgeRequest.client = clientEmailAddress + bridgeRequest.isValid(True) + bridgeRequest.generateFilters() + return bridgeRequest + + def test_EmailDistributor_rate_limit(self): + """A client's first email should return bridges. The second should + return a warning, and the third should receive no response. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges + bridges = dist.getBridges(bridgeRequest, 1) + self.assertGreater(len(bridges), 0) + [self.assertIsInstance(b, Bridge) for b in bridges] + self.assertEqual(len(bridges), 3) + + # The second gets a warning, and the third is ignored + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_unsupported_domain(self): + """An unsupported domain should raise an UnsupportedDomain exception.""" + self.assertRaises(UnsupportedDomain, normalizeEmail, + 'bad(a)email.com', self.domainmap, self.domainrules) diff --git a/lib/bridgedb/test/test_email_server.py b/lib/bridgedb/test/test_email_server.py index 561c14c..9c4fabb 100644 --- a/lib/bridgedb/test/test_email_server.py +++ b/lib/bridgedb/test/test_email_server.py @@ -31,9 +31,9 @@ from twisted.trial import unittest from zope.interface import implementedBy -from bridgedb.Dist import EmailBasedDistributor -from bridgedb.Dist import TooSoonEmail from bridgedb.email import server +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse.addr import BadEmail from bridgedb.schedule import Unscheduled from bridgedb.test import util @@ -504,7 +504,7 @@ class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase): """Unittests for :func:`bridgedb.email.server.addServer`.""" def setUp(self): - """Create a server.MailServerContext and EmailBasedDistributor.""" + """Create a MailServerContext and EmailDistributor.""" self.config = _createConfig() self.context = _createMailServerContext(self.config) self.smtpFromAddr = self.context.smtpFromAddr # 'bridges@localhost'

1 0

[bridgedb/develop] Make the @port property common to both PluggableTransports and Bridges.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 63bc3924096d0cde3373e37d85ca1101e2fb438e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jun 9 00:08:04 2015 +0000 Make the @port property common to both PluggableTransports and Bridges. Because it seems awkward to require doing type checks in order to know which port a bridge-like thing is listening on. That is, previously, we could access PluggableTransport.port and Bridge.orPort, but *not* Bridge.port. Moving the port @property from the PluggableTransport class to the BridgeAddressBase class (and updating the Bridge.orPort @property to point at the port @property) allows us to access the port property of either one without caring which type it is. In addition to allowing us to forego type checks, this allow allows us to define an interface which essentially says that "a bridge-like thing is a thing which has a fingerprint, address, and a port", which again allows us to generalise code to deal with "bridge-like" things, rather than dealing with a tuple of classes which happen to be considered "bridge-like" at the time of writing. * MOVE the port @property from the PluggableTransport class to the BridgeAddressBase class. * CHANGE the Bridge.orPort @property to point at the new port @property. --- lib/bridgedb/bridges.py | 65 +++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index f56b0e7..5093bef 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -141,12 +141,17 @@ class BridgeAddressBase(object): :type country: str :ivar country: The two-letter GeoIP country code of the :ivar:`address`. + + :type port: int + :ivar port: A integer specifying the port which this :class:`Bridge` + (or :class:`PluggableTransport`) is listening on. """ def __init__(self): self._fingerprint = None self._address = None self._country = None + self._port = None @property def fingerprint(self): @@ -242,6 +247,32 @@ class BridgeAddressBase(object): if self.address: return geo.getCountryCode(self.address) + @property + def port(self): + """Get the port number which this ``Bridge`` is listening + for incoming client connections on. + + :rtype: int or None + :returns: The port (as an int), if it is known and valid; otherwise, + returns ``None``. + """ + return self._port + + @port.setter + def port(self, value): + """Store the port number which this ``Bridge`` is listening + for incoming client connections on. + + :param int value: The transport's port. + """ + if isinstance(value, int) and (0 <= value <= 65535): + self._port = value + + @port.deleter + def port(self): + """Reset this ``Bridge``'s port to ``None``.""" + self._port = None + class PluggableTransport(BridgeAddressBase): """A single instance of a Pluggable Transport (PT) offered by a @@ -314,7 +345,6 @@ class PluggableTransport(BridgeAddressBase): :data:`arguments`. """ super(PluggableTransport, self).__init__() - self._port = None self._methodname = None self._blockedIn = {} @@ -438,32 +468,6 @@ class PluggableTransport(BridgeAddressBase): return False @property - def port(self): - """Get the port number which this ``PluggableTransport`` is listening - for incoming client connections on. - - :rtype: int or None - :returns: The port (as an int), if it is known and valid; otherwise, - returns ``None``. - """ - return self._port - - @port.setter - def port(self, value): - """Store the port number which this ``PluggableTransport`` is listening - for incoming client connections on. - - :param int value: The transport's port. - """ - if isinstance(value, int) and (0 <= value <= 65535): - self._port = value - - @port.deleter - def port(self): - """Reset this ``PluggableTransport``'s port to ``None``.""" - self._port = None - - @property def methodname(self): """Get this :class:`PluggableTransport`'s methodname. @@ -632,7 +636,7 @@ class BridgeBase(BridgeAddressBase): :rtype: int :returns: This Bridge's default ORPort. """ - return self._orPort + return self.port @orPort.setter def orPort(self, value): @@ -640,13 +644,12 @@ class BridgeBase(BridgeAddressBase): :param int value: The Bridge's ORPort. """ - if isinstance(value, int) and (0 <= value <= 65535): - self._orPort = value + self.port = value @orPort.deleter def orPort(self): """Reset this Bridge's ORPort.""" - self._orPort = None + del self.port class BridgeBackwardsCompatibility(BridgeBase):

1 0

[bridgedb/develop] Switch to using relative imports for test helpers in test/tests_*.py.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 6d309cae1e709e615b592690f1a34369f60fa709 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jun 25 05:47:04 2015 +0000 Switch to using relative imports for test helpers in test/tests_*.py. --- bridgedb/util.py | 48 +++++++++++++++++++++----------------- test/email_helpers.py | 3 ++- test/https_helpers.py | 3 ++- test/legacy_Tests.py | 20 +++++++++------- test/test_Tests.py | 37 +++++++++++++++-------------- test/test_bridgedb.py | 4 ++-- test/test_bridges.py | 6 ++--- test/test_crypto.py | 5 ++-- test/test_email_autoresponder.py | 7 +++--- test/test_email_distributor.py | 3 ++- test/test_email_server.py | 7 +++--- test/test_https.py | 4 ++-- test/test_https_distributor.py | 7 +++--- test/test_https_server.py | 11 +++++---- test/test_parse_descriptors.py | 2 +- test/test_smtp.py | 4 ++-- test/test_translations.py | 2 +- test/util.py | 8 +++---- 18 files changed, 98 insertions(+), 83 deletions(-) diff --git a/bridgedb/util.py b/bridgedb/util.py index f15e08b..4c558c4 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -153,6 +153,7 @@ def levenshteinDistance(s1, s2, len1=None, len2=None, the number of characters which must be changed in **s1** to make it identical to **s2**. + >>> from bridgedb.util import levenshteinDistance >>> levenshteinDistance('cat', 'cat') 0 >>> levenshteinDistance('cat', 'hat') @@ -188,6 +189,7 @@ def isascii(s): Note that this function differs from the str.is* methods in that it returns True for the empty string, rather than False. + >>> from bridgedb.util import isascii >>> isascii('\x80') False >>> isascii('foo\tbar\rbaz\n') @@ -206,6 +208,7 @@ def isascii_noncontrol(s): Note that this function differs from the str.is* methods in that it returns True for the empty string, rather than False. + >>> from bridgedb.util import isascii_noncontrol >>> isascii_noncontrol('\x80') False >>> isascii_noncontrol('foo\tbar\rbaz\n') @@ -220,6 +223,7 @@ def isascii_noncontrol(s): def replaceControlChars(text, replacement=None, encoding="utf-8"): """Remove ASCII control characters [0-31, 92, 127]. + >>> from bridgedb.util import replaceControlChars >>> replaceControlChars('foo\n bar\\ baz\r \t\0quux\n') 'foo bar baz quux' >>> replaceControlChars("\bI wonder if I'm outside the quotes now") @@ -344,36 +348,36 @@ class mixin: >>> from bridgedb.util import mixin >>> >>> class ClassA(object): - >>> def sayWhich(self): - >>> print("ClassA.sayWhich() called.") - >>> def doSuperThing(self): - >>> super(ClassA, self).__repr__() - >>> def doThing(self): - >>> print("ClassA is doing a thing.") - >>> + ... def sayWhich(self): + ... print("ClassA.sayWhich() called.") + ... def doSuperThing(self): + ... print("%s" % super(ClassA, self)) + ... def doThing(self): + ... print("ClassA is doing a thing.") + ... >>> class ClassB(ClassA): - >>> def sayWhich(self): - >>> print("ClassB.sayWhich() called.") - >>> def doSuperThing(self): - >>> super(ClassB, self).__repr__() - >>> def doOtherThing(self): - >>> print("ClassB is doing something else.") - >>> + ... def sayWhich(self): + ... print("ClassB.sayWhich() called.") + ... def doSuperThing(self): + ... print("%s" % super(ClassB, self)) + ... def doOtherThing(self): + ... print("ClassB is doing something else.") + ... >>> class ClassM(mixin): - >>> def sayWhich(self): - >>> print("ClassM.sayWhich() called.") - >>> + ... def sayWhich(self): + ... print("ClassM.sayWhich() called.") + ... >>> ClassM.register(ClassA) >>> >>> class ClassC(ClassM, ClassB): - >>> def sayWhich(self): - >>> super(ClassC, self).sayWhich() - >>> + ... def sayWhich(self): + ... super(ClassC, self).sayWhich() + ... >>> c = ClassC() >>> c.sayWhich() - ClassM.saywhich() called. + ClassM.sayWhich() called. >>> c.doSuperThing() - <super: <class 'ClassA'>, NULL> + <super: <class 'ClassB'>, <ClassC object>> >>> c.doThing() ClassA is doing a thing. >>> c.doOtherThing() diff --git a/test/email_helpers.py b/test/email_helpers.py index 14c86f4..8d7dd49 100644 --- a/test/email_helpers.py +++ b/test/email_helpers.py @@ -18,7 +18,8 @@ from bridgedb.email.distributor import IgnoreEmail from bridgedb.email.distributor import TooSoonEmail from bridgedb.email.server import MailServerContext from bridgedb.schedule import Unscheduled -from bridgedb.test import util + +from . import util EMAIL_DIST = True diff --git a/test/https_helpers.py b/test/https_helpers.py index e2c94ba..3fb4887 100644 --- a/test/https_helpers.py +++ b/test/https_helpers.py @@ -15,9 +15,10 @@ import io from twisted.web.test import requesthelper -from bridgedb.test import util from bridgedb.persistent import Conf +from . import util + SERVER_PUBLIC_FQDN = 'bridges.torproject.org' SERVER_PUBLIC_EXTERNAL_IP = '38.229.72.19' diff --git a/test/legacy_Tests.py b/test/legacy_Tests.py index 40f7ae4..22b2e13 100644 --- a/test/legacy_Tests.py +++ b/test/legacy_Tests.py @@ -30,19 +30,21 @@ from bridgedb.email.distributor import EmailDistributor from bridgedb.email.distributor import IgnoreEmail from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse import addr -from bridgedb.test.util import bracketIPv6 -from bridgedb.test.util import randomIP -from bridgedb.test.util import randomIPv4 -from bridgedb.test.util import randomIPv6 -from bridgedb.test.util import randomIPString -from bridgedb.test.util import randomIPv4String -from bridgedb.test.util import randomIPv6String -from bridgedb.test.util import randomPort -from bridgedb.test.util import randomValidIPv6 + +from .util import bracketIPv6 +from .util import randomIP +from .util import randomIPv4 +from .util import randomIPv6 +from .util import randomIPString +from .util import randomIPv4String +from .util import randomIPv6String +from .util import randomPort +from .util import randomValidIPv6 from math import log warnings.filterwarnings('ignore', '.*tmpnam.*') +warnings.filterwarnings('ignore', '.*Config.*') def randomPortSpec(): diff --git a/test/test_Tests.py b/test/test_Tests.py index 84c3f8b..9209245 100644 --- a/test/test_Tests.py +++ b/test/test_Tests.py @@ -4,13 +4,13 @@ # # :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> # please also see AUTHORS file -# :copyright: (c) 2013, Isis Lovecruft -# (c) 2007-2013, The Tor Project, Inc. -# (c) 2007-2013, all entities within the AUTHORS file +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Class wrappers to adapt BridgeDB old unittests in :mod:`bridgedb.Tests` -(now kept in :mod:`bridgedb.test.legacy_Tests`) to be compatible with the +"""Class wrappers to adapt BridgeDB old unittests in ``bridgedb.Tests`` +(now kept in ``test/legacy_Tests``) to be compatible with the newer :api:`twisted.trial` unittests in this directory. """ @@ -22,16 +22,17 @@ import doctest import glob import logging import os -import warnings from twisted.python import monkey from twisted.trial import unittest -from bridgedb.test import legacy_Tests as Tests -from bridgedb.test import deprecated +from . import legacy_Tests as Tests +from . import deprecated + + +logging.disable(50) -warnings.filterwarnings('ignore', module="bridgedb\.test\.legacy_Tests") pyunit = __import__('unittest') @@ -64,16 +65,16 @@ def generateTrialAdaptedDoctestsSuite(): def monkeypatchTests(): """Monkeypatch the old unittests, replacing new, refactored code with their - original equivalents from :mod:`bridgedb.test.deprecated`. + original equivalents from :mod:`deprecated`. The first patch replaces the newer parsing function, :func:`~bridgedb.parse.networkstatus.parseALine`, with the older, - :func:`deprecated one <bridgedb.test.deprecated.parseORAddressLine>` (the + :func:`deprecated one <deprecated.parseORAddressLine>` (the old function was previously located at ``bridgedb.Bridges.parseORAddressLine``). The second patch replaces the new :class:`~bridgedb.parse.addr.PortList`, - with the :class:`older one <bridgedb.test.deprecated.PortList>` (which + with the :class:`older one <deprecated.PortList>` (which was previously located at ``bridgedb.Bridges.PortList``). The third, forth, and fifth monkeypatches add some module-level attributes @@ -81,7 +82,7 @@ def monkeypatchTests(): :rtype: :api:`~twisted.python.monkey.MonkeyPatcher` :returns: A :api:`~twisted.python.monkey.MonkeyPatcher`, preloaded with - patches from :mod:`bridgedb.test.deprecated`. + patches from :mod:`deprecated`. """ patcher = monkey.MonkeyPatcher() patcher.addPatch(Tests.bridgedb.Bridges, 'PluggableTransport', @@ -189,14 +190,14 @@ class DynamicTestCaseMeta(type): class OldUnittests(unittest.TestCase): - """A wrapper around :mod:`bridgedb.Tests` to produce :api:`~twisted.trial` + """A wrapper around :mod:`legacy_Tests` to produce :api:`~twisted.trial` compatible output. Generates a :api:`twisted.trial.unittest.TestCase` containing a - test for each of the individual tests in :mod:`bridgedb.Tests`. + test for each of the individual tests in :mod:`legacy_Tests`. Each test in this :api:`~twisted.trial.unittest.TestCase`` is dynamically - generated from one of the old unittests in :mod:`bridgedb.Tests`. Then, + generated from one of the old unittests in :mod:`legacy_Tests`. Then, the class is wrapped to cause the results reporting mechanisms to be :api:`~twisted.trial` compatible. @@ -209,13 +210,13 @@ class OldUnittests(unittest.TestCase): class MonkeypatchedOldUnittests(unittest.TestCase): - """A wrapper around :mod:`bridgedb.Tests` to produce :api:`~twisted.trial` + """A wrapper around :mod:`legacy_Tests` to produce :api:`~twisted.trial` compatible output. For each test in this ``TestCase``, one of the old unittests in bridgedb/Tests.py is run. For all of the tests, some functions and classes are :api:`twisted.python.monkey.MonkeyPatcher.patch`ed with old, - deprecated code from :mod:`bridgedb.test.deprecated` to ensure that any + deprecated code from :mod:`deprecated` to ensure that any new code has not caused any regressions. """ __metaclass__ = DynamicTestCaseMeta diff --git a/test/test_bridgedb.py b/test/test_bridgedb.py index 70c71f0..00c231b 100644 --- a/test/test_bridgedb.py +++ b/test/test_bridgedb.py @@ -21,8 +21,8 @@ from twisted.trial import unittest from twisted.trial.unittest import FailTest from twisted.trial.unittest import SkipTest -from bridgedb.test.util import processExists -from bridgedb.test.util import getBridgeDBPID +from .util import processExists +from .util import getBridgeDBPID class BridgeDBCliTest(unittest.TestCase): diff --git a/test/test_bridges.py b/test/test_bridges.py index db662ae..9448e3f 100644 --- a/test/test_bridges.py +++ b/test/test_bridges.py @@ -148,10 +148,10 @@ class BridgeIntegrationTests(unittest.TestCase): .. data: OldTest (enum) These tests were refactored from the old tests for - :class:`~bridgedb.test.deprecated.Bridge`, which lived in + ``deprecated.Bridge`, which lived in ``lib/bridgedb/test/test_Bridges.py``. For the translations from the old - tests in ``bridgedb.test.test_Bridges.BridgeClassTest`` to their new - equivalents here in ``bridgedb.test.test_bridges.BridgeIntegrationTests``, + tests in ``test_Bridges.BridgeClassTest`` to their new + equivalents here in ``test_bridges.BridgeIntegrationTests``, which should test for the same things as their old equivalents, see the following table: diff --git a/test/test_crypto.py b/test/test_crypto.py index 3264ace..5220a58 100644 --- a/test/test_crypto.py +++ b/test/test_crypto.py @@ -33,8 +33,9 @@ from twisted.web.test import test_agent as txtagent from bridgedb import crypto from bridgedb import txrecaptcha from bridgedb.persistent import Conf -from bridgedb.test.util import fileCheckDecorator -from bridgedb.test.email_helpers import _createConfig + +from .util import fileCheckDecorator +from .email_helpers import _createConfig logging.disable(50) diff --git a/test/test_email_autoresponder.py b/test/test_email_autoresponder.py index 98302ca..77fb77a 100644 --- a/test/test_email_autoresponder.py +++ b/test/test_email_autoresponder.py @@ -26,9 +26,10 @@ from twisted.test import proto_helpers from bridgedb.email import autoresponder from bridgedb.email.server import SMTPMessage from bridgedb.email.distributor import TooSoonEmail -from bridgedb.test.email_helpers import _createConfig -from bridgedb.test.email_helpers import _createMailServerContext -from bridgedb.test.email_helpers import DummyEmailDistributorWithState + +from .email_helpers import _createConfig +from .email_helpers import _createMailServerContext +from .email_helpers import DummyEmailDistributorWithState class CreateResponseBodyTests(unittest.TestCase): diff --git a/test/test_email_distributor.py b/test/test_email_distributor.py index b4d88b2..f204d57 100644 --- a/test/test_email_distributor.py +++ b/test/test_email_distributor.py @@ -28,7 +28,8 @@ from bridgedb.email.request import EmailBridgeRequest from bridgedb.parse.addr import BadEmail from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import normalizeEmail -from bridgedb.test.util import generateFakeBridges + +from .util import generateFakeBridges logging.disable(50) diff --git a/test/test_email_server.py b/test/test_email_server.py index 9c4fabb..3a3d62d 100644 --- a/test/test_email_server.py +++ b/test/test_email_server.py @@ -36,9 +36,10 @@ from bridgedb.email.distributor import EmailDistributor from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse.addr import BadEmail from bridgedb.schedule import Unscheduled -from bridgedb.test import util -from bridgedb.test.email_helpers import _createConfig -from bridgedb.test.email_helpers import _createMailServerContext + +from . import util +from .email_helpers import _createConfig +from .email_helpers import _createMailServerContext class SMTPMessageTests(unittest.TestCase): diff --git a/test/test_https.py b/test/test_https.py index 1e0c778..8a5754f 100644 --- a/test/test_https.py +++ b/test/test_https.py @@ -35,8 +35,8 @@ from twisted.trial import unittest from twisted.trial.unittest import FailTest from twisted.trial.unittest import SkipTest -from bridgedb.test.util import processExists -from bridgedb.test.util import getBridgeDBPID +from .util import processExists +from .util import getBridgeDBPID HTTP_ROOT = 'http://127.0.0.1:6788' CAPTCHA_RESPONSE = 'Tvx74Pmy' diff --git a/test/test_https_distributor.py b/test/test_https_distributor.py index 83f2503..3c1416d 100644 --- a/test/test_https_distributor.py +++ b/test/test_https_distributor.py @@ -26,9 +26,10 @@ from bridgedb.filters import byIPv6 from bridgedb.https import distributor from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet -from bridgedb.test.util import randomValidIPv4String -from bridgedb.test.util import generateFakeBridges -from bridgedb.test.https_helpers import DummyRequest + +from .util import randomValidIPv4String +from .util import generateFakeBridges +from .https_helpers import DummyRequest logging.disable(50) diff --git a/test/test_https_server.py b/test/test_https_server.py index e782135..708f7e3 100644 --- a/test/test_https_server.py +++ b/test/test_https_server.py @@ -29,11 +29,12 @@ from twisted.web.test import requesthelper from bridgedb.https import server from bridgedb.schedule import ScheduledInterval -from bridgedb.test.https_helpers import _createConfig -from bridgedb.test.https_helpers import DummyRequest -from bridgedb.test.https_helpers import DummyHTTPSDistributor -from bridgedb.test.util import DummyBridge -from bridgedb.test.util import DummyMaliciousBridge + +from .https_helpers import _createConfig +from .https_helpers import DummyRequest +from .https_helpers import DummyHTTPSDistributor +from .util import DummyBridge +from .util import DummyMaliciousBridge # For additional logger output for debugging, comment out the following: diff --git a/test/test_parse_descriptors.py b/test/test_parse_descriptors.py index dd6d146..5104ccd 100644 --- a/test/test_parse_descriptors.py +++ b/test/test_parse_descriptors.py @@ -33,7 +33,7 @@ except (ImportError, NameError), error: else: HAS_STEM = True -from bridgedb.test.util import Benchmarker +from .util import Benchmarker BRIDGE_NETWORKSTATUS_0 = '''\ diff --git a/test/test_smtp.py b/test/test_smtp.py index de443b3..0ac4ada 100644 --- a/test/test_smtp.py +++ b/test/test_smtp.py @@ -15,8 +15,8 @@ from twisted.trial import unittest from twisted.trial.unittest import FailTest from twisted.trial.unittest import SkipTest -from bridgedb.test.util import processExists -from bridgedb.test.util import getBridgeDBPID +from .util import processExists +from .util import getBridgeDBPID # ------------- SMTP Client Config SMTP_DEBUG_LEVEL = 0 # set to 1 to see SMTP message exchange diff --git a/test/test_translations.py b/test/test_translations.py index 48229c6..c1f0c8c 100644 --- a/test/test_translations.py +++ b/test/test_translations.py @@ -11,7 +11,7 @@ from twisted.trial import unittest from bridgedb import translations -from bridgedb.test.test_https_server import DummyRequest +from .https_helpers import DummyRequest REALISH_HEADERS = { diff --git a/test/util.py b/test/util.py index 0f1e0f9..bd8227d 100644 --- a/test/util.py +++ b/test/util.py @@ -4,12 +4,12 @@ # # :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> # please also see AUTHORS file -# :copyright: (c) 2013, Isis Lovecruft -# (c) 2007-2013, The Tor Project, Inc. -# (c) 2007-2013, all entities within the AUTHORS file +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file # :license: 3-Clause BSD, see LICENSE for licensing information -"""Unittests utilitys the `bridgedb.test` package.""" +"""Unittest utilities.""" from __future__ import print_function from __future__ import unicode_literals

1 0

[bridgedb/develop] Declare that b.test.util.DummyBridge and decendents implement IBridge.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 8d4bb32e075b806e7e99faa07e41f79701f47d93 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jun 9 00:19:49 2015 +0000 Declare that b.test.util.DummyBridge and decendents implement IBridge. This allows for more accurate (but still mocked) testing of anything that has conditional treatment of implementers/providers of IBridge. * CHANGE bridgedb.test.util.DummyBridge and DummyMaliciousBridge to declare that they both implement the IBridge interface. Conflicts: lib/bridgedb/test/util.py --- lib/bridgedb/test/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 500bbb8..0f1e0f9 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -25,8 +25,11 @@ from functools import wraps from twisted.trial import unittest from bridgedb import util as bdbutil +from bridgedb.bridges import IBridge from bridgedb.parse.addr import isIPAddress +from zope.interface import implementer + def fileCheckDecorator(func): """Method decorator for a t.t.unittest.TestCase test_* method. @@ -248,6 +251,7 @@ class Benchmarker(object): print("Benchmark: %12fms %12fs" % (self.milliseconds, self.seconds)) +@implementer(IBridge) class DummyBridge(object): """A mock :class:`bridgedb.bridges.Bridge` which only supports a mocked ``getBridgeLine`` method.""" @@ -284,6 +288,7 @@ class DummyBridge(object): return " ".join([item for item in line]) +@implementer(IBridge) class DummyMaliciousBridge(DummyBridge): """A mock :class:`bridgedb.Bridges.Bridge` which only supports a mocked ``getConfigLine`` method and which maliciously insert an additional fake

1 0

[bridgedb/develop] Add a filter for unblocked pluggable transports.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 5088163284bd512082f113fe84f27f7af86973b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 03:08:03 2015 +0000 Add a filter for unblocked pluggable transports. * ADD Filters.filterBridgesByUnblockedTransport(). * CHANGE bridgerequest.BridgeRequestBase.generateFilters() to handle requests for unblocked bridges with pluggable transports. --- lib/bridgedb/Filters.py | 61 +++++++++++++++++++++++++++++++++++++++++ lib/bridgedb/bridgerequest.py | 25 +++++++++++------ 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index eb7db4e..d0d65e8 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -90,6 +90,67 @@ def filterBridgesByTransport(methodname, addressClass=None): funcs[ruleset] = _filterByTransport return _filterByTransport +def filterBridgesByUnblockedTransport(methodname, countryCode=None, addressClass=None): + """Return a filter function for :class:`~bridgedb.bridges.Bridge`s. + + The returned filter function should be called on a + :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` + has a :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an + instance of **addressClass**, and isn't known to be blocked in + **countryCode**. + + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :type countryCode: str or ``None`` + :param countryCode: A two-letter country code which the filtered + :class:`PluggableTransport`s should not be blocked in. + :type addressClass: ``ipaddr.IPAddress`` + :param addressClass: The IP version that the ``Bridge``'s + ``PluggableTransport`` + :data:`~bridgedb.bridges.PluggableTransport.address`` should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not countryCode: + return filterBridgesByTransport(methodname, addressClass) + + if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): + addressClass = IPv4Address + + # Ignore case + methodname = methodname.lower() + countryCode = countryCode.lower() + + ruleset = frozenset([methodname, countryCode, addressClass.__name__]) + try: + return funcs[ruleset] + except KeyError: + def _filterByUnblockedTransport(bridge): + # Since bridge.transportIsBlockedIn() will return True if the + # bridge has that type of transport AND that transport is blocked, + # we can "fail fast" here by doing this faster check before + # iterating over all the transports testing for the other + # conditions. + if bridge.transportIsBlockedIn(countryCode, methodname): + return False + else: + for transport in bridge.transports: + if (transport.methodname == methodname and + isinstance(transport.address, addressClass)): + return True + return False + _filterByUnblockedTransport.__name__ = ("filterBridgesByUnblockedTransport(%s,%s,%s)" + % (methodname, countryCode, addressClass)) + setattr(_filterByUnblockedTransport, "description", + "transport=%s unblocked=%s" % (methodname, countryCode)) + funcs[ruleset] = _filterByUnblockedTransport + return _filterByUnblockedTransport + def filterBridgesByNotBlockedIn(countryCode): """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't known to be blocked in **countryCode**. diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 532558d..b699896 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -171,15 +171,22 @@ class BridgeRequestBase(object): return ptType def generateFilters(self): - if self.addressClass is ipaddr.IPv6Address: - self.addFilter(Filters.filterBridgesByIP6) - else: - self.addFilter(Filters.filterBridgesByIP4) + self.clearFilters() transport = self.justOnePTType() + if transport: - self.clearFilters() - self.addFilter(Filters.filterBridgesByTransport(transport, - self.addressClass)) - for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) + if self.notBlockedIn: + for country in self.notBlockedIn: + self.addFilter(Filters.filterBridgesByUnblockedTransport( + transport, country, self.addressClass)) + else: + self.addFilter(Filters.filterBridgesByTransport( + transport, self.addressClass)) + else: + if self.addressClass is ipaddr.IPv6Address: + self.addFilter(Filters.filterBridgesByIP6) + else: + self.addFilter(Filters.filterBridgesByIP4) + for country in self.notBlockedIn: + self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower()))

1 0

[bridgedb/develop] Merge branch 'fix/12506-separate-dist-dirs' into develop
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 5019761881c7807ab0f09f0728f337b5ce857ca5 Merge: dfbc8e0 a51f553 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jun 25 03:17:58 2015 +0000 Merge branch 'fix/12506-separate-dist-dirs' into develop doc/sphinx/source/bridgedb.HTTPServer.rst | 8 - doc/sphinx/source/bridgedb.https.rst | 11 + doc/sphinx/source/bridgedb.rst | 2 +- doc/sphinx/source/conf.py | 4 +- lib/bridgedb/Bridges.py | 51 +- lib/bridgedb/Dist.py | 664 ++++++------ lib/bridgedb/Filters.py | 41 +- lib/bridgedb/HTTPServer.py | 935 ----------------- lib/bridgedb/Main.py | 116 +-- lib/bridgedb/bridgerequest.py | 23 +- lib/bridgedb/bridges.py | 123 ++- lib/bridgedb/email/autoresponder.py | 16 +- lib/bridgedb/email/request.py | 15 - lib/bridgedb/email/templates.py | 14 +- lib/bridgedb/https/__init__.py | 1 + lib/bridgedb/https/request.py | 143 +++ lib/bridgedb/https/server.py | 909 +++++++++++++++++ .../https/templates/assets/css/bootstrap.min.css | 7 + lib/bridgedb/https/templates/assets/css/custom.css | 158 +++ .../templates/assets/css/font-awesome-ie7.min.css | 384 +++++++ .../templates/assets/css/font-awesome.min.css | 403 ++++++++ lib/bridgedb/https/templates/assets/css/main.css | 24 + .../templates/assets/font/fontawesome-webfont.eot | Bin 0 -> 37405 bytes .../templates/assets/font/fontawesome-webfont.svg | 399 ++++++++ .../templates/assets/font/fontawesome-webfont.ttf | Bin 0 -> 79076 bytes .../templates/assets/font/fontawesome-webfont.woff | Bin 0 -> 43572 bytes .../https/templates/assets/font/lato-bold.woff | Bin 0 -> 46160 bytes .../https/templates/assets/font/lato-italic.woff | Bin 0 -> 47168 bytes .../https/templates/assets/font/lato-regular.woff | Bin 0 -> 46108 bytes .../https/templates/assets/tor-roots-blue.svg | 95 ++ lib/bridgedb/https/templates/assets/tor.svg | 6 + lib/bridgedb/https/templates/base.html | 108 ++ lib/bridgedb/https/templates/bridges.html | 203 ++++ lib/bridgedb/https/templates/captcha.html | 63 ++ lib/bridgedb/https/templates/howto.html | 39 + lib/bridgedb/https/templates/index.html | 43 + lib/bridgedb/https/templates/options.html | 164 +++ lib/bridgedb/https/templates/robots.txt | 1079 ++++++++++++++++++++ lib/bridgedb/strings.py | 270 ++++- lib/bridgedb/templates/assets/bridgedb.png | Bin 2946 -> 0 bytes .../templates/assets/css/bootstrap.min.css | 7 - lib/bridgedb/templates/assets/css/custom.css | 158 --- .../templates/assets/css/font-awesome-ie7.min.css | 384 ------- .../templates/assets/css/font-awesome.min.css | 403 -------- lib/bridgedb/templates/assets/css/main.css | 24 - .../templates/assets/font/fontawesome-webfont.eot | Bin 37405 -> 0 bytes .../templates/assets/font/fontawesome-webfont.svg | 399 -------- .../templates/assets/font/fontawesome-webfont.ttf | Bin 79076 -> 0 bytes .../templates/assets/font/fontawesome-webfont.woff | Bin 43572 -> 0 bytes lib/bridgedb/templates/assets/font/lato-bold.woff | Bin 46160 -> 0 bytes .../templates/assets/font/lato-italic.woff | Bin 47168 -> 0 bytes .../templates/assets/font/lato-regular.woff | Bin 46108 -> 0 bytes lib/bridgedb/templates/assets/tor-roots-blue.svg | 95 -- lib/bridgedb/templates/assets/tor.svg | 6 - lib/bridgedb/templates/base.html | 108 -- lib/bridgedb/templates/bridgedb.asc | 252 ----- lib/bridgedb/templates/bridges.html | 203 ---- lib/bridgedb/templates/captcha.html | 63 -- lib/bridgedb/templates/howto.html | 39 - lib/bridgedb/templates/index.html | 43 - lib/bridgedb/templates/options.html | 164 --- lib/bridgedb/templates/robots.txt | 1079 -------------------- lib/bridgedb/test/email_helpers.py | 36 +- lib/bridgedb/test/https_helpers.py | 130 +++ lib/bridgedb/test/legacy_Tests.py | 69 +- lib/bridgedb/test/test_Dist.py | 239 +++++ lib/bridgedb/test/test_HTTPServer.py | 804 --------------- lib/bridgedb/test/test_Main.py | 101 +- lib/bridgedb/test/test_bridgerequest.py | 20 +- lib/bridgedb/test/test_bridges.py | 83 +- lib/bridgedb/test/test_email_server.py | 5 +- lib/bridgedb/test/test_https_request.py | 91 ++ lib/bridgedb/test/test_https_server.py | 838 +++++++++++++++ lib/bridgedb/test/test_translations.py | 2 +- lib/bridgedb/test/util.py | 76 ++ setup.py | 5 +- 76 files changed, 6594 insertions(+), 5853 deletions(-)

1 0