tor-commits July 2015

tor-commits@lists.torproject.org

18 participants
1250 discussions

[bridgedb/master] Refactor bridgedb.Filters and move it to bridgedb.filters.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 9c09ef7bdb335fd58d48d8f8ff6d79d41fbd088c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 05:09:32 2015 +0000 Refactor bridgedb.Filters and move it to bridgedb.filters. --- doc/sphinx/source/bridgedb.Filters.rst | 8 - doc/sphinx/source/bridgedb.filters.rst | 8 + doc/sphinx/source/bridgedb.rst | 2 +- doc/sphinx/source/conf.py | 2 +- lib/bridgedb/Bridges.py | 12 +- lib/bridgedb/Dist.py | 25 ++- lib/bridgedb/Filters.py | 175 ----------------- lib/bridgedb/bridgerequest.py | 39 ++-- lib/bridgedb/distribute.py | 2 +- lib/bridgedb/filters.py | 238 +++++++++++++++++++++++ lib/bridgedb/https/server.py | 4 - lib/bridgedb/parse/addr.py | 6 +- lib/bridgedb/persistent.py | 12 +- lib/bridgedb/test/legacy_Tests.py | 5 - lib/bridgedb/test/test_Dist.py | 17 +- lib/bridgedb/test/test_filters.py | 333 ++++++++++++++++++++++++++++++++ 16 files changed, 638 insertions(+), 250 deletions(-) diff --git a/doc/sphinx/source/bridgedb.Filters.rst b/doc/sphinx/source/bridgedb.Filters.rst deleted file mode 100644 index caf8dfc..0000000 --- a/doc/sphinx/source/bridgedb.Filters.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.Filters ----------------- - -.. automodule:: bridgedb.Filters - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.filters.rst b/doc/sphinx/source/bridgedb.filters.rst new file mode 100644 index 0000000..9c14ce5 --- /dev/null +++ b/doc/sphinx/source/bridgedb.filters.rst @@ -0,0 +1,8 @@ +bridgedb.filters +---------------- + +.. automodule:: bridgedb.filters + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 31e3a90..7b1fef7 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -15,7 +15,7 @@ BridgeDB Package and Module Documentation bridgedb.crypto bridgedb.Dist bridgedb.email - bridgedb.Filters + bridgedb.filters bridgedb.geo bridgedb.https bridgedb.interfaces diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index b655c27..9b43d0f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -40,7 +40,7 @@ import bridgedb.email.dkim import bridgedb.email.request import bridgedb.email.server import bridgedb.email.templates -import bridgedb.Filters +import bridgedb.filters import bridgedb.geo import bridgedb.https import bridgedb.https.request diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 507013d..2b56884 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -569,7 +569,7 @@ class FilteredBridgeSplitter(object): filterNames = [] for filterName in [x.func_name for x in list(ringname)]: - # Using `filterAssignBridgesToRing.func_name` gives us a messy + # Using `assignBridgesToSubring.func_name` gives us a messy # string which includes all parameters and memory addresses. Get # rid of this by partitioning at the first `(`: realFilterName = filterName.partition('(')[0] @@ -599,8 +599,8 @@ class FilteredBridgeSplitter(object): # hashring '%s'!" % (inserted, ringname))`, this log message appears: # # Jan 04 23:18:37 [INFO] Inserted 12 bridges into hashring - # frozenset([<function filterBridgesByIP4 at 0x2d67cf8>, <function - # filterAssignBridgesToRing(<function hmac_fn at 0x3778398>, 4, 0) at + # frozenset([<function byIPv4 at 0x2d67cf8>, <function + # assignBridgesToSubring(<function hmac_fn at 0x3778398>, 4, 0) at # 0x37de578>])! # # I suppose since it contains memory addresses, it *is* technically @@ -615,10 +615,10 @@ class FilteredBridgeSplitter(object): subringName = [self.distributorName] subringNumber = None for filterName in filterNames: - if filterName.startswith('filterAssignBridgesToRing'): - subringNumber = filterName.lstrip('filterAssignBridgesToRing') + if filterName.startswith('assignBridgesToSubring'): + subringNumber = filterName.lstrip('assignBridgesToSubring') else: - subringName.append(filterName.lstrip('filterBridgesBy')) + subringName.append(filterName.lstrip('by')) if subring.name and 'Proxy' in subring.name: subringName.append('Proxy') elif subringNumber: diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index fe6b8da..9b9e35c 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -25,10 +25,10 @@ from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.distribute import Distributor -from bridgedb.Filters import filterAssignBridgesToRing -from bridgedb.Filters import filterBridgesByRules -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byFilters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import bySubring from bridgedb.parse import addr @@ -259,7 +259,7 @@ class HTTPSDistributor(Distributor): """ logging.info("Prepopulating %s distributor hashrings..." % self.name) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) @@ -269,8 +269,7 @@ class HTTPSDistributor(Distributor): # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, - filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) def insert(self, bridge): @@ -278,7 +277,7 @@ class HTTPSDistributor(Distributor): self.hashring.insert(bridge) def _buildHashringFilters(self, previousFilters, subring): - f = filterAssignBridgesToRing(self.hashring.hmac, self.totalSubrings, subring) + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) previousFilters.append(f) return frozenset(previousFilters) @@ -337,7 +336,7 @@ class HTTPSDistributor(Distributor): logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: @@ -457,8 +456,7 @@ class EmailBasedDistributor(Distributor): # add new ring key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules(ruleset), + self.hashring.addRing(ring, ruleset, byFilters(ruleset), populate_from=self.hashring.bridges) returnNum = self.bridgesPerResponse(ring) @@ -482,10 +480,9 @@ class EmailBasedDistributor(Distributor): def prepopulateRings(self): # populate all rings (for dumping assignments and testing) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: ruleset = frozenset([filterFn]) key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules([filterFn]), + self.hashring.addRing(ring, ruleset, byFilters([filterFn]), populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py deleted file mode 100644 index d0d65e8..0000000 --- a/lib/bridgedb/Filters.py +++ /dev/null @@ -1,175 +0,0 @@ -# BridgeDB by Nick Mathewson. -# Copyright (c) 2007-2012, The Tor Project, Inc. -# See LICENSE for licensing information - -from ipaddr import IPv6Address, IPv4Address -import logging - -funcs = {} - -def filterAssignBridgesToRing(hmac, numRings, assignedRing): - logging.debug(("Creating a filter for assigning bridges to subhashring " - "%s-of-%s...") % (assignedRing, numRings)) - ruleset = frozenset([hmac, numRings, assignedRing]) - try: - return funcs[ruleset] - except KeyError: - def _assignBridgesToRing(bridge): - digest = hmac(bridge.identity) - pos = long( digest[:8], 16 ) - which = pos % numRings + 1 - - if which == assignedRing: - return True - return False - _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing%sof%s" - % (assignedRing, numRings)) - # XXX The `description` attribute must contain an `=`, or else - # dumpAssignments() will not work correctly. - setattr(_assignBridgesToRing, "description", "ring=%d" % assignedRing) - funcs[ruleset] = _assignBridgesToRing - return _assignBridgesToRing - -def filterBridgesByRules(rules): - ruleset = frozenset(rules) - try: - return funcs[ruleset] - except KeyError: - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - setattr(g, "description", " ".join([getattr(f,'description','') for f in rules])) - funcs[ruleset] = g - return g - -def filterBridgesByIP4(bridge): - try: - if IPv4Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 4: - return True - return False -setattr(filterBridgesByIP4, "description", "ip=4") - -def filterBridgesByIP6(bridge): - try: - if IPv6Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 6: - return True - return False -setattr(filterBridgesByIP6, "description", "ip=6") - -def filterBridgesByTransport(methodname, addressClass=None): - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - - ruleset = frozenset([methodname, addressClass]) - try: - return funcs[ruleset] - except KeyError: - def _filterByTransport(bridge): - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" - % (methodname, addressClass)) - setattr(_filterByTransport, "description", "transport=%s" % methodname) - funcs[ruleset] = _filterByTransport - return _filterByTransport - -def filterBridgesByUnblockedTransport(methodname, countryCode=None, addressClass=None): - """Return a filter function for :class:`~bridgedb.bridges.Bridge`s. - - The returned filter function should be called on a - :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` - has a :class:`~bridgedb.bridges.PluggableTransport` such that: - - 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches - **methodname**, - - 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an - instance of **addressClass**, and isn't known to be blocked in - **countryCode**. - - :param str methodname: A Pluggable Transport - :data:`~bridge.bridges.PluggableTransport.methodname`. - :type countryCode: str or ``None`` - :param countryCode: A two-letter country code which the filtered - :class:`PluggableTransport`s should not be blocked in. - :type addressClass: ``ipaddr.IPAddress`` - :param addressClass: The IP version that the ``Bridge``'s - ``PluggableTransport`` - :data:`~bridgedb.bridges.PluggableTransport.address`` should have. - :rtype: callable - :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. - """ - if not countryCode: - return filterBridgesByTransport(methodname, addressClass) - - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - countryCode = countryCode.lower() - - ruleset = frozenset([methodname, countryCode, addressClass.__name__]) - try: - return funcs[ruleset] - except KeyError: - def _filterByUnblockedTransport(bridge): - # Since bridge.transportIsBlockedIn() will return True if the - # bridge has that type of transport AND that transport is blocked, - # we can "fail fast" here by doing this faster check before - # iterating over all the transports testing for the other - # conditions. - if bridge.transportIsBlockedIn(countryCode, methodname): - return False - else: - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByUnblockedTransport.__name__ = ("filterBridgesByUnblockedTransport(%s,%s,%s)" - % (methodname, countryCode, addressClass)) - setattr(_filterByUnblockedTransport, "description", - "transport=%s unblocked=%s" % (methodname, countryCode)) - funcs[ruleset] = _filterByUnblockedTransport - return _filterByUnblockedTransport - -def filterBridgesByNotBlockedIn(countryCode): - """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't - known to be blocked in **countryCode**. - - :param str countryCode: A two-letter country code. - :rtype: bool - :returns: ``True`` if at least one address of the bridge isn't blocked. - ``False`` otherwise. - """ - countryCode = countryCode.lower() - ruleset = frozenset([countryCode]) - try: - return funcs[ruleset] - except KeyError: - def _filterByNotBlockedIn(bridge): - if bridge.isBlockedIn(countryCode): - return False - return True - _filterByNotBlockedIn.__name__ = "filterBridgesByNotBlockedIn(%s)" % countryCode - setattr(_filterByNotBlockedIn, "description", "unblocked=%s" % countryCode) - funcs[ruleset] = _filterByNotBlockedIn - return _filterByNotBlockedIn diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index b699896..2ded7b3 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -11,16 +11,17 @@ #_____________________________________________________________________________ -import logging - import ipaddr +import logging from zope.interface import implements from zope.interface import Attribute from zope.interface import Interface -from bridgedb import Filters from bridgedb.crypto import getHMACFunc +from bridgedb.filters import byIPv +from bridgedb.filters import byNotBlockedIn +from bridgedb.filters import byTransport class IRequestBridges(Interface): @@ -173,20 +174,22 @@ class BridgeRequestBase(object): def generateFilters(self): self.clearFilters() - transport = self.justOnePTType() + pt = self.justOnePTType() + msg = ("Adding a filter to %s for %s for IPv%s" + % (self.__class__.__name__, self.client, self.addressClass)) - if transport: - if self.notBlockedIn: - for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByUnblockedTransport( - transport, country, self.addressClass)) - else: - self.addFilter(Filters.filterBridgesByTransport( - transport, self.addressClass)) - else: - if self.addressClass is ipaddr.IPv6Address: - self.addFilter(Filters.filterBridgesByIP6) - else: - self.addFilter(Filters.filterBridgesByIP4) + self.ipVersion = 4 + if self.addressClass is ipaddr.IPv6Address: + self.ipVersion = 6 + + if self.notBlockedIn: for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) + logging.info("%s %s bridges not blocked in %s..." % + (msg, pt or "vanilla", country)) + self.addFilter(byNotBlockedIn(country, pt, self.addressClass)) + elif pt: + logging.info("%s %s bridges..." % (msg, pt)) + self.addFilter(byTransport(pt, self.addressClass)) + else: + logging.info("%s bridges..." % msg) + self.addFilter(byIPv(self.addressClass)) diff --git a/lib/bridgedb/distribute.py b/lib/bridgedb/distribute.py index c7c9044..f48cbb6 100644 --- a/lib/bridgedb/distribute.py +++ b/lib/bridgedb/distribute.py @@ -47,7 +47,7 @@ DistributorContext { # should go in bridgedb.py } Hashring { - assignBridgesToRings() FORMERLY filterAssignBridgesToRing() + assignBridgesToSubrings() FORMERLY bridgedb.filters.assignBridgesToSubring() + filters bridges uniformly into subrings clear() / __del__() isEmpty property diff --git a/lib/bridgedb/filters.py b/lib/bridgedb/filters.py new file mode 100644 index 0000000..8843937 --- /dev/null +++ b/lib/bridgedb/filters.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_filters ; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson <nickm(a)torproject.org> +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# (c) 2013-2015, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +import logging + +from ipaddr import IPv4Address +from ipaddr import IPv6Address + +from bridgedb.parse.addr import isIPv + + +_cache = {} + + +def bySubring(hmac, assigned, total): + """Create a filter function which filters for only the bridges which fall + into the same **assigned** subhashring (based on the results of an **hmac** + function). + + :type hmac: callable + :param hmac: An HMAC function, i.e. as returned from + :func:`bridgedb.crypto.getHMACFunc`. + :param int assigned: The subring number that we wish to draw bridges from. + For example, if a user is assigned to subring 2of3 based on their IP + address, then this function should only return bridges which would + also be assigned to subring 2of3. + :param int total: The total number of subrings. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + logging.debug(("Creating a filter for assigning bridges to subhashring " + "%s-of-%s...") % (assigned, total)) + + name = "-".join([str(hmac("")[:8]).encode('hex'), + str(assigned), "of", str(total)]) + try: + return _cache[name] + except KeyError: + def _bySubring(bridge): + position = int(hmac(bridge.identity)[:8], 16) + which = (position % total) + 1 + return True if which == assigned else False + # The `description` attribute must contain an `=`, or else + # dumpAssignments() will not work correctly. + setattr(_bySubring, "description", "ring=%d" % assigned) + _bySubring.__name__ = ("bySubring%sof%s" % (assigned, total)) + _bySubring.name = name + _cache[name] = _bySubring + return _bySubring + +def byFilters(filtres): + """Returns a filter which filters by multiple **filtres**. + + :type filtres: list + :param filtres: A list (or other iterable) of callables which some + :class:`~bridgedb.bridges.Bridge`s should be filtered according to. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + name = [] + for filtre in filtres: + name.extend(filtre.name.split(" ")) + name = " ".join(set(name)) + + try: + return _cache[name] + except KeyError: + def _byFilters(bridge): + results = [f(bridge) for f in filtres] + if False in results: + return False + return True + setattr(_byFilters, "description", + " ".join([getattr(f, "description", "") for f in filtres])) + _byFilters.name = name + _cache[name] = _byFilters + return _byFilters + +def byIPv(ipVersion=None): + """Return ``True`` if at least one of the **bridge**'s addresses has the + specified **ipVersion**. + + :param int ipVersion: Either ``4`` or ``6``. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + + name = "ipv%d" % ipVersion + try: + return _cache[name] + except KeyError: + def _byIPv(bridge): + if isIPv(ipVersion, bridge.address): + return True + else: + for address, port, version in bridge.allVanillaAddresses: + if version == ipVersion or isIPv(ipVersion, address): + return True + return False + setattr(_byIPv, "description", "ip=%d" % ipVersion) + _byIPv.__name__ = "byIPv%d()" % ipVersion + _byIPv.name = name + _cache[name] = _byIPv + return _byIPv + +byIPv4 = byIPv(4) +byIPv6 = byIPv(6) + +def byTransport(methodname=None, ipVersion=None): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + The returned filter function should be called on a + :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` + has a :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, and + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an + instance of **addressClass**. + + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s ``PluggableTransport`` + :data:`~bridgedb.bridges.PluggableTransport.address`` should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not methodname: + return byIPv(ipVersion) + + methodname = methodname.lower() + name = "transport-%s ipv%d" % (methodname, ipVersion) + + try: + return _cache[name] + except KeyError: + def _byTransport(bridge): + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byTransport, "description", "transport=%s" % methodname) + _byTransport.__name__ = "byTransport(%s,%s)" % (methodname, ipVersion) + _byTransport.name = name + _cache[name] = _byTransport + return _byTransport + +def byNotBlockedIn(countryCode=None, methodname=None, ipVersion=4): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + If a Pluggable Transport **methodname** was not specified, the returned + filter function returns ``True`` if any of the ``Bridge``'s addresses or + :class:`~bridgedb.bridges.PluggableTransport` addresses aren't blocked in + **countryCode**. See :meth:`~bridgedb.bridges.Bridge.isBlockedIn`. + + Otherwise, if a Pluggable Transport **methodname** was specified, it + returns ``True`` if the ``Bridge`` has a + :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address.version`` + equals the **ipVersion**, and isn't known to be blocked in + **countryCode**. + + :type countryCode: str or ``None`` + :param countryCode: A two-letter country code which the filtered + :class:`PluggableTransport`s should not be blocked in. + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s addresses should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not countryCode: + return byTransport(methodname, ipVersion) + + methodname = methodname.lower() if methodname else methodname + countryCode = countryCode.lower() + + name = [] + if methodname: + name.append("transport-%s" % methodname) + name.append("ipv%d" % ipVersion) + name.append("not-blocked-in-%s" % countryCode) + name = " ".join(name) + + try: + return _cache[name] + except KeyError: + def _byNotBlockedIn(bridge): + if not methodname: + return not bridge.isBlockedIn(countryCode) + elif methodname == "vanilla": + if bridge.address.version == ipVersion: + if not bridge.addressIsBlockedIn(countryCode, + bridge.address, + bridge.orPort): + return True + else: + # Since bridge.transportIsBlockedIn() will return True if the + # bridge has that type of transport AND that transport is + # blocked, we can "fail fast" here by doing this faster check + # before iterating over all the transports testing for the + # other conditions. + if bridge.transportIsBlockedIn(countryCode, methodname): + return False + else: + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byNotBlockedIn, "description", "unblocked=%s" % countryCode) + _byNotBlockedIn.__name__ = ("byTransportNotBlockedIn(%s,%s,%s)" + % (methodname, countryCode, ipVersion)) + _byNotBlockedIn.name = name + _cache[name] = _byNotBlockedIn + return _byNotBlockedIn diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index e8cdd62..2a1d510 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -47,10 +47,6 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index 96dc21e..b3ad680 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -318,7 +318,7 @@ def isIPAddress(ip, compressed=True): return ip return False -def _isIPv(version, ip): +def isIPv(version, ip): """Check if **ip** is a certain **version** (IPv4 or IPv6). .. warning: Do *not* put any calls to the logging module in this function, @@ -352,7 +352,7 @@ def isIPv4(ip): :rtype: boolean :returns: True if the address is an IPv4 address. """ - return _isIPv(4, ip) + return isIPv(4, ip) def isIPv6(ip): """Check if an address is IPv6. @@ -364,7 +364,7 @@ def isIPv6(ip): :rtype: boolean :returns: True if the address is an IPv6 address. """ - return _isIPv(6, ip) + return isIPv(6, ip) def isValidIP(ip): """Check that an IP (v4 or v6) is valid. diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index c996daa..00726f6 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -4,9 +4,9 @@ # # :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> # please also see AUTHORS file -# :copyright: (c) 2013 Isis Lovecruft -# (c) 2007-2013, The Tor Project, Inc. -# (c) 2007-2013, all entities within the AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file # :license: 3-clause BSD, see included LICENSE for information """Module for functionality to persistently store state.""" @@ -23,7 +23,9 @@ except (ImportError, NameError): # pragma: no cover from twisted.python.reflect import safe_repr from twisted.spread import jelly -from bridgedb import Filters, Bridges, Dist +from bridgedb import Bridges +from bridgedb import Dist +from bridgedb import filters from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -32,7 +34,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(Filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist) class MissingState(Exception): diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 95dc34e..3ec634c 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -25,11 +25,6 @@ import bridgedb.Storage import re import ipaddr -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn - from bridgedb.Stability import BridgeHistory from bridgedb.parse import addr diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 85c1b87..46a11b0 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -24,9 +24,8 @@ from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import BridgeRingParameters -from bridgedb.Filters import filterBridgesByNotBlockedIn -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet from bridgedb.test.util import randomHighPort @@ -355,7 +354,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv4() - bridgeRequest.filters.append(filterBridgesByIP6) + bridgeRequest.filters.append(byIPv6) bridgeRequest.generateFilters() bridges = dist.getBridges(bridgeRequest, 1) @@ -367,7 +366,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): """Asking for bridge addresses which are simultaneously IPv6 and IPv4 @@ -379,7 +378,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv6() bridgeRequest.generateFilters() - bridgeRequest.filters.append(filterBridgesByIP4) + bridgeRequest.filters.append(byIPv4) bridges = dist.getBridges(bridgeRequest, 1) self.assertEqual(len(bridges), 3) @@ -390,7 +389,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6(self): """A request for IPv6 bridges should return IPv6 bridges.""" @@ -412,7 +411,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv4(self): """A request for IPv4 bridges should return IPv4 bridges.""" @@ -432,4 +431,4 @@ class HTTPSDistributorTests(unittest.TestCase): addrport, fingerprint = bridgeLine.split() address, port = addrport.rsplit(':', 1) self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_filters.py b/lib/bridgedb/test/test_filters.py new file mode 100644 index 0000000..73e5685 --- /dev/null +++ b/lib/bridgedb/test/test_filters.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# :license: see included LICENSE for information + +"""Tests for :mod:`bridgedb.filters`.""" + +from __future__ import print_function + +import ipaddr + +from twisted.trial import unittest + +from bridgedb import filters +from bridgedb.bridges import Bridge +from bridgedb.bridges import PluggableTransport +from bridgedb.crypto import getHMACFunc + + +class FiltersTests(unittest.TestCase): + """Tests for :mod:`bridgedb.filters`.""" + + def setUp(self): + """Create a Bridge whose address is 1.1.1.1, orPort is 1111, and + fingerprint is 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'. Also, + create an HMAC function whose key is 'plasma'. + """ + self.bridge = Bridge() + self.bridge.address = '1.1.1.1' + self.bridge.orPort = 1111 + self.bridge.fingerprint = 'a' * 40 + + self.hmac = getHMACFunc('plasma') + + def addIPv4VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '1.1.1.1', 1111, {}) + self.bridge.transports.append(pt) + + def addIPv6VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '2006:2222::2222', 1111, {}) + self.bridge.transports.append(pt) + + def test_bySubring_1_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 1, 2) + should return that Bridge (because it is in the sub-hashring we asked + for). + """ + filtre = filters.bySubring(self.hmac, 1, 2) + self.assertTrue(filtre(self.bridge)) + + def test_bySubring_2_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 2, 2) + should *not* return that Bridge (because it is in sub-hashring 1-of-2 + and we asked for Bridges which are in sub-hashring 2-of-2). + """ + filtre = filters.bySubring(self.hmac, 2, 2) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertTrue(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_no_filters(self): + self.addIPv4VoltronPT() + filtre = filters.byFilters([]) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv_ipv5(self): + """Calling byIPv(ipVersion=5) should default to filterint by IPv4.""" + filtre = filters.byIPv(5) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv4_address(self): + """A bridge with an IPv4 address for its main orPort address should + cause filters.byIPv4() to return True. + """ + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_orAddress(self): + """A bridge with an IPv4 address in its orAddresses address should + cause filters.byIPv4() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.bridge.orAddresses = [(ipaddr.IPv4Address('2.2.2.2'), 2222, 4)] + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_none(self): + """A bridge with no IPv4 addresses should cause filters.byIPv4() to + return False. + """ + self.bridge.address = ipaddr.IPv6Address('2006:2222::2222') + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertFalse(filters.byIPv4(self.bridge)) + + def test_byIPv6_address(self): + """A bridge with an IPv6 address for its main orPort address should + cause filters.byIPv6() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_orAddress(self): + """A bridge with an IPv6 address in its orAddresses address should + cause filters.byIPv6() to return True. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_none(self): + """A bridge with no IPv6 addresses should cause filters.byIPv6() to + return False. + """ + self.assertFalse(filters.byIPv6(self.bridge)) + + def test_byTransport_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron', ipVersion=6) to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6_filtering_by_ipv4(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_no_transports(self): + """A bridge without any transports should cause + byTransport('voltron') to return False. + """ + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv4(self): + """byTransport() without namimg a transport to filter by should just + return the bridge's IPv4 address. + """ + filtre = filters.byTransport() + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv6(self): + """byTranspfort(ipVersion=6) without namimg a transport to filter by + should just return the bridge's IPv4 address. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + filtre = filters.byTransport(ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byTransport('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_blocked(self): + """A bridge with an IPv4 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv4VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_not_blocked_ipv4(self): + """A bridge with an IPv6 voltron transport which is not blocked in China + should cause byNotBlockedIn('cn', 'voltron') to return False, because + the IP version is wrong. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6_blocked(self): + """A bridge with an IPv6 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv6VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('voltron') to return False (because it calls + filters.byTransport('voltron')). + """ + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('cn', 'voltron') to return False. + """ + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports_blocked(self): + """A bridge without any transports which is also blocked should cause + byNotBlockedIn('voltron') to return False. + """ + self.bridge.setBlockedIn('cn') + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byNotBlockedIn('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('cn', methodname='obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_ipv5(self): + """Calling byNotBlockedIn([…], ipVersion=5) should default to IPv4.""" + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', ipVersion=5) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked(self): + """Calling byNotBlockedIn('vanilla') should return the IPv4 vanilla + address, if it is not blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked_ipv6(self): + """Calling byNotBlockedIn('vanilla', ipVersion=6) should not return the + IPv4 vanilla address, even if it is not blocked, because it has the + wrong IP version. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_blocked(self): + """Calling byNotBlockedIn('vanilla') should not return the IPv4 vanilla + address, if it is blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('ru', methodname='vanilla') + self.assertFalse(filtre(self.bridge))

1 0

[bridgedb/master] Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit dc91ea443e66223999c5b7a70c4b42025c0ac25c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 07:32:49 2015 +0000 Move bridgedb.Dist.HTTPSDistributor → bridgedb.https.distributor. * FIXES part of #12506: https://bugs.torproject.org/12506 --- doc/sphinx/source/bridgedb.https.rst | 1 + doc/sphinx/source/conf.py | 1 + lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 302 ------------------- lib/bridgedb/Main.py | 21 +- lib/bridgedb/https/distributor.py | 328 ++++++++++++++++++++ lib/bridgedb/https/request.py | 2 +- lib/bridgedb/https/server.py | 2 +- lib/bridgedb/persistent.py | 3 +- lib/bridgedb/test/https_helpers.py | 4 +- lib/bridgedb/test/legacy_Tests.py | 68 +---- lib/bridgedb/test/test_Dist.py | 434 --------------------------- lib/bridgedb/test/test_https_distributor.py | 405 +++++++++++++++++++++++++ lib/bridgedb/test/test_https_server.py | 2 +- lib/bridgedb/test/util.py | 33 ++ 15 files changed, 797 insertions(+), 811 deletions(-) diff --git a/doc/sphinx/source/bridgedb.https.rst b/doc/sphinx/source/bridgedb.https.rst index 1512065..36fa6c6 100644 --- a/doc/sphinx/source/bridgedb.https.rst +++ b/doc/sphinx/source/bridgedb.https.rst @@ -7,5 +7,6 @@ bridgedb.https :depth: 3 .. automodule:: bridgedb.https.__init__ +.. automodule:: bridgedb.https.distributor .. automodule:: bridgedb.https.request .. automodule:: bridgedb.https.server diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 9b43d0f..5038abb 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -43,6 +43,7 @@ import bridgedb.email.templates import bridgedb.filters import bridgedb.geo import bridgedb.https +import bridgedb.https.distributor import bridgedb.https.request import bridgedb.https.server import bridgedb.Main diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 2b56884..7a237fe 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -510,7 +510,7 @@ class FilteredBridgeSplitter(object): :ivar bridges: DOCDOC :type distributorName: str :ivar distributorName: The name of this splitter's distributor. See - :meth:`bridgedb.Dist.HTTPSDistributor.setDistributorName`. + :meth:`~bridgedb.https.distributor.HTTPSDistributor.setDistributorName`. """ self.key = key self.filterRings = {} diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 9b9e35c..1be705f 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -12,14 +12,11 @@ """This module has functions to decide which bridges to hand out to whom.""" -import ipaddr import logging -import re import time import bridgedb.Storage -from bridgedb import proxy from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC @@ -47,305 +44,6 @@ class EmailRequestedKey(Exception): """Raised when an incoming email requested a copy of our GnuPG keys.""" -class HTTPSDistributor(Distributor): - """A Distributor that hands out bridges based on the IP address of an - incoming request and the current time period. - - :type proxies: :class:`~bridgedb.proxies.ProxySet` - :ivar proxies: All known proxies, which we treat differently. See - :param:`proxies`. - :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter` - :ivar hashring: A hashring that assigns bridges to subrings with fixed - proportions. Used to assign bridges into the subrings of this - distributor. - """ - - def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): - """Create a Distributor that decides which bridges to distribute based - upon the client's IP address and the current time. - - :param int totalSubrings: The number of subhashrings to group clients - into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, - then the actual number of clusters is one higher than - ``totalSubrings``, because the set of all known open proxies is - given its own subhashring. - :param bytes key: The master HMAC key for this distributor. All added - bridges are HMACed with this key in order to place them into the - hashrings. - :type proxies: :class:`~bridgedb.proxy.ProxySet` - :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known - Tor Exit relays and other known proxies. These will constitute - the extra cluster, and any client requesting bridges from one of - these **proxies** will be distributed bridges from a separate - subhashring that is specific to Tor/proxy users. - :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` - :param answerParameters: A mechanism for ensuring that the set of - bridges that this distributor answers a client with fit certain - parameters, i.e. that an answer has "at least two obfsproxy - bridges" or "at least one bridge on port 443", etc. - """ - super(HTTPSDistributor, self).__init__(key) - self.totalSubrings = totalSubrings - self.answerParameters = answerParameters - - if proxies: - logging.info("Added known proxies to HTTPS distributor...") - self.proxies = proxies - self.totalSubrings += 1 - self.proxySubring = self.totalSubrings - else: - logging.warn("No known proxies were added to HTTPS distributor!") - self.proxies = proxy.ProxySet() - self.proxySubring = 0 - - self.ringCacheSize = self.totalSubrings * 3 - - key2 = getHMAC(key, "Assign-Bridges-To-Rings") - key3 = getHMAC(key, "Order-Areas-In-Rings") - key4 = getHMAC(key, "Assign-Areas-To-Rings") - - self._clientToPositionHMAC = getHMACFunc(key3, hex=False) - self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) - self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) - self.name = 'HTTPS' - logging.debug("Added %s to %s distributor." % - (self.hashring.__class__.__name__, self.name)) - - def bridgesPerResponse(self, hashring=None): - return super(HTTPSDistributor, self).bridgesPerResponse(hashring) - - @classmethod - def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): - """Map all clients whose **ip**s are within the same subnet to the same - arbitrary string. - - .. hint:: For non-proxy IP addresses, any two IPv4 addresses within - the same ``/16`` subnet, or any two IPv6 addresses in the same - ``/32`` subnet, will get the same string. - - Subnets for this distributor are grouped into the number of rings - specified by the ``N_IP_CLUSTERS`` configuration option, such that - Alice (with the address ``1.2.3.4`` and Bob (with the address - ``1.2.178.234``) are placed within the same cluster, but Carol (with - address ``1.3.11.33``) *might* end up in a different cluster. - - >>> from bridgedb.Dist import HTTPSDistributor - >>> HTTPSDistributor.getSubnet('1.2.3.4') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('1.2.211.154') - '1.2.0.0/16' - >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') - '2001:f::/32' - >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') - '2a00:c98::/32' - - :param str ip: A string representing an IPv4 or IPv6 address. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - :param int proxySubnets: Place Tor/proxy users into this number of - "subnet" groups. This means that no matter how many different Tor - Exits or proxies a client uses, the most they can ever get is - **proxySubnets** different sets of bridge lines (per interval). - This parameter only has any effect when **usingProxy** is ``True``. - :rtype: str - :returns: The appropriately sized CIDR subnet representation of the **ip**. - """ - if not usingProxy: - # We aren't using bridgedb.parse.addr.isIPAddress(ip, - # compressed=False) here because adding the string "False" into - # the map would land any and all clients whose IP address appeared - # to be invalid at the same position in a hashring. - address = ipaddr.IPAddress(ip) - if address.version == 6: - truncated = ':'.join(address.exploded.split(':')[:2]) - subnet = str(ipaddr.IPv6Network(truncated + "::/32")) - else: - truncated = '.'.join(address.exploded.split('.')[:2]) - subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) - else: - group = (int(ipaddr.IPAddress(ip)) % 4) + 1 - subnet = "proxy-group-%d" % group - - logging.debug("Client IP was within area: %s" % subnet) - return subnet - - def mapSubnetToSubring(self, subnet, usingProxy=False): - """Determine the correct subhashring for a client, based upon the - **subnet**. - - :param str subnet: The subnet which contains the client's IP. See - :staticmethod:`getSubnet`. - :param bool usingProxy: Set to ``True`` if the client was using one of - the known :data:`proxies`. - """ - # If the client wasn't using a proxy, select the client's subring - # based upon the client's subnet (modulo the total subrings): - if not usingProxy: - mod = self.totalSubrings - # If there is a proxy subring, don't count it for the modulus: - if self.proxySubring: - mod -= 1 - return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 - else: - return self.proxySubring - - def mapClientToHashringPosition(self, interval, subnet): - """Map the client to a position on a (sub)hashring, based upon the - **interval** which the client's request occurred within, as well as - the **subnet** of the client's IP address. - - .. note:: For an explanation of how **subnet** is determined, see - :staticmethod:`getSubnet`. - - :param str interval: The interval which this client's request for - bridges took place within. - :param str subnet: A string representing the subnet containing the - client's IP address. - :rtype: int - :returns: The results of keyed HMAC, which should determine the - client's position in a (sub)hashring of bridges (and thus - determine which bridges they receive). - """ - position = "<%s>%s" % (interval, subnet) - mapping = self._clientToPositionHMAC(position) - return mapping - - def prepopulateRings(self): - """Prepopulate this distributor's hashrings and subhashrings with - bridges. - - The hashring structure for this distributor is influenced by the - ``N_IP_CLUSTERS`` configuration option, as well as the number of - ``PROXY_LIST_FILES``. - - Essentially, :data:`totalSubrings` is set to the specified - ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of - Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), - are stored in :data:`proxies`, and the latter is added as an - additional cluster (such that :data:`totalSubrings` becomes - ``N_IP_CLUSTERS + 1``). The number of subhashrings which this - :class:`Distributor` has active in its hashring is then - :data:`totalSubrings`, where the last cluster is reserved for all - :data:`proxies`. - - As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and - ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number - of subhashrings is five — four for the "clusters", and one for the - :data:`proxies`. Thus, the resulting hashring-subhashring structure - would look like: - - +------------------+---------------------------------------------------+-------------- - | | Directly connecting users | Tor / known | - | | | proxy users | - +------------------+------------+------------+------------+------------+-------------+ - | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | - +==================+============+============+============+============+=============+ - | Subhashrings | | | | | | - | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | - +------------------+------------+------------+------------+------------+-------------+ - | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | - | Subhashrings | | | | | | - | bBy requested +------------+------------+------------+------------+-------------+ - | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | - | | | | | | | - +------------------+------------+------------+------------+------------+-------------+ - - The "filtered subhashrings" are essentially filtered copies of their - respective subhashring, such that they only contain bridges which - support IPv4 or IPv6, respectively. Additionally, the contents of - ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. - - Thus, in this example, we end up with **10 total subhashrings**. - """ - logging.info("Prepopulating %s distributor hashrings..." % self.name) - - for filterFn in [byIPv4, byIPv6]: - for subring in range(1, self.totalSubrings + 1): - filters = self._buildHashringFilters([filterFn,], subring) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - # For consistency with previous implementation of this method, - # only set the "name" for "clusters" which are for this - # distributor's proxies: - if subring == self.proxySubring: - ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - def insert(self, bridge): - """Assign a bridge to this distributor.""" - self.hashring.insert(bridge) - - def _buildHashringFilters(self, previousFilters, subring): - f = bySubring(self.hashring.hmac, subring, self.totalSubrings) - previousFilters.append(f) - return frozenset(previousFilters) - - def getBridges(self, bridgeRequest, interval): - """Return a list of bridges to give to a user. - - :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` - :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` - with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` - attribute set to a string containing the client's IP address. - :param str interval: The time period when we got this request. This - can be any string, so long as it changes with every period. - :rtype: list - :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in - the response. See - :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` - for an example of how this is used. - """ - logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) - - if not len(self.hashring): - logging.warn("Bailing! Hashring has zero bridges!") - return [] - - usingProxy = False - - # First, check if the client's IP is one of the known :data:`proxies`: - if bridgeRequest.client in self.proxies: - # The tag is a tag applied to a proxy IP address when it is added - # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default - # is 'exit_relay'. For other proxies loaded from the - # PROXY_LIST_FILES config option, the default tag is the full - # filename that the IP address originally came from. - usingProxy = True - tag = self.proxies.getTag(bridgeRequest.client) - logging.info("Client was from known proxy (tag: %s): %s" % - (tag, bridgeRequest.client)) - - subnet = self.getSubnet(bridgeRequest.client, usingProxy) - subring = self.mapSubnetToSubring(subnet, usingProxy) - position = self.mapClientToHashringPosition(interval, subnet) - filters = self._buildHashringFilters(bridgeRequest.filters, subring) - - logging.debug("Client request within time interval: %s" % interval) - logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) - logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) - logging.debug("Total bridges: %d" % len(self.hashring)) - logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) - - # Check wheth we have a cached copy of the hashring: - if filters in self.hashring.filterRings.keys(): - logging.debug("Cache hit %s" % filters) - _, ring = self.hashring.filterRings[filters] - # Otherwise, construct a new hashring and populate it: - else: - logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, byFilters(filters), - populate_from=self.hashring.bridges) - - # Determine the appropriate number of bridges to give to the client: - returnNum = self.bridgesPerResponse(ring) - answer = ring.getBridges(position, returnNum) - - return answer - - class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index ba74b66..3a15145 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -30,6 +30,7 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig +from bridgedb.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors import bridgedb.Storage @@ -186,16 +187,17 @@ def createBridgeRings(cfg, proxyList, key): """Create the bridge distributors defined by the config file :type cfg: :class:`Conf` - :param cfg: The current configuration, including any in-memory - settings (i.e. settings whose values were not obtained from the - config file, but were set via a function somewhere) + :param cfg: The current configuration, including any in-memory settings + (i.e. settings whose values were not obtained from the config file, + but were set via a function somewhere) :type proxyList: :class:`~bridgedb.proxy.ProxySet` :param proxyList: The container for the IP addresses of any currently - known open proxies. + known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an HTTPSDistributor or None, - and an EmailBasedDistributor or None. + :returns: A BridgeSplitter hashring, an + :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an + EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -210,7 +212,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") - ipDistributor = Dist.HTTPSDistributor( + ipDistributor = HTTPSDistributor( cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, @@ -328,8 +330,9 @@ def run(options, reactor=reactor): into their hashring assignments. :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently - known open proxies. - :ivar ipDistributor: A :class:`Dist.HTTPSDistributor`. + known open proxies. + :ivar ipDistributor: A + :class:`~bridgedb.https.distributor.HTTPSDistributor`. :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some diff --git a/lib/bridgedb/https/distributor.py b/lib/bridgedb/https/distributor.py new file mode 100644 index 0000000..f8cf09d --- /dev/null +++ b/lib/bridgedb/https/distributor.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_https_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""A Distributor that hands out bridges through a web interface.""" + +import ipaddr +import logging + +import bridgedb.Storage + +from bridgedb import proxy +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import FilteredBridgeSplitter +from bridgedb.crypto import getHMAC +from bridgedb.crypto import getHMACFunc +from bridgedb.distribute import Distributor +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import byFilters +from bridgedb.filters import bySubring + + +class HTTPSDistributor(Distributor): + """A Distributor that hands out bridges based on the IP address of an + incoming request and the current time period. + + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. + :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. + """ + + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ + super(HTTPSDistributor, self).__init__(key) + self.totalSubrings = totalSubrings + self.answerParameters = answerParameters + + if proxies: + logging.info("Added known proxies to HTTPS distributor...") + self.proxies = proxies + self.totalSubrings += 1 + self.proxySubring = self.totalSubrings + else: + logging.warn("No known proxies were added to HTTPS distributor!") + self.proxies = proxy.ProxySet() + self.proxySubring = 0 + + self.ringCacheSize = self.totalSubrings * 3 + + key2 = getHMAC(key, "Assign-Bridges-To-Rings") + key3 = getHMAC(key, "Order-Areas-In-Rings") + key4 = getHMAC(key, "Assign-Areas-To-Rings") + + self._clientToPositionHMAC = getHMACFunc(key3, hex=False) + self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) + self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) + self.name = 'HTTPS' + logging.debug("Added %s to %s distributor." % + (self.hashring.__class__.__name__, self.name)) + + def bridgesPerResponse(self, hashring=None): + return super(HTTPSDistributor, self).bridgesPerResponse(hashring) + + @classmethod + def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): + """Map all clients whose **ip**s are within the same subnet to the same + arbitrary string. + + .. hint:: For non-proxy IP addresses, any two IPv4 addresses within + the same ``/16`` subnet, or any two IPv6 addresses in the same + ``/32`` subnet, will get the same string. + + Subnets for this distributor are grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option, such that + Alice (with the address ``1.2.3.4`` and Bob (with the address + ``1.2.178.234``) are placed within the same cluster, but Carol (with + address ``1.3.11.33``) *might* end up in a different cluster. + + >>> from bridgedb.https.distributor import HTTPSDistributor + >>> HTTPSDistributor.getSubnet('1.2.3.4') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('1.2.211.154') + '1.2.0.0/16' + >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') + '2001:f::/32' + >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') + '2a00:c98::/32' + + :param str ip: A string representing an IPv4 or IPv6 address. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + :param int proxySubnets: Place Tor/proxy users into this number of + "subnet" groups. This means that no matter how many different Tor + Exits or proxies a client uses, the most they can ever get is + **proxySubnets** different sets of bridge lines (per interval). + This parameter only has any effect when **usingProxy** is ``True``. + :rtype: str + :returns: The appropriately sized CIDR subnet representation of the **ip**. + """ + if not usingProxy: + # We aren't using bridgedb.parse.addr.isIPAddress(ip, + # compressed=False) here because adding the string "False" into + # the map would land any and all clients whose IP address appeared + # to be invalid at the same position in a hashring. + address = ipaddr.IPAddress(ip) + if address.version == 6: + truncated = ':'.join(address.exploded.split(':')[:2]) + subnet = str(ipaddr.IPv6Network(truncated + "::/32")) + else: + truncated = '.'.join(address.exploded.split('.')[:2]) + subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) + else: + group = (int(ipaddr.IPAddress(ip)) % 4) + 1 + subnet = "proxy-group-%d" % group + + logging.debug("Client IP was within area: %s" % subnet) + return subnet + + def mapSubnetToSubring(self, subnet, usingProxy=False): + """Determine the correct subhashring for a client, based upon the + **subnet**. + + :param str subnet: The subnet which contains the client's IP. See + :staticmethod:`getSubnet`. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + """ + # If the client wasn't using a proxy, select the client's subring + # based upon the client's subnet (modulo the total subrings): + if not usingProxy: + mod = self.totalSubrings + # If there is a proxy subring, don't count it for the modulus: + if self.proxySubring: + mod -= 1 + return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 + else: + return self.proxySubring + + def mapClientToHashringPosition(self, interval, subnet): + """Map the client to a position on a (sub)hashring, based upon the + **interval** which the client's request occurred within, as well as + the **subnet** of the client's IP address. + + .. note:: For an explanation of how **subnet** is determined, see + :staticmethod:`getSubnet`. + + :param str interval: The interval which this client's request for + bridges took place within. + :param str subnet: A string representing the subnet containing the + client's IP address. + :rtype: int + :returns: The results of keyed HMAC, which should determine the + client's position in a (sub)hashring of bridges (and thus + determine which bridges they receive). + """ + position = "<%s>%s" % (interval, subnet) + mapping = self._clientToPositionHMAC(position) + return mapping + + def prepopulateRings(self): + """Prepopulate this distributor's hashrings and subhashrings with + bridges. + + The hashring structure for this distributor is influenced by the + ``N_IP_CLUSTERS`` configuration option, as well as the number of + ``PROXY_LIST_FILES``. + + Essentially, :data:`totalSubrings` is set to the specified + ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of + Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), + are stored in :data:`proxies`, and the latter is added as an + additional cluster (such that :data:`totalSubrings` becomes + ``N_IP_CLUSTERS + 1``). The number of subhashrings which this + :class:`Distributor` has active in its hashring is then + :data:`totalSubrings`, where the last cluster is reserved for all + :data:`proxies`. + + As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and + ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number + of subhashrings is five — four for the "clusters", and one for the + :data:`proxies`. Thus, the resulting hashring-subhashring structure + would look like: + + +------------------+---------------------------------------------------+-------------- + | | Directly connecting users | Tor / known | + | | | proxy users | + +------------------+------------+------------+------------+------------+-------------+ + | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | + +==================+============+============+============+============+=============+ + | Subhashrings | | | | | | + | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | + +------------------+------------+------------+------------+------------+-------------+ + | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | + | Subhashrings | | | | | | + | bBy requested +------------+------------+------------+------------+-------------+ + | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | + | | | | | | | + +------------------+------------+------------+------------+------------+-------------+ + + The "filtered subhashrings" are essentially filtered copies of their + respective subhashring, such that they only contain bridges which + support IPv4 or IPv6, respectively. Additionally, the contents of + ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. + + Thus, in this example, we end up with **10 total subhashrings**. + """ + logging.info("Prepopulating %s distributor hashrings..." % self.name) + + for filterFn in [byIPv4, byIPv6]: + for subring in range(1, self.totalSubrings + 1): + filters = self._buildHashringFilters([filterFn,], subring) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + # For consistency with previous implementation of this method, + # only set the "name" for "clusters" which are for this + # distributor's proxies: + if subring == self.proxySubring: + ring.setName('{0} Proxy Ring'.format(self.name)) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + def insert(self, bridge): + """Assign a bridge to this distributor.""" + self.hashring.insert(bridge) + + def _buildHashringFilters(self, previousFilters, subring): + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) + previousFilters.append(f) + return frozenset(previousFilters) + + def getBridges(self, bridgeRequest, interval): + """Return a list of bridges to give to a user. + + :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` + :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` + with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` + attribute set to a string containing the client's IP address. + :param str interval: The time period when we got this request. This + can be any string, so long as it changes with every period. + :rtype: list + :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in + the response. See + :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` + for an example of how this is used. + """ + logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) + + if not len(self.hashring): + logging.warn("Bailing! Hashring has zero bridges!") + return [] + + usingProxy = False + + # First, check if the client's IP is one of the known :data:`proxies`: + if bridgeRequest.client in self.proxies: + # The tag is a tag applied to a proxy IP address when it is added + # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default + # is 'exit_relay'. For other proxies loaded from the + # PROXY_LIST_FILES config option, the default tag is the full + # filename that the IP address originally came from. + usingProxy = True + tag = self.proxies.getTag(bridgeRequest.client) + logging.info("Client was from known proxy (tag: %s): %s" % + (tag, bridgeRequest.client)) + + subnet = self.getSubnet(bridgeRequest.client, usingProxy) + subring = self.mapSubnetToSubring(subnet, usingProxy) + position = self.mapClientToHashringPosition(interval, subnet) + filters = self._buildHashringFilters(bridgeRequest.filters, subring) + + logging.debug("Client request within time interval: %s" % interval) + logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) + logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Total bridges: %d" % len(self.hashring)) + logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) + + # Check wheth we have a cached copy of the hashring: + if filters in self.hashring.filterRings.keys(): + logging.debug("Cache hit %s" % filters) + _, ring = self.hashring.filterRings[filters] + # Otherwise, construct a new hashring and populate it: + else: + logging.debug("Cache miss %s" % filters) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) + ring = BridgeRing(key1, self.answerParameters) + self.hashring.addRing(ring, filters, byFilters(filters), + populate_from=self.hashring.bridges) + + # Determine the appropriate number of bridges to give to the client: + returnNum = self.bridgesPerResponse(ring) + answer = ring.getBridges(position, returnNum) + + return answer diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py index 0ae52c7..b106a13 100644 --- a/lib/bridgedb/https/request.py +++ b/lib/bridgedb/https/request.py @@ -55,7 +55,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self, addClientCountryCode=True): """Process a new bridge request received through the - :class:`~bridgedb.Dist.HTTPSDistributor`. + :class:`~bridgedb.https.distributor.HTTPSDistributor`. :param bool addClientCountryCode: If ``True``, then calling :meth:`withoutBlockInCountry` will attempt to add the client's own diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index 2a1d510..2106dbf 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -815,7 +815,7 @@ def addWebServer(config, distributor): GIMP_CAPTCHA_DIR GIMP_CAPTCHA_HMAC_KEYFILE GIMP_CAPTCHA_RSA_KEYFILE - :type distributor: :class:`bridgedb.Dist.HTTPSDistributor` + :type distributor: :class:`bridgedb.https.distributor.HTTPSDistributor` :param distributor: A bridge distributor. :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 00726f6..0853c1e 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -26,6 +26,7 @@ from twisted.spread import jelly from bridgedb import Bridges from bridgedb import Dist from bridgedb import filters +from bridgedb.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -34,7 +35,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist, httpsDistributor) class MissingState(Exception): diff --git a/lib/bridgedb/test/https_helpers.py b/lib/bridgedb/test/https_helpers.py index 3d8ec19..e2c94ba 100644 --- a/lib/bridgedb/test/https_helpers.py +++ b/lib/bridgedb/test/https_helpers.py @@ -99,8 +99,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE): class DummyHTTPSDistributor(object): - """A mocked :class:`bridgedb.Dist.HTTPSDistributor` which is used to test - :class:`bridgedb.https.server.BridgesResource`. + """A mocked :class:`bridgedb.https.distributor.HTTPSDistributor` which is + used to test :class:`bridgedb.https.server.BridgesResource`. """ _bridge_class = util.DummyBridge _bridgesPerResponseMin = 3 diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 3ec634c..c0a9ff0 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -35,15 +35,13 @@ from bridgedb.test.util import randomIPv6 from bridgedb.test.util import randomIPString from bridgedb.test.util import randomIPv4String from bridgedb.test.util import randomIPv6String +from bridgedb.test.util import randomPort +from bridgedb.test.util import randomValidIPv6 from math import log +warnings.filterwarnings('ignore', '.*tmpnam.*') -def suppressWarnings(): - warnings.filterwarnings('ignore', '.*tmpnam.*') - -def randomPort(): - return random.randint(1,65535) def randomPortSpec(): """ @@ -57,8 +55,8 @@ def randomPortSpec(): def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv4()) + ip = randomIPv4() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -66,21 +64,7 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv4String() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: for i in xrange(0,8): @@ -91,8 +75,8 @@ def fakeBridge(orport=8080, running=True, stable=True, or_addresses=False, def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, transports=False): - nn = "bridge-%s"%random.randrange(0,1000000) - ip = ipaddr.IPAddress(randomIPv6()) + ip = randomIPv6() + nn = "bridge-%s" % int(ip) fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) b = bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) b.setStatus(running, stable) @@ -100,47 +84,15 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, oraddrs = [] if or_addresses: for i in xrange(8): - # Only add or_addresses if they are valid. Otherwise, the test - # will randomly fail if an invalid address is chosen: - address = randomIPv6() - portlist = addr.PortList(randomPortSpec()) - if addr.isValidIP(address): - address = bracketIPv6(address) - oraddrs.append((address, portlist,)) - - for address, portlist in oraddrs: - networkstatus.parseALine("{0}:{1}".format(address, portlist)) - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist - - try: - portlist.add(b.or_addresses[address]) - except KeyError: - pass - finally: - b.or_addresses[address] = portlist + b.orAddresses.append((randomValidIPv6(), randomPort(), 6)) if transports: for i in xrange(0,8): b.transports.append(bridgedb.Bridges.PluggableTransport(b, random.choice(["obfs", "obfs2", "pt1"]), randomIP(), randomPort())) - return b -simpleDesc = "router Unnamed %s %s 0 9030\n"\ -"opt fingerprint DEAD BEEF F00F DEAD BEEF F00F DEAD BEEF F00F DEAD\n"\ -"opt @purpose bridge\n" -orAddress = "or-address %s:%s\n" - - -class RhymesWith255ProxySet: - def __contains__(self, ip): - return ip.endswith(".255") class EmailBridgeDistTests(unittest.TestCase): def setUp(self): @@ -412,6 +364,4 @@ def testSuite(): return suite def main(): - suppressWarnings() - unittest.TextTestRunner(verbosity=1).run(testSuite()) diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py deleted file mode 100644 index 46a11b0..0000000 --- a/lib/bridgedb/test/test_Dist.py +++ /dev/null @@ -1,434 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> -# please also see AUTHORS file -# :copyright: (c) 2013-2015 Isis Lovecruft -# (c) 2007-2015, The Tor Project, Inc. -# (c) 2007-2015, all entities within the AUTHORS file -# :license: 3-clause BSD, see included LICENSE for information - -"""Tests for :mod:`bridgedb.Dist`.""" - -from __future__ import print_function - -import hashlib -import ipaddr -import random - -from twisted.trial import unittest - -from bridgedb import Dist -from bridgedb.bridges import Bridge -from bridgedb.bridges import PluggableTransport -from bridgedb.Bridges import BridgeRing -from bridgedb.Bridges import BridgeRingParameters -from bridgedb.filters import byIPv4 -from bridgedb.filters import byIPv6 -from bridgedb.https.request import HTTPSBridgeRequest -from bridgedb.proxy import ProxySet -from bridgedb.test.util import randomHighPort -from bridgedb.test.util import randomValidIPv4String -from bridgedb.test.util import randomValidIPv6 -from bridgedb.test.https_helpers import DummyRequest - - -def _generateFakeBridges(n=500): - bridges = [] - - for i in range(n): - addr = randomValidIPv4String() - nick = 'bridge-%d' % i - port = randomHighPort() - # Real tor currently only supports one extra ORAddress, and it can - # only be IPv6. - addrs = [(randomValidIPv6(), randomHighPort(), 6)] - fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) - - # We only support the ones without PT args, because they're easier to fake. - supported = ["obfs2", "obfs3", "fte"] - transports = [] - for j, method in zip(range(1, len(supported) + 1), supported): - pt = PluggableTransport(fpr, method, addr, port - j, {}) - transports.append(pt) - - bridge = Bridge(nick, addr, port, fpr) - bridge.flags.update("Running Stable") - bridge.transports = transports - bridge.orAddresses = addrs - bridges.append(bridge) - - return bridges - - -BRIDGES = _generateFakeBridges() - - -class HTTPSDistributorTests(unittest.TestCase): - """Tests for :class:`HTTPSDistributor`.""" - - def setUp(self): - self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' - self.bridges = BRIDGES - - def tearDown(self): - """Reset all bridge blocks in between test method runs.""" - for bridge in self.bridges: - bridge._blockedIn = {} - - def coinFlip(self): - return bool(random.getrandbits(1)) - - def randomClientRequest(self): - bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) - bridgeRequest.client = randomValidIPv4String() - bridgeRequest.isValid(True) - bridgeRequest.generateFilters() - return bridgeRequest - - def randomClientRequestForNotBlockedIn(self, cc): - httpRequest = DummyRequest(['']) - httpRequest.args.update({'unblocked': [cc]}) - bridgeRequest = self.randomClientRequest() - bridgeRequest.withoutBlockInCountry(httpRequest) - bridgeRequest.generateFilters() - return bridgeRequest - - def test_HTTPSDistributor_init_with_proxies(self): - """The HTTPSDistributor, when initialised with proxies, should add an - extra hashring for proxy users. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - self.assertIsNotNone(dist.proxies) - self.assertGreater(dist.proxySubring, 0) - self.assertEqual(dist.proxySubring, 4) - self.assertEqual(dist.totalSubrings, 4) - - def test_HTTPSDistributor_bridgesPerResponse_120(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:120]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_100(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 3) - - def test_HTTPSDistributor_bridgesPerResponse_50(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:60]] - self.assertEqual(dist.bridgesPerResponse(), 2) - - def test_HTTPSDistributor_bridgesPerResponse_15(self): - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:15]] - self.assertEqual(dist.bridgesPerResponse(), 1) - - def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): - dist = Dist.HTTPSDistributor(3, self.key) - dist._bridgesPerResponseMax = 5 - [dist.insert(bridge) for bridge in self.bridges[:100]] - self.assertEqual(dist.bridgesPerResponse(), 5) - - def test_HTTPSDistributor_getSubnet_usingProxy(self): - """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy - group number. - """ - clientRequest = self.randomClientRequest() - expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 - subnet = Dist.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) - self.assertTrue(subnet.startswith('proxy-group-')) - self.assertEqual(int(subnet[-1]), expectedGroup) - - def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): - """HTTPSDistributor.mapSubnetToSubring() when the client was using a - proxy should map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - subnet = 'proxy-group-3' - subring = dist.mapSubnetToSubring(subnet, usingProxy=True) - self.assertEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): - """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using - a proxy, but the distributor does have some known proxies and a - proxySubring, should not map the client to the proxy subhashring. - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - # Note that if they were actually from a proxy, their subnet would be - # something like "proxy-group-3". - subnet = '15.1.0.0/16' - subring = dist.mapSubnetToSubring(subnet, usingProxy=False) - self.assertNotEqual(subring, dist.proxySubring) - - def test_HTTPSDistributor_prepopulateRings_with_proxies(self): - """An HTTPSDistributor with proxies should prepopulate two extra - subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). - """ - dist = Dist.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 8) - - def test_HTTPSDistributor_prepopulateRings_without_proxies(self): - """An HTTPSDistributor without proxies should prepopulate - totalSubrings * 2 subrings. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges] - dist.prepopulateRings() - self.assertEqual(len(dist.hashring.filterRings), 6) - - ipv4subrings = [] - ipv6subrings = [] - - for subringName, (filters, subring) in dist.hashring.filterRings.items(): - if 'IPv4' in subringName: - ipv6subrings.append(subring) - if 'IPv6' in subringName: - ipv6subrings.append(subring) - - self.assertEqual(len(ipv4subrings), len(ipv6subrings)) - - def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - bridge.setBlockedIn('cn') - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - b = dist.getBridges(clientRequest1, 1) - self.assertEqual(len(b), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - b = dist.getBridges(clientRequest2, 1) - self.assertEqual(len(b), 3) - - def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - blockedCN = [] - blockedIR = [] - - for bridge in bridges: - if self.coinFlip(): - bridge.setBlockedIn('cn') - blockedCN.append(bridge.fingerprint) - - if self.coinFlip(): - bridge.setBlockedIn('ir') - blockedIR.append(bridge.fingerprint) - - [dist.insert(bridge) for bridge in bridges] - - for _ in range(5): - clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridges = dist.getBridges(clientRequest1, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('cn')) - self.assertNotIn(b.fingerprint, blockedCN) - # The client *should* have gotten some bridges still. - self.assertGreater(len(bridges), 0) - - clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') - bridges = dist.getBridges(clientRequest2, 1) - for b in bridges: - self.assertFalse(b.isBlockedIn('ir')) - self.assertNotIn(b.fingerprint, blockedIR) - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): - dist = Dist.HTTPSDistributor(1, self.key) - bridges = self.bridges[:] - - for bridge in bridges: - # Pretend that China blocks all vanilla bridges: - bridge.setBlockedIn('cn', methodname='vanilla') - # Pretend that China blocks all obfs2: - bridge.setBlockedIn('cn', methodname='obfs2') - # Pretend that China blocks some obfs3: - if self.coinFlip(): - bridge.setBlockedIn('cn', methodname='obfs3') - - [dist.insert(bridge) for bridge in bridges] - - for i in xrange(5): - bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest1.transports.append('obfs2') - bridgeRequest1.generateFilters() - # We shouldn't get any obfs2 bridges, since they're all blocked in - # China: - bridges = dist.getBridges(bridgeRequest1, "faketimestamp") - self.assertEqual(len(bridges), 0) - - bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') - bridgeRequest2.transports.append('obfs3') - bridgeRequest2.generateFilters() - # We probably will get at least one bridge back! It's pretty - # unlikely to lose a coin flip 500 times in a row. - bridges = dist.getBridges(bridgeRequest2, "faketimestamp") - self.assertGreater(len(bridges), 0) - - bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') - bridgeRequest3.transports.append('obfs3') - bridgeRequest3.generateFilters() - # We should get bridges, since obfs3 isn't blocked in netherlands: - bridges = dist.getBridges(bridgeRequest3, "faketimestamp") - self.assertGreater(len(bridges), 0) - - def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): - """An HTTPSDistributor should give separate bridges to proxy users.""" - proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) - dist = Dist.HTTPSDistributor(3, self.key, proxies) - [dist.insert(bridge) for bridge in self.bridges] - - for _ in range(10): - bridgeRequest1 = self.randomClientRequest() - bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) - - bridgeRequest2 = self.randomClientRequest() - bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) - - n1 = dist.getBridges(bridgeRequest1, 1) - n2 = dist.getBridges(bridgeRequest2, 1) - - self.assertGreater(len(n1), 0) - self.assertGreater(len(n2), 0) - - for b in n1: - self.assertNotIn(b, n2) - for b in n2: - self.assertNotIn(b, n1) - - def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): - """The same client asking for bridges from the HTTPSDistributor - multiple times in a row should get the same bridges in response each - time. - """ - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - responses = {} - for i in range(5): - responses[i] = dist.getBridges(bridgeRequest, 1) - for i in range(4): - self.assertItemsEqual(responses[i], responses[i+1]) - - def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): - param = BridgeRingParameters(needPorts=[(443, 1)]) - dist = Dist.HTTPSDistributor(3, self.key, answerParameters=param) - - bridges = self.bridges[:32] - for b in self.bridges: - b.orPort = 443 - - [dist.insert(bridge) for bridge in bridges] - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for _ in xrange(32): - bridgeRequest = self.randomClientRequest() - answer = dist.getBridges(bridgeRequest, 1) - count = 0 - fingerprints = {} - for bridge in answer: - fingerprints[bridge.identity] = 1 - if bridge.orPort == 443: - count += 1 - self.assertEquals(len(fingerprints), len(answer)) - self.assertGreater(len(fingerprints), 0) - self.assertTrue(count >= 1) - - def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): - """Asking for bridge addresses which are simultaneously IPv4 and IPv6 - (in that order) should return IPv4 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv4() - bridgeRequest.filters.append(byIPv6) - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): - """Asking for bridge addresses which are simultaneously IPv6 and IPv4 - (in that order) should return IPv6 bridges. - """ - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - bridgeRequest.filters.append(byIPv4) - - bridges = dist.getBridges(bridgeRequest, 1) - self.assertEqual(len(bridges), 3) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv6(self): - """A request for IPv6 bridges should return IPv6 bridges.""" - dist = Dist.HTTPSDistributor(3, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.withIPv6() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - address = address.strip('[]') - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(byIPv6(random.choice(bridges))) - - def test_HTTPSDistributor_getBridges_ipv4(self): - """A request for IPv4 bridges should return IPv4 bridges.""" - dist = Dist.HTTPSDistributor(1, self.key) - [dist.insert(bridge) for bridge in self.bridges[:250]] - - for i in xrange(500): - bridgeRequest = self.randomClientRequest() - bridgeRequest.generateFilters() - - bridges = dist.getBridges(bridgeRequest, "faketimestamp") - self.assertTrue(type(bridges) is list) - self.assertGreater(len(bridges), 0) - - bridge = random.choice(bridges) - bridgeLine = bridge.getBridgeLine(bridgeRequest) - addrport, fingerprint = bridgeLine.split() - address, port = addrport.rsplit(':', 1) - self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_distributor.py b/lib/bridgedb/test/test_https_distributor.py new file mode 100644 index 0000000..83f2503 --- /dev/null +++ b/lib/bridgedb/test/test_https_distributor.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Tests for :mod:`bridgedb.https.distributor`.""" + +from __future__ import print_function + +import ipaddr +import logging +import random + +from twisted.trial import unittest + +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import BridgeRingParameters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.https import distributor +from bridgedb.https.request import HTTPSBridgeRequest +from bridgedb.proxy import ProxySet +from bridgedb.test.util import randomValidIPv4String +from bridgedb.test.util import generateFakeBridges +from bridgedb.test.https_helpers import DummyRequest + +logging.disable(50) + + +BRIDGES = generateFakeBridges() + + +class HTTPSDistributorTests(unittest.TestCase): + """Tests for :class:`HTTPSDistributor`.""" + + def setUp(self): + self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' + self.bridges = BRIDGES + + def tearDown(self): + """Reset all bridge blocks in between test method runs.""" + for bridge in self.bridges: + bridge._blockedIn = {} + + def coinFlip(self): + return bool(random.getrandbits(1)) + + def randomClientRequest(self): + bridgeRequest = HTTPSBridgeRequest(addClientCountryCode=False) + bridgeRequest.client = randomValidIPv4String() + bridgeRequest.isValid(True) + bridgeRequest.generateFilters() + return bridgeRequest + + def randomClientRequestForNotBlockedIn(self, cc): + httpRequest = DummyRequest(['']) + httpRequest.args.update({'unblocked': [cc]}) + bridgeRequest = self.randomClientRequest() + bridgeRequest.withoutBlockInCountry(httpRequest) + bridgeRequest.generateFilters() + return bridgeRequest + + def test_HTTPSDistributor_init_with_proxies(self): + """The HTTPSDistributor, when initialised with proxies, should add an + extra hashring for proxy users. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + self.assertIsNotNone(dist.proxies) + self.assertGreater(dist.proxySubring, 0) + self.assertEqual(dist.proxySubring, 4) + self.assertEqual(dist.totalSubrings, 4) + + def test_HTTPSDistributor_bridgesPerResponse_120(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:120]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_100(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.hashring.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 3) + + def test_HTTPSDistributor_bridgesPerResponse_50(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:60]] + self.assertEqual(dist.bridgesPerResponse(), 2) + + def test_HTTPSDistributor_bridgesPerResponse_15(self): + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:15]] + self.assertEqual(dist.bridgesPerResponse(), 1) + + def test_HTTPSDistributor_bridgesPerResponse_100_max_5(self): + dist = distributor.HTTPSDistributor(3, self.key) + dist._bridgesPerResponseMax = 5 + [dist.insert(bridge) for bridge in self.bridges[:100]] + self.assertEqual(dist.bridgesPerResponse(), 5) + + def test_HTTPSDistributor_getSubnet_usingProxy(self): + """HTTPSDistributor.getSubnet(usingProxy=True) should return a proxy + group number. + """ + clientRequest = self.randomClientRequest() + expectedGroup = (int(ipaddr.IPAddress(clientRequest.client)) % 4) + 1 + subnet = distributor.HTTPSDistributor.getSubnet(clientRequest.client, usingProxy=True) + self.assertTrue(subnet.startswith('proxy-group-')) + self.assertEqual(int(subnet[-1]), expectedGroup) + + def test_HTTPSDistributor_mapSubnetToSubring_usingProxy(self): + """HTTPSDistributor.mapSubnetToSubring() when the client was using a + proxy should map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + subnet = 'proxy-group-3' + subring = dist.mapSubnetToSubring(subnet, usingProxy=True) + self.assertEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_mapSubnetToSubring_with_proxies(self): + """HTTPSDistributor.mapSubnetToSubring() when the client wasn't using + a proxy, but the distributor does have some known proxies and a + proxySubring, should not map the client to the proxy subhashring. + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + # Note that if they were actually from a proxy, their subnet would be + # something like "proxy-group-3". + subnet = '15.1.0.0/16' + subring = dist.mapSubnetToSubring(subnet, usingProxy=False) + self.assertNotEqual(subring, dist.proxySubring) + + def test_HTTPSDistributor_prepopulateRings_with_proxies(self): + """An HTTPSDistributor with proxies should prepopulate two extra + subhashrings (one for each of HTTP-Proxy-IPv4 and HTTP-Proxy-IPv6). + """ + dist = distributor.HTTPSDistributor(3, self.key, ProxySet(['1.1.1.1', '2.2.2.2'])) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 8) + + def test_HTTPSDistributor_prepopulateRings_without_proxies(self): + """An HTTPSDistributor without proxies should prepopulate + totalSubrings * 2 subrings. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges] + dist.prepopulateRings() + self.assertEqual(len(dist.hashring.filterRings), 6) + + ipv4subrings = [] + ipv6subrings = [] + + for subringName, (filters, subring) in dist.hashring.filterRings.items(): + if 'IPv4' in subringName: + ipv6subrings.append(subring) + if 'IPv6' in subringName: + ipv6subrings.append(subring) + + self.assertEqual(len(ipv4subrings), len(ipv6subrings)) + + def test_HTTPSDistributor_getBridges_with_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + bridge.setBlockedIn('cn') + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + b = dist.getBridges(clientRequest1, 1) + self.assertEqual(len(b), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + b = dist.getBridges(clientRequest2, 1) + self.assertEqual(len(b), 3) + + def test_HTTPSDistributor_getBridges_with_some_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + blockedCN = [] + blockedIR = [] + + for bridge in bridges: + if self.coinFlip(): + bridge.setBlockedIn('cn') + blockedCN.append(bridge.fingerprint) + + if self.coinFlip(): + bridge.setBlockedIn('ir') + blockedIR.append(bridge.fingerprint) + + [dist.insert(bridge) for bridge in bridges] + + for _ in range(5): + clientRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridges = dist.getBridges(clientRequest1, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('cn')) + self.assertNotIn(b.fingerprint, blockedCN) + # The client *should* have gotten some bridges still. + self.assertGreater(len(bridges), 0) + + clientRequest2 = self.randomClientRequestForNotBlockedIn('ir') + bridges = dist.getBridges(clientRequest2, 1) + for b in bridges: + self.assertFalse(b.isBlockedIn('ir')) + self.assertNotIn(b.fingerprint, blockedIR) + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): + dist = distributor.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + # Pretend that China blocks all vanilla bridges: + bridge.setBlockedIn('cn', methodname='vanilla') + # Pretend that China blocks all obfs2: + bridge.setBlockedIn('cn', methodname='obfs2') + # Pretend that China blocks some obfs3: + if self.coinFlip(): + bridge.setBlockedIn('cn', methodname='obfs3') + + [dist.insert(bridge) for bridge in bridges] + + for i in xrange(5): + bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest1.transports.append('obfs2') + bridgeRequest1.generateFilters() + # We shouldn't get any obfs2 bridges, since they're all blocked in + # China: + bridges = dist.getBridges(bridgeRequest1, "faketimestamp") + self.assertEqual(len(bridges), 0) + + bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest2.transports.append('obfs3') + bridgeRequest2.generateFilters() + # We probably will get at least one bridge back! It's pretty + # unlikely to lose a coin flip 500 times in a row. + bridges = dist.getBridges(bridgeRequest2, "faketimestamp") + self.assertGreater(len(bridges), 0) + + bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') + bridgeRequest3.transports.append('obfs3') + bridgeRequest3.generateFilters() + # We should get bridges, since obfs3 isn't blocked in netherlands: + bridges = dist.getBridges(bridgeRequest3, "faketimestamp") + self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): + """An HTTPSDistributor should give separate bridges to proxy users.""" + proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) + dist = distributor.HTTPSDistributor(3, self.key, proxies) + [dist.insert(bridge) for bridge in self.bridges] + + for _ in range(10): + bridgeRequest1 = self.randomClientRequest() + bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) + + bridgeRequest2 = self.randomClientRequest() + bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) + + n1 = dist.getBridges(bridgeRequest1, 1) + n2 = dist.getBridges(bridgeRequest2, 1) + + self.assertGreater(len(n1), 0) + self.assertGreater(len(n2), 0) + + for b in n1: + self.assertNotIn(b, n2) + for b in n2: + self.assertNotIn(b, n1) + + def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): + """The same client asking for bridges from the HTTPSDistributor + multiple times in a row should get the same bridges in response each + time. + """ + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + responses = {} + for i in range(5): + responses[i] = dist.getBridges(bridgeRequest, 1) + for i in range(4): + self.assertItemsEqual(responses[i], responses[i+1]) + + def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): + param = BridgeRingParameters(needPorts=[(443, 1)]) + dist = distributor.HTTPSDistributor(3, self.key, answerParameters=param) + + bridges = self.bridges[:32] + for b in self.bridges: + b.orPort = 443 + + [dist.insert(bridge) for bridge in bridges] + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for _ in xrange(32): + bridgeRequest = self.randomClientRequest() + answer = dist.getBridges(bridgeRequest, 1) + count = 0 + fingerprints = {} + for bridge in answer: + fingerprints[bridge.identity] = 1 + if bridge.orPort == 443: + count += 1 + self.assertEquals(len(fingerprints), len(answer)) + self.assertGreater(len(fingerprints), 0) + self.assertTrue(count >= 1) + + def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): + """Asking for bridge addresses which are simultaneously IPv4 and IPv6 + (in that order) should return IPv4 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv4() + bridgeRequest.filters.append(byIPv6) + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): + """Asking for bridge addresses which are simultaneously IPv6 and IPv4 + (in that order) should return IPv6 bridges. + """ + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + bridgeRequest.filters.append(byIPv4) + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6(self): + """A request for IPv6 bridges should return IPv6 bridges.""" + dist = distributor.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(byIPv6(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv4(self): + """A request for IPv4 bridges should return IPv4 bridges.""" + dist = distributor.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_https_server.py b/lib/bridgedb/test/test_https_server.py index e4f56f0..e782135 100644 --- a/lib/bridgedb/test/test_https_server.py +++ b/lib/bridgedb/test/test_https_server.py @@ -794,7 +794,7 @@ class HTTPSServerServiceTests(unittest.TestCase): """Unittests for :func:`bridgedb.email.server.addWebServer`.""" def setUp(self): - """Create a server.MailServerContext and EmailBasedDistributor.""" + """Create a config and an HTTPSDistributor.""" self.config = _createConfig() self.distributor = DummyHTTPSDistributor() diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 2d0c020..f164aeb 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -168,6 +168,39 @@ randomValidIPv4String = valid(randomIPv4String) randomValidIPv6String = valid(randomIPv6String) randomValidIPString = valid(randomIPString) +def generateFakeBridges(n=500): + """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random + data. + """ + from bridgedb.bridges import Bridge + from bridgedb.bridges import PluggableTransport + + bridges = [] + + for i in range(n): + addr = randomValidIPv4String() + nick = 'bridge-%d' % i + port = randomHighPort() + # Real tor currently only supports one extra ORAddress, and it can + # only be IPv6. + addrs = [(randomValidIPv6(), randomHighPort(), 6)] + fpr = "".join(random.choice('abcdef0123456789') for _ in xrange(40)) + + # We only support the ones without PT args, because they're easier to fake. + supported = ["obfs2", "obfs3", "fte"] + transports = [] + for j, method in zip(range(1, len(supported) + 1), supported): + pt = PluggableTransport(fpr, method, addr, port - j, {}) + transports.append(pt) + + bridge = Bridge(nick, addr, port, fpr) + bridge.flags.update("Running Stable") + bridge.transports = transports + bridge.orAddresses = addrs + bridges.append(bridge) + + return bridges + #: Mixin class for use with :api:`~twisted.trial.unittest.TestCase`. A #: ``TestCaseMixin`` can be used to add additional methods, which should be

1 0

[bridgedb/master] Reuse mocked Bridges across various unittests.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 7632085c0725e32001e41774564bffe7cb221584 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 22:18:26 2015 +0000 Reuse mocked Bridges across various unittests. --- lib/bridgedb/test/util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index f164aeb..2cddc11 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -168,6 +168,8 @@ randomValidIPv4String = valid(randomIPv4String) randomValidIPv6String = valid(randomIPv6String) randomValidIPString = valid(randomIPString) +_FAKE_BRIDGES = [] + def generateFakeBridges(n=500): """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random data. @@ -175,6 +177,11 @@ def generateFakeBridges(n=500): from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport + global _FAKE_BRIDGES + + if _FAKE_BRIDGES: + return _FAKE_BRIDGES + bridges = [] for i in range(n): @@ -199,6 +206,7 @@ def generateFakeBridges(n=500): bridge.orAddresses = addrs bridges.append(bridge) + _FAKE_BRIDGES = bridges return bridges

1 0

[bridgedb/master] Rename in EmailBasedDistributor.getBridges() `epoch` → `interval`.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 53732c492d29f094d17530bf60d740d957b21eab Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 9 05:03:37 2015 +0000 Rename in EmailBasedDistributor.getBridges() `epoch` → `interval`. --- lib/bridgedb/Dist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 3f7835b..69d9eb6 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -448,7 +448,7 @@ class EmailBasedDistributor(Distributor): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def getBridges(self, bridgeRequest, epoch, N=1): + def getBridges(self, bridgeRequest, interval, N=1): """Return a list of bridges to give to a user. :type bridgeRequest: :class:`~bridgedb.email.request.EmailBridgeRequest` @@ -456,7 +456,7 @@ class EmailBasedDistributor(Distributor): with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's full, canonicalized email address. - :param epoch: The time period when we got this request. This can be + :param interval: The time period when we got this request. This can be any string, so long as it changes with every period. :param int N: The number of bridges to try to give back. """ @@ -501,7 +501,7 @@ class EmailBasedDistributor(Distributor): elif wasWarned: db.setWarnedEmail(bridgeRequest.client, False) - pos = self.emailHmac("<%s>%s" % (epoch, bridgeRequest.client)) + pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) ring = None ruleset = frozenset(bridgeRequest.filters)

1 0

[bridgedb/master] Separate BridgeHolder class from Distributor.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit de555a5908d18cbb04c34afa88a7138ab204753f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 9 06:35:27 2015 +0000 Separate BridgeHolder class from Distributor. * FIXES part of #12506: https://bugs.torproject.org/12506 --- lib/bridgedb/Bridges.py | 1 - lib/bridgedb/Dist.py | 3 ++- lib/bridgedb/test/test_Main.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index d9b28e2..c727c9a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -449,7 +449,6 @@ class BridgeSplitter(BridgeHolder): p -- the relative proportion of bridges to assign to this bridgeholder. """ - assert isinstance(ring, BridgeHolder) self.ringsByName[ringname] = ring self.pValues.append(self.totalP) self.rings.append(ringname) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 69d9eb6..954b570 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -92,7 +92,8 @@ def getNumBridgesPerAnswer(ring, max_bridges_per_answer=3): return n_bridges_per_answer -class Distributor(bridgedb.Bridges.BridgeHolder): + +class Distributor(object): """Distributes bridges to clients.""" def __init__(self): diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 5ae4025..9c8b68c 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -27,7 +27,6 @@ from twisted.internet.threads import deferToThread from twisted.trial import unittest from bridgedb import Main -from bridgedb.Bridges import BridgeHolder from bridgedb.parse.options import parseOptions @@ -59,13 +58,17 @@ def mockUpdateBridgeHistory(bridges, timestamps): (fingerprint, timestamp)) -class MockBridgeHolder(BridgeHolder): +class MockBridgeHolder(object): def __init__(self): self._bridges = {} def __len__(self): return len(self._bridges.keys()) def insert(self, bridge): self._bridges[bridge.fingerprint] = bridge + def clear(self): + pass + def dumpAssignments(self): + pass class MainTests(unittest.TestCase):

1 0

[bridgedb/master] Rename ipCategories → proxySets.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 423f75d309abe304297cad2449e9a210347877da Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Apr 10 07:37:24 2015 +0000 Rename ipCategories → proxySets. --- lib/bridgedb/Dist.py | 16 ++++++++-------- lib/bridgedb/test/test_HTTPServer.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index f9f61de..2f63574 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -134,7 +134,7 @@ class IPBasedDistributor(Distributor): hashrings, one for each area in the ``areaMapper``. Every inserted bridge will go into one of these rings, and every area is associated with one. - :ivar categories: DOCDOC See :param:`ipCategories`. + :ivar categories: DOCDOC See :param:`proxySets`. :type splitter: :class:`bridgedb.Bridges.FixedBridgeSplitter` :ivar splitter: A hashring that assigns bridges to subrings with fixed proportions. Used to assign bridges into the subrings of this @@ -142,7 +142,7 @@ class IPBasedDistributor(Distributor): """ def __init__(self, areaMapper, numberOfClusters, key, - ipCategories=None, answerParameters=None): + proxySets=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. @@ -160,8 +160,8 @@ class IPBasedDistributor(Distributor): :param bytes key: The master HMAC key for this distributor. All added bridges are HMACed with this key in order to place them into the hashrings. - :type ipCategories: iterable or None - :param ipCategories: DOCDOC + :type proxySets: iterable or None + :param proxySets: DOCDOC :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` :param answerParameters: A mechanism for ensuring that the set of bridges that this distributor answers a client with fit certain @@ -172,14 +172,14 @@ class IPBasedDistributor(Distributor): self.numberOfClusters = numberOfClusters self.answerParameters = answerParameters - if not ipCategories: - ipCategories = [] + if not proxySets: + proxySets = [] if not answerParameters: answerParameters = [] self.rings = [] self.categories = [] - for c in ipCategories: + for c in proxySets: self.categories.append(c) key2 = getHMAC(key, "Assign-Bridges-To-Rings") @@ -193,7 +193,7 @@ class IPBasedDistributor(Distributor): # # XXX Why is the "extra room" hardcoded to be 5? Shouldn't it be some # fraction of the number of clusters/categories? --isis - ring_cache_size = self.numberOfClusters + len(ipCategories) + 5 + ring_cache_size = self.numberOfClusters + len(proxySets) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=ring_cache_size) logging.debug("Added splitter %s to IPBasedDistributor." diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index c3cf79a..103976e 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -488,7 +488,7 @@ class DummyIPBasedDistributor(object): def _dumbAreaMapper(ip): return ip def __init__(self, areaMapper=None, numberOfClusters=None, key=None, - ipCategories=None, answerParameters=None): + proxySets=None, answerParameters=None): """None of the parameters are really used, they are just there to retain an identical method signature. """ @@ -496,7 +496,7 @@ class DummyIPBasedDistributor(object): self.numberOfClusters = 3 self.nBridgesToGive = 3 self.key = self.__class__.__name__ - self.ipCategories = ipCategories + self.proxySets = proxySets self.answerParameters = answerParameters def getBridges(self, bridgeRequest=None, epoch=None, N=1):

1 0

[bridgedb/master] Why were there ever unreachable subhashrings? Kill them with fire.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit a45591af98741f5b4d7cf9a6cd5fbd1603c3bfe8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 9 06:44:43 2015 +0000 Why were there ever unreachable subhashrings? Kill them with fire. As part of #4297, $SOMEONE added the prepopulateRings() methods, which are used when BridgeDB parses incoming descriptors. These methods add subhashrings and allocate bridges to them. When a client requests bridges, they are either requesting IPv4 or IPv6 bridges, and thus clients *always* have either filterBridgesByIPv4 or filterBridgesByIPv6, respectively. So why is there an extra subhashring with no IP version filters if clients *can never get to it*? Blame $SOMEONE. The $SOMEONE who authored the commit below would be a good start. * BUGFIX on e6ce57e728802689544c130867edcadfeecd38ec. * REMOVE the "filterless" subhashring from the IPBasedDistributor. * UPDATE documentation for IPBasedDistributor.prepopulateRings(). --- lib/bridgedb/Dist.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 954b570..0658504 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -233,33 +233,25 @@ class IPBasedDistributor(Distributor): | Subhashrings | | | | | | | | (total, assigned)| (6,0) | (6,1) | (6,2) | (6,3) | (6,4) | (6,5) | +------------------+------------+------------+------------+------------+------------+------------+ - | Filtered | (6,0) | (6,1) | (6,2) | (6,3) | (6,4) | (6,5) | - | Subhashrings +------------+------------+------------+------------+------------+------------+ - | bBy requested | (6,0)-IPv4 | (6,1)-IPv4 | (6,2)-IPv4 | (6,3)-IPv4 | (6,4)-IPv4 | (6,5)-IPv4 | - | bridge type) +------------+------------+------------+------------+------------+------------+ - | | (6,0)-IPv6 | (6,1)-IPv6 | (6,2)-IPv6 | (6,3)-IPv6 | (6,4)-IPv6 | (6,5)-IPv6 | + | Filtered | (6,0)-IPv4 | (6,1)-IPv4 | (6,2)-IPv4 | (6,3)-IPv4 | (6,4)-IPv4 | (6,5)-IPv4 | + | Subhashrings | | | | | | | + | bBy requested +------------+------------+------------+------------+------------+------------+ + | bridge type) | (6,0)-IPv6 | (6,1)-IPv6 | (6,2)-IPv6 | (6,3)-IPv6 | (6,4)-IPv6 | (6,5)-IPv6 | + | | | | | | | | +------------------+------------+------------+------------+------------+------------+------------+ - The "filtered subhashrings" are essentially copies of their respective - subhashring, that is, subhashring ``(6,0)`` contains both IPv4 and - IPv6 bridges, meaning that its contents are a superset of the filtered - subhashrings ``(6,0)-IPv4`` and ``(6,0)-IPv6``. (I have no idea of - the relation between ``(6,0)-IPv4`` and ``(6,0)-IPv6``, including - whether or not their contents are disjoint. I didn't design this shit, - I'm just redesigning it.) - - "Why does the ``(6,0)`` superset subhashring exist then?" + The "filtered subhashrings" are essentially filtered copies of their + respective subhashring, such that they only contain bridges which + support IPv4 or IPv6, respectively. (I have no idea of the relation + between ``(6,0)-IPv4`` and ``(6,0)-IPv6``, including whether or not + their contents are disjoint. I didn't design this shit, I'm just + redesigning it.) - you might ask. That's a very good question. I don't know either. - I'm inclined to think it shouldn't exist, unless we wish to allow - clients to request IPv4 bridges and IPv6 bridges simultaneously - (there's currently no interface to do this, however). - - Thus, in this example, we end up with **18 total subhashrings**. + Thus, in this example, we end up with **12 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) # populate all rings (for dumping assignments and testing) - for filterFn in [None, filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: n = self.nClusters for category in self.categories: g = filterAssignBridgesToRing(self.splitter.hmac,

1 0

[bridgedb/master] Remove IPBasedDistributor.clear() and EmailBasedDistributor.clear().
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit de631604e282c92bc3f54e56cfa0faa3416503b4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Apr 10 07:14:12 2015 +0000 Remove IPBasedDistributor.clear() and EmailBasedDistributor.clear(). These were never used anywhere. --- lib/bridgedb/Dist.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 127004a..770198a 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -268,9 +268,6 @@ class IPBasedDistributor(Distributor): filterBridgesByRules(filters), populate_from=self.splitter.bridges) - def clear(self): - self.splitter.clear() - def insert(self, bridge): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) @@ -410,10 +407,6 @@ class EmailBasedDistributor(Distributor): self.setDistributorName('Email') - def clear(self): - self.splitter.clear() - #self.ring.clear() # should be take care of by above - def insert(self, bridge): """Assign a bridge to this distributor.""" self.splitter.insert(bridge)

1 0

[bridgedb/master] Remove the idea of "categories". Proxies constitute another cluster.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 9ba68b455a8adeaef34785ecccbe7ac5a5d5ac26 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Apr 11 01:10:05 2015 +0000 Remove the idea of "categories". Proxies constitute another cluster. This completely removes the half-baked idea of "IP categories" from BridgeDB's codebase. An IPBasedDistributor now holds a bunch of proxies (in one single bridgedb.proxy.ProxySet). These proxies have their own "cluster". End of story. No more messing around with both "categories" and "clusters", because nobody ever knew what either of those were anyway, much less why they were different. * CHANGE IPBasedDistributor.categories to be a single bridgedb.proxy.ProxySet, rather than a list of lists. ProxySets have "tags" for each proxy within them, and can efficiently separate proxies based upon tag(s), so there is no longer any need to have a list of lists to keep the proxies separate. Besides, in the end, we treat all clients coming from any proxy the same, regardless of whether the proxy is a Tor Exit relay or some other known open proxy. * RENAME IPBasedDistributor.categories → IPBasedDistributor.proxies, because no one ever understood what an "IP category" was. * CHANGE IPBasedDistributor.numberOfClusters to always include the "cluster" for clients coming from proxies, if IPBasedDistributor.proxies is configured. * ADD IPBasedDistributor.proxyCluster, which is an integer pointing to the cluster number for the proxies cluster. If there is no cluster for proxies, it is set to 0. Otherwise, it points to the last cluster. * CHANGE the numbering of clusters in IPBasedDistributor to start from 1 instead of 0. While this might make iteration over all of the clusters increasingly subject to fencepost errors, it still makes the code feel more intuitive, i.e. "The first cluster is 1 out of 5 total clusters, and the last cluster is 5/5" rather than "The first cluster is number 0 out of 5 total clusters, and the last 4/5". * CHANGE a couple test cases which expected IPBasedDistributor.categories to exist. * UPDATE documentation pertaining to all of the above, as well as the documentation on (sub)hashring structure. --- lib/bridgedb/Dist.py | 172 +++++++++++++++++++------------------ lib/bridgedb/Main.py | 41 +++------ lib/bridgedb/test/legacy_Tests.py | 11 ++- lib/bridgedb/test/test_Main.py | 4 +- 4 files changed, 107 insertions(+), 121 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 2f63574..ab7ae6e 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -20,6 +20,7 @@ import time import bridgedb.Bridges import bridgedb.Storage +from bridgedb import proxy from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.Filters import filterAssignBridgesToRing @@ -134,7 +135,9 @@ class IPBasedDistributor(Distributor): hashrings, one for each area in the ``areaMapper``. Every inserted bridge will go into one of these rings, and every area is associated with one. - :ivar categories: DOCDOC See :param:`proxySets`. + :type proxies: :class:`~bridgedb.proxies.ProxySet` + :ivar proxies: All known proxies, which we treat differently. See + :param:`proxies`. :type splitter: :class:`bridgedb.Bridges.FixedBridgeSplitter` :ivar splitter: A hashring that assigns bridges to subrings with fixed proportions. Used to assign bridges into the subrings of this @@ -142,7 +145,7 @@ class IPBasedDistributor(Distributor): """ def __init__(self, areaMapper, numberOfClusters, key, - proxySets=None, answerParameters=None): + proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. @@ -160,27 +163,33 @@ class IPBasedDistributor(Distributor): :param bytes key: The master HMAC key for this distributor. All added bridges are HMACed with this key in order to place them into the hashrings. - :type proxySets: iterable or None - :param proxySets: DOCDOC + :type proxies: :class:`~bridgedb.proxy.ProxySet` + :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known + Tor Exit relays and other known proxies. These will constitute + the extra cluster, and any client requesting bridges from one of + these **proxies** will be distributed bridges from a separate + subhashring that is specific to Tor/proxy users. :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` :param answerParameters: A mechanism for ensuring that the set of bridges that this distributor answers a client with fit certain parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ + self.rings = [] self.areaMapper = areaMapper - self.numberOfClusters = numberOfClusters self.answerParameters = answerParameters + self.numberOfClusters = numberOfClusters - if not proxySets: - proxySets = [] - if not answerParameters: - answerParameters = [] - self.rings = [] + if proxies: + logging.info("Added known proxies to HTTPS distributor...") + self.proxies = proxies + self.numberOfClusters += 1 + self.proxyCluster = self.numberOfClusters + else: + logging.warn("No known proxies were added to HTTPS distributor!") + self.proxies = proxy.ProxySet() + self.proxyCluster = 0 - self.categories = [] - for c in proxySets: - self.categories.append(c) key2 = getHMAC(key, "Assign-Bridges-To-Rings") key3 = getHMAC(key, "Order-Areas-In-Rings") @@ -193,7 +202,7 @@ class IPBasedDistributor(Distributor): # # XXX Why is the "extra room" hardcoded to be 5? Shouldn't it be some # fraction of the number of clusters/categories? --isis - ring_cache_size = self.numberOfClusters + len(proxySets) + 5 + ring_cache_size = self.numberOfClusters + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=ring_cache_size) logging.debug("Added splitter %s to IPBasedDistributor." @@ -207,63 +216,59 @@ class IPBasedDistributor(Distributor): The hashring structure for this distributor is influenced by the ``N_IP_CLUSTERS`` configuration option, as well as the number of - ``PROXY_LIST_FILES``. Essentially, :data:`numberOfClusters` is set to the - specified ``N_IP_CLUSTERS``. The ``PROXY_LIST_FILES`` (plus the - :class:`bridgedb.proxy.ProxySet` for the Tor Exit list downloaded into - memory with :script:`get-tor-exits`) are stored in :data:`categories`. - - The number of subhashrings which this :class:`Distributor` has active - in its hashring is then the :data:`numberOfClusters` plus the number of - :data:`categories`. + ``PROXY_LIST_FILES``. + + Essentially, :data:`numberOfClusters` is set to the specified + ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of + Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), + are stored in :data:`proxies`, and the latter is added as an + additional cluster (such that :data:`numberOfClusters` becomes + ``N_IP_CLUSTERS + 1``). The number of subhashrings which this + :class:`Distributor` has active in its hashring is then + :data:`numberOfClusters`, where the last cluster is reserved for all + :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number - of subhashrings is six — four for the "clusters", and two - "categories": one for everything contained within the - ``"open-socks-proxies.txt"`` file and the other for the downloaded - list of Tor Exits. Thus, the resulting hashring-subhashring structure + of subhashrings is five — four for the "clusters", and one for the + :data:`proxies`. Thus, the resulting hashring-subhashring structure would look like: - +------------------+---------------------------------------------------+-------------------------+ - | | Directly connecting users | Tor / known proxy users | - +------------------+------------+------------+------------+------------+------------+------------+ - | Clusters / | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cat-1 | Cat-2 | - | Categories | | | | | | | - +==================+============+============+============+============+============+============+ - | Subhashrings | | | | | | | - | (total, assigned)| (6,0) | (6,1) | (6,2) | (6,3) | (6,4) | (6,5) | - +------------------+------------+------------+------------+------------+------------+------------+ - | Filtered | (6,0)-IPv4 | (6,1)-IPv4 | (6,2)-IPv4 | (6,3)-IPv4 | (6,4)-IPv4 | (6,5)-IPv4 | - | Subhashrings | | | | | | | - | bBy requested +------------+------------+------------+------------+------------+------------+ - | bridge type) | (6,0)-IPv6 | (6,1)-IPv6 | (6,2)-IPv6 | (6,3)-IPv6 | (6,4)-IPv6 | (6,5)-IPv6 | - | | | | | | | | - +------------------+------------+------------+------------+------------+------------+------------+ + +------------------+---------------------------------------------------+-------------- + | | Directly connecting users | Tor / known | + | | | proxy users | + +------------------+------------+------------+------------+------------+-------------+ + | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | + +==================+============+============+============+============+=============+ + | Subhashrings | | | | | | + | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | + +------------------+------------+------------+------------+------------+-------------+ + | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | + | Subhashrings | | | | | | + | bBy requested +------------+------------+------------+------------+-------------+ + | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | + | | | | | | | + +------------------+------------+------------+------------+------------+-------------+ The "filtered subhashrings" are essentially filtered copies of their respective subhashring, such that they only contain bridges which - support IPv4 or IPv6, respectively. (I have no idea of the relation - between ``(6,0)-IPv4`` and ``(6,0)-IPv6``, including whether or not - their contents are disjoint. I didn't design this shit, I'm just - redesigning it.) + support IPv4 or IPv6, respectively. Additionally, the contents of + ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. - Thus, in this example, we end up with **12 total subhashrings**. + Thus, in this example, we end up with **10 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: - # XXX Distributors should have a "totalClusters" property in order - # to avoid reusing this unclear construct all over the place. (Or - # just get rid of the idea of "categories".) - for cluster in range(self.numberOfClusters + len(self.categories)): + for cluster in range(1, self.numberOfClusters): filters = self._buildHashringFilters([filterFn,], cluster) key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, - # only set the "name" for "clusters" which are in this - # distributor's categories: - if cluster >= self.numberOfClusters: - ring.setName('{0} Ring'.format(self.name)) + # only set the "name" for "clusters" which are for this + # distributor's proxies: + if cluster == self.proxyCluster: + ring.setName('{0} Proxy Ring'.format(self.name)) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) @@ -273,8 +278,8 @@ class IPBasedDistributor(Distributor): self.splitter.insert(bridge) def _buildHashringFilters(self, previousFilters, clientCluster): - totalRings = self.numberOfClusters + len(self.categories) - g = filterAssignBridgesToRing(self.splitter.hmac, totalRings, clientCluster) + g = filterAssignBridgesToRing(self.splitter.hmac, + self.numberOfClusters, clientCluster) previousFilters.append(g) return frozenset(previousFilters) @@ -301,41 +306,38 @@ class IPBasedDistributor(Distributor): logging.warn("Bailing! Splitter has zero bridges!") return [] - # The cluster the client should draw bridges from: - clientCluster = self.numberOfClusters - # First, check if the client's IP is one of the known proxies in one - # of our :data:`catagories`: - for category in self.categories: - if bridgeRequest.client in category: - # The tag is a tag applied to a proxy IP address when it is - # added to the bridgedb.proxy.ProxySet. For Tor Exit relays, - # the default is 'exit_relay'. For other proxies loaded from - # the PROXY_LIST_FILES config option, the default tag is the - # full filename that the IP address originally came from. - tag = category.getTag(bridgeRequest.client) - logging.info("Client was from known proxy (tag: %s): %s" % - (tag, bridgeRequest.client)) - # Cluster Tor/proxy users into four groups. This means that - # no matter how many different Tor Exits or proxies a client - # uses, the most they can ever get is four different sets of - # bridge lines (per period). - group = (int(ipaddr.IPAddress(bridgeRequest.client)) % 4) + 1 - area = "known-proxy-group-%d" % group - break - clientCluster += 1 - # If the client wasn't using Tor or any other known proxy, select the - # client's cluster number based upon the /16 of the client's IP - # address: + # First, check if the client's IP is one of the known :data:`proxies`: + if bridgeRequest.client in self.proxies: + cluster = self.proxyCluster + # The tag is a tag applied to a proxy IP address when it is added + # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default + # is 'exit_relay'. For other proxies loaded from the + # PROXY_LIST_FILES config option, the default tag is the full + # filename that the IP address originally came from. + tag = self.proxies.getTag(bridgeRequest.client) + logging.info("Client was from known proxy (tag: %s): %s" % + (tag, bridgeRequest.client)) + # Place Tor/proxy users into four groups. This means that no + # matter how many different Tor Exits or proxies a client + # uses, the most they can ever get is four different sets of + # bridge lines (per period). + group = (int(ipaddr.IPAddress(bridgeRequest.client)) % 4) + 1 + area = "known-proxy-group-%d" % group + # If the client wasn't using a proxy, select the client's cluster + # based upon the client's area (i.e. the /16 of the client's IP + # address): else: # Areas (i.e. /16s) are grouped into the number of rings specified # by the N_IP_CLUSTERS configuration option. area = self.areaMapper(bridgeRequest.client) - logging.debug("IP mapped to area:\t%s" % area) - clientCluster = int(self.areaClusterHmac(area)[:8], 16) % self.numberOfClusters + cluster = (int(self.areaClusterHmac(area)[:8], 16) + % (self.numberOfClusters - 1)) pos = self.areaOrderHmac("<%s>%s" % (interval, area)) - filters = self._buildHashringFilters(bridgeRequest.filters, clientCluster) + filters = self._buildHashringFilters(bridgeRequest.filters, cluster) + logging.debug("Assigned client to cluster %d/%d" % + (cluster, self.numberOfClusters)) logging.debug("Assigned client hashring position based on: <%s>%s" % (interval, area)) logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) @@ -349,7 +351,7 @@ class IPBasedDistributor(Distributor): # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % clientCluster) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index c2e44ca..23ed262 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -210,17 +210,11 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") - categories = [] - if proxyList: - logging.debug("Adding proxyList to HTTPS Distributor categories.") - categories.append(proxyList) - logging.debug("HTTPS Distributor categories: '%s'" % categories) - ipDistributor = Dist.IPBasedDistributor( Dist.uniformMap, cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), - categories, + proxyList, answerParameters=ringParams) splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) @@ -306,17 +300,12 @@ def run(options, reactor=reactor): # Load the master key, or create a new one. key = crypto.getKey(config.MASTER_KEY_FILE) - - # Get a proxy list. - proxyList = proxy.ProxySet() - for proxyfile in config.PROXY_LIST_FILES: - logging.info("Loading proxies from: %s" % proxyfile) - proxy.loadProxiesFromFile(proxyfile, proxyList) - - emailDistributor = ipDistributor = None + proxies = proxy.ProxySet() + emailDistributor = None + ipDistributor = None # Save our state - state.proxyList = proxyList + state.proxies = proxies state.key = key state.save() @@ -363,26 +352,22 @@ def run(options, reactor=reactor): level = getattr(logging, level) logging.getLogger().setLevel(level) - logging.debug("Saving state again before reparsing descriptors...") - state.save() - logging.info("Reparsing bridge descriptors...") + logging.info("Reloading the list of open proxies...") + for proxyfile in cfg.PROXY_LIST_FILES: + logging.info("Loading proxies from: %s" % proxyfile) + proxy.loadProxiesFromFile(proxyfile, state.proxies, removeStale=True) + logging.info("Reparsing bridge descriptors...") (splitter, emailDistributorTmp, - ipDistributorTmp) = createBridgeRings(cfg, proxyList, key) + ipDistributorTmp) = createBridgeRings(cfg, state.proxies, key) + logging.info("Bridges loaded: %d" % len(splitter)) # Initialize our DB. bridgedb.Storage.initializeDBLock() bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") load(state, splitter, clear=False) - state = persistent.load() - logging.info("Bridges loaded: %d" % len(splitter)) - - logging.debug("Replacing the list of open proxies...") - for proxyfile in cfg.PROXY_LIST_FILES: - proxy.loadProxiesFromFile(proxyfile, state.proxyList, removeStale=True) - if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings logging.info("Bridges allotted for %s distribution: %d" @@ -462,7 +447,7 @@ def run(options, reactor=reactor): if config.TASKS['GET_TOR_EXIT_LIST']: tasks['GET_TOR_EXIT_LIST'] = task.LoopingCall( proxy.downloadTorExits, - proxyList, + state.proxies, config.SERVER_PUBLIC_EXTERNAL_IP) # Schedule all configured repeating tasks: diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 97134e4..7d7d236 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -143,8 +143,8 @@ simpleDesc = "router Unnamed %s %s 0 9030\n"\ orAddress = "or-address %s:%s\n" -class RhymesWith255Category: - def contains(self, ip): +class RhymesWith255ProxySet: + def __contains__(self, ip): return ip.endswith(".255") class EmailBridgeDistTests(unittest.TestCase): @@ -194,15 +194,14 @@ class IPBridgeDistTests(unittest.TestCase): n2 = d.getBridges("1.2.3.4", "x", 2) self.assertEquals(n, n2) - def testDistWithCategories(self): + def testDistWithProxies(self): d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo", - [RhymesWith255Category()]) - assert len(d.categories) == 1 + [RhymesWith255ProxySet()]) for _ in xrange(256): d.insert(fakeBridge()) for _ in xrange(256): - # Make sure that the categories do not overlap + # Make sure that the ProxySets do not overlap f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) n = d.getBridges(g(), "x", 10) diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 9c8b68c..43e008c 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -278,8 +278,8 @@ class MainTests(unittest.TestCase): # Should have an IPBasedDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: self.assertEqual(len(splitter.ringsByName.keys()), 3) - self.assertGreater(len(httpsDist.categories), 0) - self.assertItemsEqual(exitRelays, httpsDist.categories[-1]) + self.assertGreater(len(httpsDist.proxies), 0) + self.assertItemsEqual(exitRelays, httpsDist.proxies) def test_Main_createBridgeRings_no_https_dist(self): """When HTTPS_DIST=False, Main.createBridgeRings() should add only

1 0

[bridgedb/master] Make subhashring cache size relative to numberOfClusters.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 397cc7d316a618bcadfca6663f318d3b798f4405 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Apr 11 02:53:20 2015 +0000 Make subhashring cache size relative to numberOfClusters. * RENAME ring_cache_size → IPBasedDistributor.ringCacheSize. * CHANGE IPBasedDistributor.ringCacheSize to be relative to the total number of clusters, rather than equal to the total number of clusters plus five. --- lib/bridgedb/Dist.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 129bc7c..1a01543 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -21,6 +21,7 @@ import bridgedb.Bridges import bridgedb.Storage from bridgedb import proxy +from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.Filters import filterAssignBridgesToRing @@ -193,23 +194,17 @@ class IPBasedDistributor(Distributor): self.proxies = proxy.ProxySet() self.proxyCluster = 0 + self.ringCacheSize = self.numberOfClusters * 3 key2 = getHMAC(key, "Assign-Bridges-To-Rings") key3 = getHMAC(key, "Order-Areas-In-Rings") - self.areaOrderHmac = getHMACFunc(key3, hex=False) key4 = getHMAC(key, "Assign-Areas-To-Rings") - self.areaClusterHmac = getHMACFunc(key4, hex=True) - # add splitter and cache the default rings - # plus leave room for dynamic filters - # - # XXX Why is the "extra room" hardcoded to be 5? Shouldn't it be some - # fraction of the number of clusters/categories? --isis - ring_cache_size = self.numberOfClusters + 5 - self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( - key2, max_cached_rings=ring_cache_size) - logging.debug("Added splitter %s to IPBasedDistributor." - % self.splitter.__class__) + self.areaOrderHmac = getHMACFunc(key3, hex=False) + self.areaClusterHmac = getHMACFunc(key4, hex=True) + self.splitter = FilteredBridgeSplitter(key2, self.ringCacheSize) + logging.debug("Added %s to HTTPS distributor." % + self.splitter.__class__.__name__) self.setDistributorName('HTTPS')

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits July 2015