commit 9c09ef7bdb335fd58d48d8f8ff6d79d41fbd088c Author: Isis Lovecruft <isis@torproject.org> Date: Tue Apr 21 05:09:32 2015 +0000 Refactor bridgedb.Filters and move it to bridgedb.filters. --- doc/sphinx/source/bridgedb.Filters.rst | 8 - doc/sphinx/source/bridgedb.filters.rst | 8 + doc/sphinx/source/bridgedb.rst | 2 +- doc/sphinx/source/conf.py | 2 +- lib/bridgedb/Bridges.py | 12 +- lib/bridgedb/Dist.py | 25 ++- lib/bridgedb/Filters.py | 175 ----------------- lib/bridgedb/bridgerequest.py | 39 ++-- lib/bridgedb/distribute.py | 2 +- lib/bridgedb/filters.py | 238 +++++++++++++++++++++++ lib/bridgedb/https/server.py | 4 - lib/bridgedb/parse/addr.py | 6 +- lib/bridgedb/persistent.py | 12 +- lib/bridgedb/test/legacy_Tests.py | 5 - lib/bridgedb/test/test_Dist.py | 17 +- lib/bridgedb/test/test_filters.py | 333 ++++++++++++++++++++++++++++++++ 16 files changed, 638 insertions(+), 250 deletions(-) diff --git a/doc/sphinx/source/bridgedb.Filters.rst b/doc/sphinx/source/bridgedb.Filters.rst deleted file mode 100644 index caf8dfc..0000000 --- a/doc/sphinx/source/bridgedb.Filters.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.Filters ----------------- - -.. automodule:: bridgedb.Filters - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.filters.rst b/doc/sphinx/source/bridgedb.filters.rst new file mode 100644 index 0000000..9c14ce5 --- /dev/null +++ b/doc/sphinx/source/bridgedb.filters.rst @@ -0,0 +1,8 @@ +bridgedb.filters +---------------- + +.. automodule:: bridgedb.filters + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 31e3a90..7b1fef7 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -15,7 +15,7 @@ BridgeDB Package and Module Documentation bridgedb.crypto bridgedb.Dist bridgedb.email - bridgedb.Filters + bridgedb.filters bridgedb.geo bridgedb.https bridgedb.interfaces diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index b655c27..9b43d0f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -40,7 +40,7 @@ import bridgedb.email.dkim import bridgedb.email.request import bridgedb.email.server import bridgedb.email.templates -import bridgedb.Filters +import bridgedb.filters import bridgedb.geo import bridgedb.https import bridgedb.https.request diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 507013d..2b56884 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -569,7 +569,7 @@ class FilteredBridgeSplitter(object): filterNames = [] for filterName in [x.func_name for x in list(ringname)]: - # Using `filterAssignBridgesToRing.func_name` gives us a messy + # Using `assignBridgesToSubring.func_name` gives us a messy # string which includes all parameters and memory addresses. Get # rid of this by partitioning at the first `(`: realFilterName = filterName.partition('(')[0] @@ -599,8 +599,8 @@ class FilteredBridgeSplitter(object): # hashring '%s'!" % (inserted, ringname))`, this log message appears: # # Jan 04 23:18:37 [INFO] Inserted 12 bridges into hashring - # frozenset([<function filterBridgesByIP4 at 0x2d67cf8>, <function - # filterAssignBridgesToRing(<function hmac_fn at 0x3778398>, 4, 0) at + # frozenset([<function byIPv4 at 0x2d67cf8>, <function + # assignBridgesToSubring(<function hmac_fn at 0x3778398>, 4, 0) at # 0x37de578>])! # # I suppose since it contains memory addresses, it *is* technically @@ -615,10 +615,10 @@ class FilteredBridgeSplitter(object): subringName = [self.distributorName] subringNumber = None for filterName in filterNames: - if filterName.startswith('filterAssignBridgesToRing'): - subringNumber = filterName.lstrip('filterAssignBridgesToRing') + if filterName.startswith('assignBridgesToSubring'): + subringNumber = filterName.lstrip('assignBridgesToSubring') else: - subringName.append(filterName.lstrip('filterBridgesBy')) + subringName.append(filterName.lstrip('by')) if subring.name and 'Proxy' in subring.name: subringName.append('Proxy') elif subringNumber: diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index fe6b8da..9b9e35c 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -25,10 +25,10 @@ from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.distribute import Distributor -from bridgedb.Filters import filterAssignBridgesToRing -from bridgedb.Filters import filterBridgesByRules -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byFilters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import bySubring from bridgedb.parse import addr @@ -259,7 +259,7 @@ class HTTPSDistributor(Distributor): """ logging.info("Prepopulating %s distributor hashrings..." % self.name) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) @@ -269,8 +269,7 @@ class HTTPSDistributor(Distributor): # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, - filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) def insert(self, bridge): @@ -278,7 +277,7 @@ class HTTPSDistributor(Distributor): self.hashring.insert(bridge) def _buildHashringFilters(self, previousFilters, subring): - f = filterAssignBridgesToRing(self.hashring.hmac, self.totalSubrings, subring) + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) previousFilters.append(f) return frozenset(previousFilters) @@ -337,7 +336,7 @@ class HTTPSDistributor(Distributor): logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: @@ -457,8 +456,7 @@ class EmailBasedDistributor(Distributor): # add new ring key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules(ruleset), + self.hashring.addRing(ring, ruleset, byFilters(ruleset), populate_from=self.hashring.bridges) returnNum = self.bridgesPerResponse(ring) @@ -482,10 +480,9 @@ class EmailBasedDistributor(Distributor): def prepopulateRings(self): # populate all rings (for dumping assignments and testing) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: ruleset = frozenset([filterFn]) key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules([filterFn]), + self.hashring.addRing(ring, ruleset, byFilters([filterFn]), populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py deleted file mode 100644 index d0d65e8..0000000 --- a/lib/bridgedb/Filters.py +++ /dev/null @@ -1,175 +0,0 @@ -# BridgeDB by Nick Mathewson. -# Copyright (c) 2007-2012, The Tor Project, Inc. -# See LICENSE for licensing information - -from ipaddr import IPv6Address, IPv4Address -import logging - -funcs = {} - -def filterAssignBridgesToRing(hmac, numRings, assignedRing): - logging.debug(("Creating a filter for assigning bridges to subhashring " - "%s-of-%s...") % (assignedRing, numRings)) - ruleset = frozenset([hmac, numRings, assignedRing]) - try: - return funcs[ruleset] - except KeyError: - def _assignBridgesToRing(bridge): - digest = hmac(bridge.identity) - pos = long( digest[:8], 16 ) - which = pos % numRings + 1 - - if which == assignedRing: - return True - return False - _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing%sof%s" - % (assignedRing, numRings)) - # XXX The `description` attribute must contain an `=`, or else - # dumpAssignments() will not work correctly. - setattr(_assignBridgesToRing, "description", "ring=%d" % assignedRing) - funcs[ruleset] = _assignBridgesToRing - return _assignBridgesToRing - -def filterBridgesByRules(rules): - ruleset = frozenset(rules) - try: - return funcs[ruleset] - except KeyError: - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - setattr(g, "description", " ".join([getattr(f,'description','') for f in rules])) - funcs[ruleset] = g - return g - -def filterBridgesByIP4(bridge): - try: - if IPv4Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 4: - return True - return False -setattr(filterBridgesByIP4, "description", "ip=4") - -def filterBridgesByIP6(bridge): - try: - if IPv6Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 6: - return True - return False -setattr(filterBridgesByIP6, "description", "ip=6") - -def filterBridgesByTransport(methodname, addressClass=None): - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - - ruleset = frozenset([methodname, addressClass]) - try: - return funcs[ruleset] - except KeyError: - def _filterByTransport(bridge): - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" - % (methodname, addressClass)) - setattr(_filterByTransport, "description", "transport=%s" % methodname) - funcs[ruleset] = _filterByTransport - return _filterByTransport - -def filterBridgesByUnblockedTransport(methodname, countryCode=None, addressClass=None): - """Return a filter function for :class:`~bridgedb.bridges.Bridge`s. - - The returned filter function should be called on a - :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` - has a :class:`~bridgedb.bridges.PluggableTransport` such that: - - 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches - **methodname**, - - 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an - instance of **addressClass**, and isn't known to be blocked in - **countryCode**. - - :param str methodname: A Pluggable Transport - :data:`~bridge.bridges.PluggableTransport.methodname`. - :type countryCode: str or ``None`` - :param countryCode: A two-letter country code which the filtered - :class:`PluggableTransport`s should not be blocked in. - :type addressClass: ``ipaddr.IPAddress`` - :param addressClass: The IP version that the ``Bridge``'s - ``PluggableTransport`` - :data:`~bridgedb.bridges.PluggableTransport.address`` should have. - :rtype: callable - :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. - """ - if not countryCode: - return filterBridgesByTransport(methodname, addressClass) - - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - countryCode = countryCode.lower() - - ruleset = frozenset([methodname, countryCode, addressClass.__name__]) - try: - return funcs[ruleset] - except KeyError: - def _filterByUnblockedTransport(bridge): - # Since bridge.transportIsBlockedIn() will return True if the - # bridge has that type of transport AND that transport is blocked, - # we can "fail fast" here by doing this faster check before - # iterating over all the transports testing for the other - # conditions. - if bridge.transportIsBlockedIn(countryCode, methodname): - return False - else: - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByUnblockedTransport.__name__ = ("filterBridgesByUnblockedTransport(%s,%s,%s)" - % (methodname, countryCode, addressClass)) - setattr(_filterByUnblockedTransport, "description", - "transport=%s unblocked=%s" % (methodname, countryCode)) - funcs[ruleset] = _filterByUnblockedTransport - return _filterByUnblockedTransport - -def filterBridgesByNotBlockedIn(countryCode): - """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't - known to be blocked in **countryCode**. - - :param str countryCode: A two-letter country code. - :rtype: bool - :returns: ``True`` if at least one address of the bridge isn't blocked. - ``False`` otherwise. - """ - countryCode = countryCode.lower() - ruleset = frozenset([countryCode]) - try: - return funcs[ruleset] - except KeyError: - def _filterByNotBlockedIn(bridge): - if bridge.isBlockedIn(countryCode): - return False - return True - _filterByNotBlockedIn.__name__ = "filterBridgesByNotBlockedIn(%s)" % countryCode - setattr(_filterByNotBlockedIn, "description", "unblocked=%s" % countryCode) - funcs[ruleset] = _filterByNotBlockedIn - return _filterByNotBlockedIn diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index b699896..2ded7b3 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -11,16 +11,17 @@ #_____________________________________________________________________________ -import logging - import ipaddr +import logging from zope.interface import implements from zope.interface import Attribute from zope.interface import Interface -from bridgedb import Filters from bridgedb.crypto import getHMACFunc +from bridgedb.filters import byIPv +from bridgedb.filters import byNotBlockedIn +from bridgedb.filters import byTransport class IRequestBridges(Interface): @@ -173,20 +174,22 @@ class BridgeRequestBase(object): def generateFilters(self): self.clearFilters() - transport = self.justOnePTType() + pt = self.justOnePTType() + msg = ("Adding a filter to %s for %s for IPv%s" + % (self.__class__.__name__, self.client, self.addressClass)) - if transport: - if self.notBlockedIn: - for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByUnblockedTransport( - transport, country, self.addressClass)) - else: - self.addFilter(Filters.filterBridgesByTransport( - transport, self.addressClass)) - else: - if self.addressClass is ipaddr.IPv6Address: - self.addFilter(Filters.filterBridgesByIP6) - else: - self.addFilter(Filters.filterBridgesByIP4) + self.ipVersion = 4 + if self.addressClass is ipaddr.IPv6Address: + self.ipVersion = 6 + + if self.notBlockedIn: for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) + logging.info("%s %s bridges not blocked in %s..." % + (msg, pt or "vanilla", country)) + self.addFilter(byNotBlockedIn(country, pt, self.addressClass)) + elif pt: + logging.info("%s %s bridges..." % (msg, pt)) + self.addFilter(byTransport(pt, self.addressClass)) + else: + logging.info("%s bridges..." % msg) + self.addFilter(byIPv(self.addressClass)) diff --git a/lib/bridgedb/distribute.py b/lib/bridgedb/distribute.py index c7c9044..f48cbb6 100644 --- a/lib/bridgedb/distribute.py +++ b/lib/bridgedb/distribute.py @@ -47,7 +47,7 @@ DistributorContext { # should go in bridgedb.py } Hashring { - assignBridgesToRings() FORMERLY filterAssignBridgesToRing() + assignBridgesToSubrings() FORMERLY bridgedb.filters.assignBridgesToSubring() + filters bridges uniformly into subrings clear() / __del__() isEmpty property diff --git a/lib/bridgedb/filters.py b/lib/bridgedb/filters.py new file mode 100644 index 0000000..8843937 --- /dev/null +++ b/lib/bridgedb/filters.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_filters ; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson <nickm@torproject.org> +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# (c) 2013-2015, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +import logging + +from ipaddr import IPv4Address +from ipaddr import IPv6Address + +from bridgedb.parse.addr import isIPv + + +_cache = {} + + +def bySubring(hmac, assigned, total): + """Create a filter function which filters for only the bridges which fall + into the same **assigned** subhashring (based on the results of an **hmac** + function). + + :type hmac: callable + :param hmac: An HMAC function, i.e. as returned from + :func:`bridgedb.crypto.getHMACFunc`. + :param int assigned: The subring number that we wish to draw bridges from. + For example, if a user is assigned to subring 2of3 based on their IP + address, then this function should only return bridges which would + also be assigned to subring 2of3. + :param int total: The total number of subrings. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + logging.debug(("Creating a filter for assigning bridges to subhashring " + "%s-of-%s...") % (assigned, total)) + + name = "-".join([str(hmac("")[:8]).encode('hex'), + str(assigned), "of", str(total)]) + try: + return _cache[name] + except KeyError: + def _bySubring(bridge): + position = int(hmac(bridge.identity)[:8], 16) + which = (position % total) + 1 + return True if which == assigned else False + # The `description` attribute must contain an `=`, or else + # dumpAssignments() will not work correctly. + setattr(_bySubring, "description", "ring=%d" % assigned) + _bySubring.__name__ = ("bySubring%sof%s" % (assigned, total)) + _bySubring.name = name + _cache[name] = _bySubring + return _bySubring + +def byFilters(filtres): + """Returns a filter which filters by multiple **filtres**. + + :type filtres: list + :param filtres: A list (or other iterable) of callables which some + :class:`~bridgedb.bridges.Bridge`s should be filtered according to. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + name = [] + for filtre in filtres: + name.extend(filtre.name.split(" ")) + name = " ".join(set(name)) + + try: + return _cache[name] + except KeyError: + def _byFilters(bridge): + results = [f(bridge) for f in filtres] + if False in results: + return False + return True + setattr(_byFilters, "description", + " ".join([getattr(f, "description", "") for f in filtres])) + _byFilters.name = name + _cache[name] = _byFilters + return _byFilters + +def byIPv(ipVersion=None): + """Return ``True`` if at least one of the **bridge**'s addresses has the + specified **ipVersion**. + + :param int ipVersion: Either ``4`` or ``6``. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + + name = "ipv%d" % ipVersion + try: + return _cache[name] + except KeyError: + def _byIPv(bridge): + if isIPv(ipVersion, bridge.address): + return True + else: + for address, port, version in bridge.allVanillaAddresses: + if version == ipVersion or isIPv(ipVersion, address): + return True + return False + setattr(_byIPv, "description", "ip=%d" % ipVersion) + _byIPv.__name__ = "byIPv%d()" % ipVersion + _byIPv.name = name + _cache[name] = _byIPv + return _byIPv + +byIPv4 = byIPv(4) +byIPv6 = byIPv(6) + +def byTransport(methodname=None, ipVersion=None): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + The returned filter function should be called on a + :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` + has a :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, and + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an + instance of **addressClass**. + + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s ``PluggableTransport`` + :data:`~bridgedb.bridges.PluggableTransport.address`` should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not methodname: + return byIPv(ipVersion) + + methodname = methodname.lower() + name = "transport-%s ipv%d" % (methodname, ipVersion) + + try: + return _cache[name] + except KeyError: + def _byTransport(bridge): + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byTransport, "description", "transport=%s" % methodname) + _byTransport.__name__ = "byTransport(%s,%s)" % (methodname, ipVersion) + _byTransport.name = name + _cache[name] = _byTransport + return _byTransport + +def byNotBlockedIn(countryCode=None, methodname=None, ipVersion=4): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + If a Pluggable Transport **methodname** was not specified, the returned + filter function returns ``True`` if any of the ``Bridge``'s addresses or + :class:`~bridgedb.bridges.PluggableTransport` addresses aren't blocked in + **countryCode**. See :meth:`~bridgedb.bridges.Bridge.isBlockedIn`. + + Otherwise, if a Pluggable Transport **methodname** was specified, it + returns ``True`` if the ``Bridge`` has a + :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address.version`` + equals the **ipVersion**, and isn't known to be blocked in + **countryCode**. + + :type countryCode: str or ``None`` + :param countryCode: A two-letter country code which the filtered + :class:`PluggableTransport`s should not be blocked in. + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s addresses should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not countryCode: + return byTransport(methodname, ipVersion) + + methodname = methodname.lower() if methodname else methodname + countryCode = countryCode.lower() + + name = [] + if methodname: + name.append("transport-%s" % methodname) + name.append("ipv%d" % ipVersion) + name.append("not-blocked-in-%s" % countryCode) + name = " ".join(name) + + try: + return _cache[name] + except KeyError: + def _byNotBlockedIn(bridge): + if not methodname: + return not bridge.isBlockedIn(countryCode) + elif methodname == "vanilla": + if bridge.address.version == ipVersion: + if not bridge.addressIsBlockedIn(countryCode, + bridge.address, + bridge.orPort): + return True + else: + # Since bridge.transportIsBlockedIn() will return True if the + # bridge has that type of transport AND that transport is + # blocked, we can "fail fast" here by doing this faster check + # before iterating over all the transports testing for the + # other conditions. + if bridge.transportIsBlockedIn(countryCode, methodname): + return False + else: + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byNotBlockedIn, "description", "unblocked=%s" % countryCode) + _byNotBlockedIn.__name__ = ("byTransportNotBlockedIn(%s,%s,%s)" + % (methodname, countryCode, ipVersion)) + _byNotBlockedIn.name = name + _cache[name] = _byNotBlockedIn + return _byNotBlockedIn diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index e8cdd62..2a1d510 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -47,10 +47,6 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index 96dc21e..b3ad680 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -318,7 +318,7 @@ def isIPAddress(ip, compressed=True): return ip return False -def _isIPv(version, ip): +def isIPv(version, ip): """Check if **ip** is a certain **version** (IPv4 or IPv6). .. warning: Do *not* put any calls to the logging module in this function, @@ -352,7 +352,7 @@ def isIPv4(ip): :rtype: boolean :returns: True if the address is an IPv4 address. """ - return _isIPv(4, ip) + return isIPv(4, ip) def isIPv6(ip): """Check if an address is IPv6. @@ -364,7 +364,7 @@ def isIPv6(ip): :rtype: boolean :returns: True if the address is an IPv6 address. """ - return _isIPv(6, ip) + return isIPv(6, ip) def isValidIP(ip): """Check that an IP (v4 or v6) is valid. diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index c996daa..00726f6 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -4,9 +4,9 @@ # # :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org> # please also see AUTHORS file -# :copyright: (c) 2013 Isis Lovecruft -# (c) 2007-2013, The Tor Project, Inc. -# (c) 2007-2013, all entities within the AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file # :license: 3-clause BSD, see included LICENSE for information """Module for functionality to persistently store state.""" @@ -23,7 +23,9 @@ except (ImportError, NameError): # pragma: no cover from twisted.python.reflect import safe_repr from twisted.spread import jelly -from bridgedb import Filters, Bridges, Dist +from bridgedb import Bridges +from bridgedb import Dist +from bridgedb import filters from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -32,7 +34,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(Filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist) class MissingState(Exception): diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 95dc34e..3ec634c 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -25,11 +25,6 @@ import bridgedb.Storage import re import ipaddr -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn - from bridgedb.Stability import BridgeHistory from bridgedb.parse import addr diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 85c1b87..46a11b0 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -24,9 +24,8 @@ from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import BridgeRingParameters -from bridgedb.Filters import filterBridgesByNotBlockedIn -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet from bridgedb.test.util import randomHighPort @@ -355,7 +354,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv4() - bridgeRequest.filters.append(filterBridgesByIP6) + bridgeRequest.filters.append(byIPv6) bridgeRequest.generateFilters() bridges = dist.getBridges(bridgeRequest, 1) @@ -367,7 +366,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): """Asking for bridge addresses which are simultaneously IPv6 and IPv4 @@ -379,7 +378,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv6() bridgeRequest.generateFilters() - bridgeRequest.filters.append(filterBridgesByIP4) + bridgeRequest.filters.append(byIPv4) bridges = dist.getBridges(bridgeRequest, 1) self.assertEqual(len(bridges), 3) @@ -390,7 +389,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6(self): """A request for IPv6 bridges should return IPv6 bridges.""" @@ -412,7 +411,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv4(self): """A request for IPv4 bridges should return IPv4 bridges.""" @@ -432,4 +431,4 @@ class HTTPSDistributorTests(unittest.TestCase): addrport, fingerprint = bridgeLine.split() address, port = addrport.rsplit(':', 1) self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_filters.py b/lib/bridgedb/test/test_filters.py new file mode 100644 index 0000000..73e5685 --- /dev/null +++ b/lib/bridgedb/test/test_filters.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# :license: see included LICENSE for information + +"""Tests for :mod:`bridgedb.filters`.""" + +from __future__ import print_function + +import ipaddr + +from twisted.trial import unittest + +from bridgedb import filters +from bridgedb.bridges import Bridge +from bridgedb.bridges import PluggableTransport +from bridgedb.crypto import getHMACFunc + + +class FiltersTests(unittest.TestCase): + """Tests for :mod:`bridgedb.filters`.""" + + def setUp(self): + """Create a Bridge whose address is 1.1.1.1, orPort is 1111, and + fingerprint is 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'. Also, + create an HMAC function whose key is 'plasma'. + """ + self.bridge = Bridge() + self.bridge.address = '1.1.1.1' + self.bridge.orPort = 1111 + self.bridge.fingerprint = 'a' * 40 + + self.hmac = getHMACFunc('plasma') + + def addIPv4VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '1.1.1.1', 1111, {}) + self.bridge.transports.append(pt) + + def addIPv6VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '2006:2222::2222', 1111, {}) + self.bridge.transports.append(pt) + + def test_bySubring_1_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 1, 2) + should return that Bridge (because it is in the sub-hashring we asked + for). + """ + filtre = filters.bySubring(self.hmac, 1, 2) + self.assertTrue(filtre(self.bridge)) + + def test_bySubring_2_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 2, 2) + should *not* return that Bridge (because it is in sub-hashring 1-of-2 + and we asked for Bridges which are in sub-hashring 2-of-2). + """ + filtre = filters.bySubring(self.hmac, 2, 2) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertTrue(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_no_filters(self): + self.addIPv4VoltronPT() + filtre = filters.byFilters([]) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv_ipv5(self): + """Calling byIPv(ipVersion=5) should default to filterint by IPv4.""" + filtre = filters.byIPv(5) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv4_address(self): + """A bridge with an IPv4 address for its main orPort address should + cause filters.byIPv4() to return True. + """ + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_orAddress(self): + """A bridge with an IPv4 address in its orAddresses address should + cause filters.byIPv4() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.bridge.orAddresses = [(ipaddr.IPv4Address('2.2.2.2'), 2222, 4)] + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_none(self): + """A bridge with no IPv4 addresses should cause filters.byIPv4() to + return False. + """ + self.bridge.address = ipaddr.IPv6Address('2006:2222::2222') + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertFalse(filters.byIPv4(self.bridge)) + + def test_byIPv6_address(self): + """A bridge with an IPv6 address for its main orPort address should + cause filters.byIPv6() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_orAddress(self): + """A bridge with an IPv6 address in its orAddresses address should + cause filters.byIPv6() to return True. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_none(self): + """A bridge with no IPv6 addresses should cause filters.byIPv6() to + return False. + """ + self.assertFalse(filters.byIPv6(self.bridge)) + + def test_byTransport_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron', ipVersion=6) to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6_filtering_by_ipv4(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_no_transports(self): + """A bridge without any transports should cause + byTransport('voltron') to return False. + """ + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv4(self): + """byTransport() without namimg a transport to filter by should just + return the bridge's IPv4 address. + """ + filtre = filters.byTransport() + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv6(self): + """byTranspfort(ipVersion=6) without namimg a transport to filter by + should just return the bridge's IPv4 address. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + filtre = filters.byTransport(ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byTransport('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_blocked(self): + """A bridge with an IPv4 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv4VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_not_blocked_ipv4(self): + """A bridge with an IPv6 voltron transport which is not blocked in China + should cause byNotBlockedIn('cn', 'voltron') to return False, because + the IP version is wrong. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6_blocked(self): + """A bridge with an IPv6 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv6VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('voltron') to return False (because it calls + filters.byTransport('voltron')). + """ + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('cn', 'voltron') to return False. + """ + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports_blocked(self): + """A bridge without any transports which is also blocked should cause + byNotBlockedIn('voltron') to return False. + """ + self.bridge.setBlockedIn('cn') + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byNotBlockedIn('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('cn', methodname='obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_ipv5(self): + """Calling byNotBlockedIn([…], ipVersion=5) should default to IPv4.""" + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', ipVersion=5) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked(self): + """Calling byNotBlockedIn('vanilla') should return the IPv4 vanilla + address, if it is not blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked_ipv6(self): + """Calling byNotBlockedIn('vanilla', ipVersion=6) should not return the + IPv4 vanilla address, even if it is not blocked, because it has the + wrong IP version. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_blocked(self): + """Calling byNotBlockedIn('vanilla') should not return the IPv4 vanilla + address, if it is blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('ru', methodname='vanilla') + self.assertFalse(filtre(self.bridge))