July 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Deduplicate subhashring logic in IPBasedDistributor.prepopulateRings().
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 023596e567e17fd801a7cbc4827b4836908c73fe Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 9 23:27:36 2015 +0000 Deduplicate subhashring logic in IPBasedDistributor.prepopulateRings(). --- lib/bridgedb/Dist.py | 51 ++++++++++++++------------------------------------ 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 0658504..127004a 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -250,45 +250,22 @@ class IPBasedDistributor(Distributor): Thus, in this example, we end up with **12 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) - # populate all rings (for dumping assignments and testing) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: - n = self.nClusters - for category in self.categories: - g = filterAssignBridgesToRing(self.splitter.hmac, - self.nClusters + - len(self.categories), - n) - bridgeFilterRules = [g] - if filterFn: - bridgeFilterRules.append(filterFn) - ruleset = frozenset(bridgeFilterRules) - key1 = getHMAC(self.splitter.key, - "Order-Bridges-In-Ring-%d" % n) - n += 1 - ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - ring.setName('{0} Ring'.format(self.name)) - self.splitter.addRing(ring, - ruleset, - filterBridgesByRules(bridgeFilterRules), - populate_from=self.splitter.bridges) - - # populate all ip clusters - for clusterNum in xrange(self.nClusters): - g = filterAssignBridgesToRing(self.splitter.hmac, - self.nClusters + - len(self.categories), - clusterNum) - bridgeFilterRules = [g] - if filterFn: - bridgeFilterRules.append(filterFn) - ruleset = frozenset(bridgeFilterRules) - key1 = getHMAC(self.splitter.key, - "Order-Bridges-In-Ring-%d" % clusterNum) + for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + # XXX Distributors should have a "totalClusters" property in order + # to avoid reusing this unclear construct all over the place. (Or + # just get rid of the idea of "categories".) + for cluster in range(self.nClusters + len(self.categories)): + filters = self._buildHashringFilters([filterFn,], cluster) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, - ruleset, - filterBridgesByRules(bridgeFilterRules), + # For consistency with previous implementation of this method, + # only set the "name" for "clusters" which are in this + # distributor's categories: + if cluster >= self.nClusters: + ring.setName('{0} Ring'.format(self.name)) + self.splitter.addRing(ring, filters, + filterBridgesByRules(filters), populate_from=self.splitter.bridges) def clear(self):

1 0

[bridgedb/master] Refactor getBridgesForIP() to use b.https.request.HTTPSBridgeRequest.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 7b880a1fe0f346c85364b255eee93995a6b31b94 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 1 08:40:50 2015 +0000 Refactor getBridgesForIP() to use b.https.request.HTTPSBridgeRequest. * RENAME b.D.IPBasedDistributor.getBridgesForIP() → b.D.IPBasedDistributor.getBridges(). * CREATE new bridgedb.https package. * CREATE new bridgedb.https.request module. * CHANGE b.D.IPBasedDistributor.getBridges() to use bridgedb.https.request.HTTPSBridgeRequest. --- doc/sphinx/source/bridgedb.https.rst | 10 +++ doc/sphinx/source/bridgedb.rst | 1 + lib/bridgedb/Dist.py | 143 ++++++++++++++-------------------- lib/bridgedb/HTTPServer.py | 142 ++++++++++++++------------------- lib/bridgedb/https/__init__.py | 1 + lib/bridgedb/https/request.py | 143 ++++++++++++++++++++++++++++++++++ lib/bridgedb/test/legacy_Tests.py | 26 +++---- lib/bridgedb/test/test_HTTPServer.py | 27 ++++--- setup.py | 1 + 9 files changed, 301 insertions(+), 193 deletions(-) diff --git a/doc/sphinx/source/bridgedb.https.rst b/doc/sphinx/source/bridgedb.https.rst new file mode 100644 index 0000000..b0ee95c --- /dev/null +++ b/doc/sphinx/source/bridgedb.https.rst @@ -0,0 +1,10 @@ +.. _https-pkg: + +bridgedb.https +--------------- + +.. contents:: bridgedb.https + :depth: 3 + +.. automodule:: bridgedb.https.__init__ +.. automodule:: bridgedb.https.request diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 3f87647..61ab07f 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -17,6 +17,7 @@ BridgeDB Package and Module Documentation bridgedb.email bridgedb.Filters bridgedb.geo + bridgedb.https bridgedb.HTTPServer bridgedb.interfaces bridgedb.Main diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 26e952e..3f7835b 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -305,118 +305,92 @@ class IPBasedDistributor(Distributor): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, - bridgeFilterRules=None): + def _buildHashringFilters(self, previousFilters, clientCluster): + totalRings = self.nClusters + len(self.categories) + g = filterAssignBridgesToRing(self.splitter.hmac, totalRings, clientCluster) + previousFilters.append(g) + return frozenset(previousFilters) + + def getBridges(self, bridgeRequest, interval, N=1): """Return a list of bridges to give to a user. - :param str ip: The user's IP address, as a dotted quad. - :param str epoch: The time period when we got this request. This can - be any string, so long as it changes with every - period. + :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` + :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` + with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` + attribute set to a string containing the client's IP address. + :param str interval: The time period when we got this request. This + can be any string, so long as it changes with every period. :param int N: The number of bridges to try to give back. (default: 1) - :param str countryCode: The two-letter geoip country code of the - client's IP address. If given, it is assumed that any bridges - distributed to that client should not be blocked in that - country. (default: None) - :param list bridgeFilterRules: A list of callables used filter the - bridges returned in the response to the - client. See :mod:`~bridgedb.Filters`. :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in - the response. See - :meth:`bridgedb.HTTPServer.WebResource.getBridgeRequestAnswer` - for an example of how this is used. + the response. See + :meth:`bridgedb.HTTPServer.WebResourceBridges.getBridgeRequestAnswer` + for an example of how this is used. """ logging.info("Attempting to return %d bridges to client %s..." - % (N, ip)) - - if not bridgeFilterRules: - bridgeFilterRules=[] + % (N, bridgeRequest.client)) if not len(self.splitter): logging.warn("Bailing! Splitter has zero bridges!") return [] - logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) - logging.debug("Client request epoch:\t%s" % epoch) - logging.debug("Active bridge filters:\t%s" - % ' '.join([x.func_name for x in bridgeFilterRules])) - - area = self.areaMapper(ip) - logging.debug("IP mapped to area:\t%s" % area) - - key1 = '' - pos = 0 - n = self.nClusters - - # only one of ip categories or area clustering is active - # try to match the request to an ip category + # The cluster the client should draw bridges from: + clientCluster = self.nClusters + # First, check if the client's IP is one of the known proxies in one + # of our :data:`catagories`: for category in self.categories: - # IP Categories - if ip in category: + if bridgeRequest.client in category: # The tag is a tag applied to a proxy IP address when it is # added to the bridgedb.proxy.ProxySet. For Tor Exit relays, # the default is 'exit_relay'. For other proxies loaded from # the PROXY_LIST_FILES config option, the default tag is the # full filename that the IP address originally came from. - tag = category.getTag(ip) - logging.info("Client was from known proxy (tag: %s): %s" % (tag, ip)) - g = filterAssignBridgesToRing(self.splitter.hmac, - self.nClusters + - len(self.categories), - n) - bridgeFilterRules.append(g) + tag = category.getTag(bridgeRequest.client) + logging.info("Client was from known proxy (tag: %s): %s" % + (tag, bridgeRequest.client)) # Cluster Tor/proxy users into four groups. This means that # no matter how many different Tor Exits or proxies a client # uses, the most they can ever get is four different sets of # bridge lines (per period). - group = (int(ipaddr.IPAddress(ip)) % 4) + 1 - logging.debug(("Assigning client hashring position based on: " - "known-proxy<%s>%s") % (epoch, group)) - pos = self.areaOrderHmac("known-proxy<%s>%s" % (epoch, group)) - key1 = getHMAC(self.splitter.key, - "Order-Bridges-In-Ring-%d" % n) + group = (int(ipaddr.IPAddress(bridgeRequest.client)) % 4) + 1 + area = "known-proxy-group-%d" % group break - n += 1 - - # if no category matches, use area clustering + clientCluster += 1 + # If the client wasn't using Tor or any other known proxy, select the + # client's cluster number based upon the /16 of the client's IP + # address: else: - # IP clustering - h = int( self.areaClusterHmac(area)[:8], 16) - # length of numClusters - clusterNum = h % self.nClusters - - g = filterAssignBridgesToRing(self.splitter.hmac, - self.nClusters + - len(self.categories), - clusterNum) - bridgeFilterRules.append(g) - pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) - key1 = getHMAC(self.splitter.key, - "Order-Bridges-In-Ring-%d" % clusterNum) - - # try to find a cached copy - ruleset = frozenset(bridgeFilterRules) - - # See if we have a cached copy of the ring, - # otherwise, add a new ring and populate it - if ruleset in self.splitter.filterRings.keys(): - logging.debug("Cache hit %s" % ruleset) - _, ring = self.splitter.filterRings[ruleset] - - # else create the ring and populate it + # Areas (i.e. /16s) are grouped into the number of rings specified + # by the N_IP_CLUSTERS configuration option. + area = self.areaMapper(bridgeRequest.client) + logging.debug("IP mapped to area:\t%s" % area) + clientCluster = int(self.areaClusterHmac(area)[:8], 16) % self.nClusters + + pos = self.areaOrderHmac("<%s>%s" % (interval, area)) + filters = self._buildHashringFilters(bridgeRequest.filters, clientCluster) + + logging.debug("Assigned client hashring position based on: <%s>%s" % + (interval, area)) + logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) + logging.debug("Active bridge filters:\t%s" % + ' '.join([x.func_name for x in filters])) + + # Check wheth we have a cached copy of the hashring: + if filters in self.splitter.filterRings.keys(): + logging.debug("Cache hit %s" % filters) + _, ring = self.splitter.filterRings[filters] + # Otherwise, construct a new hashring and populate it: else: - logging.debug("Cache miss %s" % ruleset) + logging.debug("Cache miss %s" % filters) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % clientCluster) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, - ruleset, - filterBridgesByRules(bridgeFilterRules), + self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) - # get an appropriate number of bridges - numBridgesToReturn = getNumBridgesPerAnswer(ring, - max_bridges_per_answer=N) - answer = ring.getBridges(pos, numBridgesToReturn) + # Determine the appropriate number of bridges to give to the client: + returnNum = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) + answer = ring.getBridges(pos, returnNum) + return answer def __len__(self): @@ -425,6 +399,7 @@ class IPBasedDistributor(Distributor): def dumpAssignments(self, f, description=""): self.splitter.dumpAssignments(f, description) + class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 8c58380..179c061 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -23,8 +23,6 @@ import os from functools import partial from ipaddr import IPv4Address -from ipaddr import IPv6Address -from ipaddr import IPAddress import mako.exceptions from mako.template import Template @@ -32,15 +30,11 @@ from mako.lookup import TemplateLookup from twisted.internet import reactor from twisted.internet.error import CannotListenError -from twisted.python import filepath from twisted.web import resource from twisted.web import server from twisted.web import static from twisted.web.util import redirectTo -import bridgedb.Dist -import bridgedb.geo - from bridgedb import captcha from bridgedb import crypto from bridgedb import strings @@ -50,6 +44,7 @@ from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByTransport from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress from bridgedb.qrcodes import generateQR @@ -671,6 +666,18 @@ class WebResourceBridges(resource.Resource): return response + def getClientIP(self, request): + """Get the client's IP address from the :header:`X-Forwarded-For` + header, or from the :api:`request <twisted.web.server.Request>`. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object for a + :api:`twisted.web.resource.Resource`. + :rtype: None or str + :returns: The client's IP address, if it was obtainable. + """ + return getClientIP(request, self.useForwardedHeader) + def getBridgeRequestAnswer(self, request): """Respond to a client HTTP request for bridges. @@ -680,104 +687,75 @@ class WebResourceBridges(resource.Resource): :rtype: str :returns: A plaintext or HTML response to serve. """ - # XXX why are we getting the interval if our distributor might be - # using bridgedb.schedule.Unscheduled? + bridgeLines = None interval = self.schedule.intervalStart(time.time()) - bridges = ( ) - ip = None - countryCode = None - - ip = getClientIP(request, self.useForwardedForHeader) - - # Look up the country code of the input IP - if isIPAddress(ip): - countryCode = bridgedb.geo.getCountryCode(IPAddress(ip)) - else: - logging.warn("Invalid IP detected; skipping country lookup.") - countryCode = None - - # XXX separate function again - format = request.args.get("format", None) - if format and len(format): format = format[0] # choose the first arg - - # do want any options? - transport = ipv6 = unblocked = False - - ipv6 = request.args.get("ipv6", False) - if ipv6: ipv6 = True # if anything after ?ipv6= - - # XXX oh dear hell. why not check for the '?transport=' arg before - # regex'ing? And why not compile the regex once, somewhere outside - # this function and class? - try: - # validate method name - transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*', - request.args.get("transport")[0]).group() - except (TypeError, IndexError, AttributeError): - transport = None - - try: - unblocked = re.match('[a-zA-Z]{2,4}', - request.args.get("unblocked")[0]).group() - except (TypeError, IndexError, AttributeError): - unblocked = False + ip = self.getClientIP(request) logging.info("Replying to web request from %s. Parameters were %r" % (ip, request.args)) - rules = [] - bridgeLines = None - if ip: - if ipv6: - rules.append(filterBridgesByIP6) - addressClass = IPv6Address - else: - rules.append(filterBridgesByIP4) - addressClass = IPv4Address + bridgeRequest = HTTPSBridgeRequest() + bridgeRequest.client = ip + bridgeRequest.isValid(True) + bridgeRequest.withIPversion(request.args) + bridgeRequest.withPluggableTransportType(request.args) + bridgeRequest.withoutBlockInCountry(request) + bridgeRequest.generateFilters() + + bridges = self.distributor.getBridges(bridgeRequest, interval) + bridgeLines = [replaceControlChars(bridge.getBridgeLine( + bridgeRequest, self.includeFingerprints)) for bridge in bridges] - if transport: - #XXX: A cleaner solution would differentiate between - # addresses by protocol rather than have separate lists - # Tor to be a transport, and selecting between them. - rules = [filterBridgesByTransport(transport, addressClass)] + return self.renderAnswer(request, bridgeLines) - if unblocked: - rules.append(filterBridgesByNotBlockedIn(unblocked, - addressClass, transport)) + def getResponseFormat(self, request): + """Determine the requested format for the response. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object containing the HTTP method, full + URI, and any URL/POST arguments and headers present. + :rtype: ``None`` or str + :returns: The argument of the first occurence of the ``format=`` HTTP + GET parameter, if any were present. (The only one which currently + has any effect is ``format=plain``, see note in + :meth:`renderAnswer`.) Otherwise, returns ``None``. + """ + format = request.args.get("format", None) + if format and len(format): + format = format[0] # Choose the first arg + return format - bridges = self.distributor.getBridgesForIP(ip, interval, - self.nBridgesToGive, - countryCode, - bridgeFilterRules=rules) - bridgeLines = [replaceControlChars(b.getConfigLine( - includeFingerprint=self.includeFingerprints, - addressClass=addressClass, - transport=transport, - request=bridgedb.Dist.uniformMap(ip))) for b in bridges] + def renderAnswer(self, request, bridgeLines=None): + """Generate a response for a client which includes **bridgesLines**. - answer = self.renderAnswer(request, bridgeLines, format) - return answer + .. note: The generated response can be plain or HTML. A plain response + looks like:: - def renderAnswer(self, request, bridgeLines=None, format=None): - """Generate a response for a client which includes **bridges**. + voltron 1.2.3.4:1234 ABCDEF01234567890ABCDEF01234567890ABCDEF + voltron 5.5.5.5:5555 0123456789ABCDEF0123456789ABCDEF01234567 - The generated response can be plaintext or HTML. + That is, there is no HTML, what you see is what you get, and what + you get is suitable for pasting directly into Tor Launcher (or + into a torrc, if you prepend ``"Bridge "`` to each line). The + plain format can be requested from BridgeDB's web service by + adding an ``&format=plain`` HTTP GET parameter to the URL. Also + note that you won't get a QRCode, usage instructions, error + messages, or any other fanciness if you use the plain format. :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object containing the HTTP method, full URI, and any URL/POST arguments and headers present. :type bridgeLines: list or None :param bridgeLines: A list of strings used to configure a Tor client - to use a bridge. - :type format: str or None - :param format: If ``'plain'``, return a plaintext response. Otherwise, - use the :file:`bridgedb/templates/bridges.html` template to render - an HTML response page which includes the **bridges**. + to use a bridge. If ``None``, then the returned page will instead + explain that there were no bridges of the type they requested, + with instructions on how to proceed. :rtype: str :returns: A plaintext or HTML response to serve. """ rtl = False + format = self.getResponseFormat(request) if format == 'plain': request.setHeader("Content-Type", "text/plain") diff --git a/lib/bridgedb/https/__init__.py b/lib/bridgedb/https/__init__.py new file mode 100644 index 0000000..f1210ec --- /dev/null +++ b/lib/bridgedb/https/__init__.py @@ -0,0 +1 @@ +"""Servers for BridgeDB's HTTPS bridge distributor.""" diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py new file mode 100644 index 0000000..a8f6920 --- /dev/null +++ b/lib/bridgedb/https/request.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8; test-case-name: bridgedb.test.test_https_request; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft <isis(a)torproject.org> 0xA3ADB67A2CDB8B35 +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# (c) 2013-2015, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +""" +.. py:module:: bridgedb.https.request + :synopsis: Classes for parsing and storing information about requests for + bridges which are sent to the HTTPS distributor. + +bridgedb.https.request +====================== + +Classes for parsing and storing information about requests for bridges +which are sent to the HTTPS distributor. + +:: + + bridgedb.https.request + | + |_ HTTPSBridgeRequest - A request for bridges which was received through + the HTTPS distributor. +.. +""" + +from __future__ import print_function +from __future__ import unicode_literals + +import ipaddr +import logging +import re + +from bridgedb import bridgerequest +from bridgedb import geo +from bridgedb.parse import addr + + +#: A regular expression for matching the Pluggable Transport methodname in +#: HTTP GET request parameters. +TRANSPORT_REGEXP = "[_a-zA-Z][_a-zA-Z0-9]*" +TRANSPORT_PATTERN = re.compile(TRANSPORT_REGEXP) + +UNBLOCKED_REGEXP = "[a-zA-Z]{2}" +UNBLOCKED_PATTERN = re.compile(UNBLOCKED_REGEXP) + + +class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): + """We received a request for bridges through the HTTPS distributor.""" + + def __init__(self, addClientCountryCode=True): + """Process a new bridge request received through the + :class:`~bridgedb.Dist.IPBasedDistributor`. + + :param bool addClientCountryCode: If ``True``, then calling + :meth:`withoutBlockInCountry` will attempt to add the client's own + country code, geolocated from her IP, to the ``notBlockedIn`` + countries list. + """ + super(HTTPSBridgeRequest, self).__init__() + self.addClientCountryCode = addClientCountryCode + + def withIPversion(self, parameters): + """Determine if the request **parameters** were for bridges with IPv6 + addresses or not. + + .. note:: If there is an ``ipv6=`` parameter with anything non-zero + after it, then we assume the client wanted IPv6 bridges. + + :param parameters: The :api:`twisted.web.http.Request.args`. + """ + if parameters.get("ipv6", False): + logging.info("HTTPS request for bridges with IPv6 addresses.") + self.withIPv6() + + def withoutBlockInCountry(self, request): + """Determine which countries the bridges for this **request** should + not be blocked in. + + .. note:: Currently, a **request** for unblocked bridges is recognized + if it contains an HTTP GET parameter ``unblocked=`` whose value is + a comma-separater list of two-letter country codes. Any + two-letter country code found in the + :api:`request <twisted.web.http.Request>` ``unblocked=`` HTTP GET + parameter will be added to the :data:`notBlockedIn` list. + + If :data:`addClientCountryCode` is ``True``, the the client's own + geolocated country code will be added to the to the + :data`notBlockedIn` list. + + :type request: :api:`twisted.web.http.Request` + :param request: A ``Request`` object containing the HTTP method, full + URI, and any URL/POST arguments and headers present. + """ + countryCodes = request.args.get("unblocked", list()) + + for countryCode in countryCodes: + try: + country = UNBLOCKED_PATTERN.match(countryCode).group() + except (TypeError, AttributeError): + pass + else: + if country: + self.notBlockedIn.append(country) + logging.info("HTTPS request for bridges not blocked in: %r" + % country) + + if self.addClientCountryCode: + # Look up the country code of the input IP, and request bridges + # not blocked in that country. + if addr.isIPAddress(self.client): + country = geo.getCountryCode(ipaddr.IPAddress(self.client)) + if country: + self.notBlockedIn.append(country) + logging.info( + ("HTTPS client's bridges also shouldn't be blocked " + "in their GeoIP country code: %s") % country) + + def withPluggableTransportType(self, parameters): + """This request included a specific Pluggable Transport identifier. + + Add any Pluggable Transport methodname found in the HTTP GET + **parameters** to the list of ``transports``. Currently, a request for + a transport is recognized if the request contains the + ``'transport='`` parameter. + + :param parameters: The :api:`twisted.web.http.Request.args`. + """ + for methodname in parameters.get("transport", list()): + try: + transport = TRANSPORT_PATTERN.match(methodname).group() + except (TypeError, AttributeError): + pass + else: + if transport: + self.transports.append(transport) + logging.info("HTTPS request for transport type: %r" + % transport) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 2e4d56f..97134e4 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -190,8 +190,8 @@ class IPBridgeDistTests(unittest.TestCase): d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(256): d.insert(fakeBridge()) - n = d.getBridgesForIP("1.2.3.4", "x", 2) - n2 = d.getBridgesForIP("1.2.3.4", "x", 2) + n = d.getBridges("1.2.3.4", "x", 2) + n2 = d.getBridges("1.2.3.4", "x", 2) self.assertEquals(n, n2) def testDistWithCategories(self): @@ -205,8 +205,8 @@ class IPBridgeDistTests(unittest.TestCase): # Make sure that the categories do not overlap f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) - n = d.getBridgesForIP(g(), "x", 10) - n2 = d.getBridgesForIP(f(), "x", 10) + n = d.getBridges(g(), "x", 10) + n2 = d.getBridges(f(), "x", 10) assert(len(n) > 0) assert(len(n2) > 0) @@ -228,7 +228,7 @@ class IPBridgeDistTests(unittest.TestCase): # d.insert(fakeBridge()) # for _ in xrange(32): # i = randomIP() - # n = d.getBridgesForIP(i, "x", 5) + # n = d.getBridges(i, "x", 5) # count = 0 # fps = {} # for b in n: @@ -246,7 +246,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge(or_addresses=True)) for i in xrange(500): - bridges = d.getBridgesForIP(randomIPv4String(), + bridges = d.getBridges(randomIPv4String(), "faketimestamp", bridgeFilterRules=[filterBridgesByIP6]) bridge = random.choice(bridges) @@ -262,7 +262,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge(or_addresses=True)) for i in xrange(500): - bridges = d.getBridgesForIP(randomIPv4String(), + bridges = d.getBridges(randomIPv4String(), "faketimestamp", bridgeFilterRules=[filterBridgesByIP4]) bridge = random.choice(bridges) @@ -278,7 +278,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge(or_addresses=True)) for i in xrange(50): - bridges = d.getBridgesForIP(randomIPv4String(), + bridges = d.getBridges(randomIPv4String(), "faketimestamp", 1, bridgeFilterRules=[ filterBridgesByIP4, @@ -302,7 +302,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge(or_addresses=True)) for i in xrange(5): - b = d.getBridgesForIP(randomIPv4String(), "x", 1, bridgeFilterRules=[ + b = d.getBridges(randomIPv4String(), "x", 1, bridgeFilterRules=[ filterBridgesByIP4, filterBridgesByIP6]) assert len(b) == 0 @@ -325,10 +325,10 @@ class IPBridgeDistTests(unittest.TestCase): b.blockingCountries[key] = set(['cn']) for i in xrange(5): - b = d.getBridgesForIP(randomIPv4String(), "x", 1, bridgeFilterRules=[ + b = d.getBridges(randomIPv4String(), "x", 1, bridgeFilterRules=[ filterBridgesByNotBlockedIn("cn")]) assert len(b) == 0 - b = d.getBridgesForIP(randomIPv4String(), "x", 1, bridgeFilterRules=[ + b = d.getBridges(randomIPv4String(), "x", 1, bridgeFilterRules=[ filterBridgesByNotBlockedIn("us")]) assert len(b) > 0 @@ -356,7 +356,7 @@ class IPBridgeDistTests(unittest.TestCase): # we probably will get at least one bridge back! # it's pretty unlikely to lose a coin flip 250 times in a row for i in xrange(5): - b = d.getBridgesForIP(randomIPString(), "x", 1, + b = d.getBridges(randomIPString(), "x", 1, bridgeFilterRules=[ filterBridgesByNotBlockedIn("cn", methodname='obfs2'), filterBridgesByTransport('obfs2'), @@ -364,7 +364,7 @@ class IPBridgeDistTests(unittest.TestCase): try: assert len(b) > 0 except AssertionError: print("epic fail") - b = d.getBridgesForIP(randomIPString(), "x", 1, bridgeFilterRules=[ + b = d.getBridges(randomIPString(), "x", 1, bridgeFilterRules=[ filterBridgesByNotBlockedIn("us")]) assert len(b) > 0 diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index d597c55..415fb8d 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -448,22 +448,22 @@ class DummyBridge(object): self.or_addresses = {ipaddr.IPv6Address(self._returnSix()): self._randORPort()} - def getConfigLine(self, includeFingerprint=True, - addressClass=ipaddr.IPv4Address, - transport=None, - request=None): + def getBridgeLine(self, bridgeRequest, includeFingerprint=True): """Get a "torrc" bridge config line to give to a client.""" + if not bridgeRequest.isValid(): + return line = [] - if transport is not None: - #print("getConfigLine got transport=%r" % transport) - line.append(str(transport)) - line.append("%s:%s" % (self.ip, self.orport)) - if includeFingerprint is True: line.append(self.fingerprint) + if bridgeRequest.transports: + line.append(bridgeRequest.transports[-1]) # Just the last PT + if bridgeRequest.addressClass is ipaddr.IPv6Address: + line.append("[%s]:%s" % self.or_addresses.items()[0]) + else: + line.append("%s:%s" % (self.ip, self.orport)) + if includeFingerprint is True: + line.append(self.fingerprint) if self.ptArgs: line.append(','.join(['='.join(x) for x in self.ptArgs.items()])) - bridgeLine = " ".join([item for item in line]) - #print "Created config line: %r" % bridgeLine - return bridgeLine + return " ".join([item for item in line]) class DummyMaliciousBridge(DummyBridge): @@ -499,8 +499,7 @@ class DummyIPBasedDistributor(object): self.ipCategories = ipCategories self.answerParameters = answerParameters - def getBridgesForIP(self, ip=None, epoch=None, N=1, - countyCode=None, bridgeFilterRules=None): + def getBridges(self, bridgeRequest=None, epoch=None, N=1): """Needed because it's called in :meth:`WebResourceBridges.getBridgesForIP`. """ diff --git a/setup.py b/setup.py index b788a7e..c75239a 100644 --- a/setup.py +++ b/setup.py @@ -392,6 +392,7 @@ setuptools.setup( package_dir={'': 'lib'}, packages=['bridgedb', 'bridgedb.email', + 'bridgedb.https', 'bridgedb.parse', 'bridgedb.test'], scripts=['scripts/bridgedb',

1 0

[bridgedb/master] Update documentation on IPBasedDistributor area mapping.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 33cb6c45d12d83c441595ff0fc2bf6cc21e46623 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Apr 11 02:52:31 2015 +0000 Update documentation on IPBasedDistributor area mapping. --- lib/bridgedb/Dist.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index ab7ae6e..129bc7c 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -151,9 +151,12 @@ class IPBasedDistributor(Distributor): :type areaMapper: callable :param areaMapper: A function that maps IP addresses arbitrarily to - strings, such that addresses which map to identical strings are - considered to be in the same "area" (for some arbitrary definition - of "area"). See :func:`bridgedb.Dist.uniformMap` for an example. + strings, such that IP addresses which map to identical strings are + considered to be in the same "area". The default **areaMapper** + is :func:`bridgedb.Dist.uniformMap`, which maps all IPv4 addresses + within the same /16 and all IPv6 addresses within the same /32 to + the same area. Areas are then grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option. :param integer numberOfClusters: The number of clusters to group IP addresses into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then the actual number of clusters is one higher than ``numberOfClusters``, @@ -327,8 +330,6 @@ class IPBasedDistributor(Distributor): # based upon the client's area (i.e. the /16 of the client's IP # address): else: - # Areas (i.e. /16s) are grouped into the number of rings specified - # by the N_IP_CLUSTERS configuration option. area = self.areaMapper(bridgeRequest.client) cluster = (int(self.areaClusterHmac(area)[:8], 16) % (self.numberOfClusters - 1))

1 0

[bridgedb/master] Rename nCluster → numberOfClusters.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit d85f4f710ebd67a32f43ea3db0426a9a7e67036b Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Apr 10 07:34:44 2015 +0000 Rename nCluster → numberOfClusters. --- lib/bridgedb/Dist.py | 24 ++++++++++++------------ lib/bridgedb/test/test_HTTPServer.py | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 770198a..f9f61de 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -141,7 +141,7 @@ class IPBasedDistributor(Distributor): distributor. """ - def __init__(self, areaMapper, nClusters, key, + def __init__(self, areaMapper, numberOfClusters, key, ipCategories=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. @@ -151,9 +151,9 @@ class IPBasedDistributor(Distributor): strings, such that addresses which map to identical strings are considered to be in the same "area" (for some arbitrary definition of "area"). See :func:`bridgedb.Dist.uniformMap` for an example. - :param integer nClusters: The number of clusters to group IP addresses + :param integer numberOfClusters: The number of clusters to group IP addresses into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then - the actual number of clusters is one higher than ``nClusters``, + the actual number of clusters is one higher than ``numberOfClusters``, because the set of known open proxies constitutes its own category. DOCDOC What exactly does a cluster *do*? @@ -169,7 +169,7 @@ class IPBasedDistributor(Distributor): bridges" or "at least one bridge on port 443", etc. """ self.areaMapper = areaMapper - self.nClusters = nClusters + self.numberOfClusters = numberOfClusters self.answerParameters = answerParameters if not ipCategories: @@ -193,7 +193,7 @@ class IPBasedDistributor(Distributor): # # XXX Why is the "extra room" hardcoded to be 5? Shouldn't it be some # fraction of the number of clusters/categories? --isis - ring_cache_size = self.nClusters + len(ipCategories) + 5 + ring_cache_size = self.numberOfClusters + len(ipCategories) + 5 self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=ring_cache_size) logging.debug("Added splitter %s to IPBasedDistributor." @@ -207,13 +207,13 @@ class IPBasedDistributor(Distributor): The hashring structure for this distributor is influenced by the ``N_IP_CLUSTERS`` configuration option, as well as the number of - ``PROXY_LIST_FILES``. Essentially, :data:`nClusters` is set to the + ``PROXY_LIST_FILES``. Essentially, :data:`numberOfClusters` is set to the specified ``N_IP_CLUSTERS``. The ``PROXY_LIST_FILES`` (plus the :class:`bridgedb.proxy.ProxySet` for the Tor Exit list downloaded into memory with :script:`get-tor-exits`) are stored in :data:`categories`. The number of subhashrings which this :class:`Distributor` has active - in its hashring is then the :data:`nClusters` plus the number of + in its hashring is then the :data:`numberOfClusters` plus the number of :data:`categories`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and @@ -255,14 +255,14 @@ class IPBasedDistributor(Distributor): # XXX Distributors should have a "totalClusters" property in order # to avoid reusing this unclear construct all over the place. (Or # just get rid of the idea of "categories".) - for cluster in range(self.nClusters + len(self.categories)): + for cluster in range(self.numberOfClusters + len(self.categories)): filters = self._buildHashringFilters([filterFn,], cluster) key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are in this # distributor's categories: - if cluster >= self.nClusters: + if cluster >= self.numberOfClusters: ring.setName('{0} Ring'.format(self.name)) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), @@ -273,7 +273,7 @@ class IPBasedDistributor(Distributor): self.splitter.insert(bridge) def _buildHashringFilters(self, previousFilters, clientCluster): - totalRings = self.nClusters + len(self.categories) + totalRings = self.numberOfClusters + len(self.categories) g = filterAssignBridgesToRing(self.splitter.hmac, totalRings, clientCluster) previousFilters.append(g) return frozenset(previousFilters) @@ -302,7 +302,7 @@ class IPBasedDistributor(Distributor): return [] # The cluster the client should draw bridges from: - clientCluster = self.nClusters + clientCluster = self.numberOfClusters # First, check if the client's IP is one of the known proxies in one # of our :data:`catagories`: for category in self.categories: @@ -331,7 +331,7 @@ class IPBasedDistributor(Distributor): # by the N_IP_CLUSTERS configuration option. area = self.areaMapper(bridgeRequest.client) logging.debug("IP mapped to area:\t%s" % area) - clientCluster = int(self.areaClusterHmac(area)[:8], 16) % self.nClusters + clientCluster = int(self.areaClusterHmac(area)[:8], 16) % self.numberOfClusters pos = self.areaOrderHmac("<%s>%s" % (interval, area)) filters = self._buildHashringFilters(bridgeRequest.filters, clientCluster) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 415fb8d..c3cf79a 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -487,13 +487,13 @@ class DummyIPBasedDistributor(object): def _dumbAreaMapper(ip): return ip - def __init__(self, areaMapper=None, nClusters=None, key=None, + def __init__(self, areaMapper=None, numberOfClusters=None, key=None, ipCategories=None, answerParameters=None): """None of the parameters are really used, they are just there to retain an identical method signature. """ self.areaMapper = self._dumbAreaMapper - self.nClusters = 3 + self.numberOfClusters = 3 self.nBridgesToGive = 3 self.key = self.__class__.__name__ self.ipCategories = ipCategories

1 0

[bridgedb/master] Remove most "splitter"s — they're just crappy consistent hashrings.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit d83438d24104c5834cd0230a757b17a8e1941edb Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 09:23:40 2015 +0000 Remove most "splitter"s — they're just crappy consistent hashrings. --- lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 60 ++++++++++++++-------------- lib/bridgedb/Main.py | 66 +++++++++++++++--------------- lib/bridgedb/test/legacy_Tests.py | 4 +- lib/bridgedb/test/test_Main.py | 80 ++++++++++++++++++------------------- lib/bridgedb/test/test_bridges.py | 24 +++++------ 6 files changed, 118 insertions(+), 118 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index c727c9a..b06ffc4 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -569,7 +569,7 @@ class FilteredBridgeSplitter(BridgeHolder): return index = 0 - logging.debug("Inserting %s into splitter" + logging.debug("Inserting %s into hashring" % (logSafely(bridge.fingerprint))) for old_bridge in self.bridges[:]: if bridge.fingerprint == old_bridge.fingerprint: diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 445d2d8..44cb526 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -81,14 +81,14 @@ class Distributor(object): >>> ipDist = Dist.IPBasedDistributor(5, 'fake-hmac-key') >>> ipDist.setDistributorName('HTTPS Distributor') >>> ipDist.prepopulateRings() - >>> hashrings = ipDist.splitter.filterRings + >>> hashrings = ipDist.hashring.filterRings >>> firstSubring = hashrings.items()[0][1][1] >>> assert firstSubring.name :param str name: A name for this distributor. """ self.name = name - self.splitter.distributorName = name + self.hashring.distributorName = name class IPBasedDistributor(Distributor): @@ -98,8 +98,8 @@ class IPBasedDistributor(Distributor): :type proxies: :class:`~bridgedb.proxies.ProxySet` :ivar proxies: All known proxies, which we treat differently. See :param:`proxies`. - :type splitter: :class:`bridgedb.Bridges.FixedBridgeSplitter` - :ivar splitter: A hashring that assigns bridges to subrings with fixed + :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed proportions. Used to assign bridges into the subrings of this distributor. """ @@ -150,9 +150,9 @@ class IPBasedDistributor(Distributor): self._clientToPositionHMAC = getHMACFunc(key3, hex=False) self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) - self.splitter = FilteredBridgeSplitter(key2, self.ringCacheSize) + self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) logging.debug("Added %s to HTTPS distributor." % - self.splitter.__class__.__name__) + self.hashring.__class__.__name__) self.setDistributorName('HTTPS') @@ -311,16 +311,16 @@ class IPBasedDistributor(Distributor): # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) - self.splitter.addRing(ring, filters, + self.hashring.addRing(ring, filters, filterBridgesByRules(filters), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) def insert(self, bridge): """Assign a bridge to this distributor.""" - self.splitter.insert(bridge) + self.hashring.insert(bridge) def _buildHashringFilters(self, previousFilters, subring): - f = filterAssignBridgesToRing(self.splitter.hmac, self.totalSubrings, subring) + f = filterAssignBridgesToRing(self.hashring.hmac, self.totalSubrings, subring) previousFilters.append(f) return frozenset(previousFilters) @@ -343,8 +343,8 @@ class IPBasedDistributor(Distributor): logging.info("Attempting to return %d bridges to client %s..." % (N, bridgeRequest.client)) - if not len(self.splitter): - logging.warn("Bailing! Splitter has zero bridges!") + if not len(self.hashring): + logging.warn("Bailing! Hashring has zero bridges!") return [] usingProxy = False @@ -373,16 +373,16 @@ class IPBasedDistributor(Distributor): logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: - if filters in self.splitter.filterRings.keys(): + if filters in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % filters) - _, ring = self.splitter.filterRings[filters] + _, ring = self.hashring.filterRings[filters] # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, filters, filterBridgesByRules(filters), - populate_from=self.splitter.bridges) + self.hashring.addRing(ring, filters, filterBridgesByRules(filters), + populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: returnNum = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) @@ -391,18 +391,18 @@ class IPBasedDistributor(Distributor): return answer def __len__(self): - return len(self.splitter) + return len(self.hashring) def dumpAssignments(self, f, description=""): - self.splitter.dumpAssignments(f, description) + self.hashring.dumpAssignments(f, description) class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. - :type splitter: :class:`~bridgedb.Bridges.BridgeRing` - :ivar splitter: A hashring to hold all the bridges we hand out. + :type hashring: :class:`~bridgedb.Bridges.BridgeRing` + :ivar hashring: A hashring to hold all the bridges we hand out. """ def __init__(self, key, domainmap, domainrules, @@ -434,14 +434,14 @@ class EmailBasedDistributor(Distributor): self.answerParameters = answerParameters #XXX cache options not implemented - self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( + self.hashring = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=5) self.setDistributorName('Email') def insert(self, bridge): """Assign a bridge to this distributor.""" - self.splitter.insert(bridge) + self.hashring.insert(bridge) def getBridges(self, bridgeRequest, interval, N=1): """Return a list of bridges to give to a user. @@ -500,9 +500,9 @@ class EmailBasedDistributor(Distributor): ring = None ruleset = frozenset(bridgeRequest.filters) - if ruleset in self.splitter.filterRings.keys(): + if ruleset in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) - _, ring = self.splitter.filterRings[ruleset] + _, ring = self.hashring.filterRings[ruleset] else: # cache miss, add new ring logging.debug("Cache miss %s" % ruleset) @@ -510,9 +510,9 @@ class EmailBasedDistributor(Distributor): # add new ring key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, + self.hashring.addRing(ring, ruleset, filterBridgesByRules(ruleset), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) numBridgesToReturn = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) @@ -524,7 +524,7 @@ class EmailBasedDistributor(Distributor): return result def __len__(self): - return len(self.splitter) + return len(self.hashring) def cleanDatabase(self): with bridgedb.Storage.getDB() as db: @@ -538,7 +538,7 @@ class EmailBasedDistributor(Distributor): db.commit() def dumpAssignments(self, f, description=""): - self.splitter.dumpAssignments(f, description) + self.hashring.dumpAssignments(f, description) def prepopulateRings(self): # populate all rings (for dumping assignments and testing) @@ -546,6 +546,6 @@ class EmailBasedDistributor(Distributor): ruleset = frozenset([filterFn]) key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, + self.hashring.addRing(ring, ruleset, filterBridgesByRules([filterFn]), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index d2fd2da..53189fd 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -39,19 +39,19 @@ from bridgedb import Dist from bridgedb.Stability import updateBridgeHistory -def load(state, splitter, clear=False): - """Read and parse all descriptors, and load into a bridge splitter. +def load(state, hashring, clear=False): + """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then - store them into our ``state.splitter`` instance. The ``state`` will be + store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. - :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` - :param splitter: A class which provides a mechanism for HMACing + :type hashring: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` + :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the - splitter before parsing for new ones. + hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.Main.load() could not retrieve state!") @@ -59,7 +59,7 @@ def load(state, splitter, clear=False): if clear: logging.info("Clearing old bridges...") - splitter.clear() + hashring.clear() logging.info("Loading bridges...") @@ -137,7 +137,7 @@ def load(state, splitter, clear=False): % router.fingerprint) inserted = 0 - logging.info("Inserting %d bridges into splitter..." % len(bridges)) + logging.info("Inserting %d bridges into hashring..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute @@ -148,9 +148,9 @@ def load(state, splitter, clear=False): else: # If the bridge is not running, then it is skipped during the # insertion process. - splitter.insert(bridge) + hashring.insert(bridge) inserted += 1 - logging.info("Done inserting %d bridges into splitter." % inserted) + logging.info("Done inserting %d bridges into hashring." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) @@ -180,7 +180,7 @@ def _handleSIGUSR1(*args): def replaceBridgeRings(current, replacement): """Replace the current thing with the new one""" - current.splitter = replacement.splitter + current.hashring = replacement.hashring def createBridgeRings(cfg, proxyList, key): """Create the bridge distributors defined by the config file @@ -192,15 +192,15 @@ def createBridgeRings(cfg, proxyList, key): :type proxyList: :class:`~bridgedb.proxy.ProxySet` :param proxyList: The container for the IP addresses of any currently known open proxies. - :param bytes key: Splitter master key + :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter splitter, an IPBasedDistributor or None, + :returns: A BridgeSplitter hashring, an IPBasedDistributor or None, and an EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. - splitter = Bridges.BridgeSplitter(crypto.getHMAC(key, "Splitter-Key")) - logging.debug("Created splitter: %r" % splitter) + hashring = Bridges.BridgeSplitter(crypto.getHMAC(key, "Hashring-Key")) + logging.debug("Created hashring: %r" % hashring) # Create ring parameters. ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS, @@ -215,7 +215,7 @@ def createBridgeRings(cfg, proxyList, key): crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, answerParameters=ringParams) - splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) + hashring.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) # As appropriate, create an email-based distributor. if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: @@ -226,19 +226,19 @@ def createBridgeRings(cfg, proxyList, key): cfg.EMAIL_DOMAIN_RULES.copy(), answerParameters=ringParams, whitelist=cfg.EMAIL_WHITELIST.copy()) - splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) + hashring.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) - # As appropriate, tell the splitter to leave some bridges unallocated. + # As appropriate, tell the hashring to leave some bridges unallocated. if cfg.RESERVED_SHARE: - splitter.addRing(Bridges.UnallocatedHolder(), + hashring.addRing(Bridges.UnallocatedHolder(), "unallocated", cfg.RESERVED_SHARE) - # Add pseudo distributors to splitter + # Add pseudo distributors to hashring for pseudoRing in cfg.FILE_BUCKETS.keys(): - splitter.addPseudoRing(pseudoRing) + hashring.addPseudoRing(pseudoRing) - return splitter, emailDistributor, ipDistributor + return hashring, emailDistributor, ipDistributor def run(options, reactor=reactor): """This is BridgeDB's main entry point and main runtime loop. @@ -314,7 +314,7 @@ def run(options, reactor=reactor): State should be saved before calling this method, and will be saved again at the end of it. - The internal variables, ``cfg``, ``splitter``, ``proxyList``, + The internal variables, ``cfg``, ``hashring``, ``proxyList``, ``ipDistributor``, and ``emailDistributor`` are all taken from a :class:`~bridgedb.persistent.State` instance, which has been saved to a statefile with :meth:`bridgedb.persistent.State.save`. @@ -323,8 +323,8 @@ def run(options, reactor=reactor): :ivar cfg: The current configuration, including any in-memory settings (i.e. settings whose values were not obtained from the config file, but were set via a function somewhere) - :type splitter: A :class:`bridgedb.Bridges.BridgeHolder` - :ivar splitter: A class which takes an HMAC key and splits bridges + :type hashring: A :class:`bridgedb.Bridges.BridgeHolder` + :ivar hashring: A class which takes an HMAC key and splits bridges into their hashring assignments. :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently @@ -357,21 +357,21 @@ def run(options, reactor=reactor): proxy.loadProxiesFromFile(proxyfile, state.proxies, removeStale=True) logging.info("Reparsing bridge descriptors...") - (splitter, + (hashring, emailDistributorTmp, ipDistributorTmp) = createBridgeRings(cfg, state.proxies, key) - logging.info("Bridges loaded: %d" % len(splitter)) + logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. bridgedb.Storage.initializeDBLock() bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") - load(state, splitter, clear=False) + load(state, hashring, clear=False) if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings logging.info("Bridges allotted for %s distribution: %d" % (emailDistributorTmp.name, - len(emailDistributorTmp.splitter))) + len(emailDistributorTmp.hashring))) else: logging.warn("No email distributor created!") @@ -380,15 +380,15 @@ def run(options, reactor=reactor): logging.info("Bridges allotted for %s distribution: %d" % (ipDistributorTmp.name, - len(ipDistributorTmp.splitter))) + len(ipDistributorTmp.hashring))) logging.info("\tNum bridges:\tFilter set:") nSubrings = 0 - ipSubrings = ipDistributorTmp.splitter.filterRings + ipSubrings = ipDistributorTmp.hashring.filterRings for (ringname, (filterFn, subring)) in ipSubrings.items(): nSubrings += 1 filterSet = ' '.join( - ipDistributorTmp.splitter.extractFilterNames(ringname)) + ipDistributorTmp.hashring.extractFilterNames(ringname)) logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) logging.info("Total subrings for %s: %d" @@ -403,7 +403,7 @@ def run(options, reactor=reactor): fh = open(state.ASSIGNMENTS_FILE, 'a') fh.write("bridge-pool-assignment %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")) - splitter.dumpAssignments(fh) + hashring.dumpAssignments(fh) fh.flush() fh.close() except IOError: diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 7d7d236..dc91c9b 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -311,7 +311,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - for b in d.splitter.bridges: + for b in d.hashring.bridges: # china blocks all :-( for pt in b.transports: key = "%s:%s" % (pt.address, pt.port) @@ -337,7 +337,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True, transports=True)) d.insert(fakeBridge(or_addresses=True, transports=True)) - for b in d.splitter.bridges: + for b in d.hashring.bridges: # china blocks some transports for pt in b.transports: if random.choice(xrange(2)) > 0: diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 43e008c..ec3fdd2 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -111,18 +111,18 @@ class MainTests(unittest.TestCase): return updatedPaths def _cbAssertFingerprints(self, d): - """Assert that there are some bridges in the splitter.""" - self.assertGreater(len(self.splitter), 0) + """Assert that there are some bridges in the hashring.""" + self.assertGreater(len(self.hashring), 0) return d - def _cbCallUpdateBridgeHistory(self, d, splitter): - """Fake some timestamps for the bridges in the splitter, and then call + def _cbCallUpdateBridgeHistory(self, d, hashring): + """Fake some timestamps for the bridges in the hashring, and then call Main.updateBridgeHistory(). """ def timestamp(): return datetime.fromtimestamp(random.randint(1324285117, 1524285117)) - bridges = splitter._bridges + bridges = hashring._bridges timestamps = {} for fingerprint, _ in bridges.items(): @@ -171,7 +171,7 @@ class MainTests(unittest.TestCase): self.key = base64.b64decode('TvPS1y36BFguBmSOvhChgtXB2Lt+BOw0mGfz9SZe12Y=') # Create a BridgeSplitter - self.splitter = MockBridgeHolder() + self.hashring = MockBridgeHolder() # Functions which some tests mock, which we'll need to re-replace # later in tearDown(): @@ -193,30 +193,30 @@ class MainTests(unittest.TestCase): # access the database: Main.updateBridgeHistory = mockUpdateBridgeHistory - # Get the bridges into the mocked splitter - d = deferToThread(Main.load, self.state, self.splitter) + # Get the bridges into the mocked hashring + d = deferToThread(Main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) - d.addCallback(self._cbCallUpdateBridgeHistory, self.splitter) + d.addCallback(self._cbCallUpdateBridgeHistory, self.hashring) d.addErrback(self._eb_Failure) return d def test_Main_load(self): """Main.load() should run without error.""" - d = deferToThread(Main.load, self.state, self.splitter) + d = deferToThread(Main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d def test_Main_load_no_state(self): """Main.load() should raise SystemExit without a state object.""" - self.assertRaises(SystemExit, Main.load, None, self.splitter) + self.assertRaises(SystemExit, Main.load, None, self.hashring) def test_Main_load_clear(self): """When called with clear=True, load() should run and clear the hashrings. """ - d = deferToThread(Main.load, self.state, self.splitter, clear=True) + d = deferToThread(Main.load, self.state, self.hashring, clear=True) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d @@ -234,14 +234,14 @@ class MainTests(unittest.TestCase): # The reactor is deferring this to a thread, so the test execution # here isn't actually covering the Storage.updateBridgeHistory() # function: - Main.load(state, self.splitter) + Main.load(state, self.hashring) def test_Main_load_malformed_networkstatus(self): """When called with a networkstatus file with an invalid descriptor, Main.load() should raise a ValueError. """ self._appendToFile(self.state.STATUS_FILE, NETWORKSTATUS_MALFORMED) - self.assertRaises(ValueError, Main.load, self.state, self.splitter) + self.assertRaises(ValueError, Main.load, self.state, self.hashring) def test_Main_reloadFn(self): """Main._reloadFn() should return True.""" @@ -255,79 +255,79 @@ class MainTests(unittest.TestCase): def test_Main_createBridgeRings(self): """Main.createBridgeRings() should add three hashrings to the - splitter. + hashring. """ proxyList = None - (splitter, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) def test_Main_createBridgeRings_with_proxyList(self): """Main.createBridgeRings() should add three hashrings to the - splitter and add the proxyList to the IPBasedDistibutor. + hashring and add the proxyList to the IPBasedDistibutor. """ exitRelays = ['1.1.1.1', '2.2.2.2', '3.3.3.3'] proxyList = Main.proxy.ProxySet() proxyList.addExitRelays(exitRelays) - (splitter, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertGreater(len(httpsDist.proxies), 0) self.assertItemsEqual(exitRelays, httpsDist.proxies) def test_Main_createBridgeRings_no_https_dist(self): """When HTTPS_DIST=False, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.HTTPS_DIST = False - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an EmailDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('https', splitter.rings) - self.assertNotIn(httpsDist, splitter.ringsByName.values()) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('https', hashring.rings) + self.assertNotIn(httpsDist, hashring.ringsByName.values()) def test_Main_createBridgeRings_no_email_dist(self): """When EMAIL_DIST=False, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.EMAIL_DIST = False - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('email', splitter.rings) - self.assertNotIn(emailDist, splitter.ringsByName.values()) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('email', hashring.rings) + self.assertNotIn(emailDist, hashring.ringsByName.values()) def test_Main_createBridgeRings_no_reserved_share(self): """When RESERVED_SHARE=0, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.RESERVED_SHARE = 0 - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, and an EmailDistributor ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('unallocated', splitter.rings) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('unallocated', hashring.rings) def test_Main_createBridgeRings_two_file_buckets(self): """When FILE_BUCKETS has two filenames in it, Main.createBridgeRings() - should add three hashrings to the splitter, then add two + should add three hashrings to the hashring, then add two "pseudo-rings". """ proxyList = Main.proxy.ProxySet() @@ -336,17 +336,17 @@ class MainTests(unittest.TestCase): 'bridges-for-support-desk': 10, 'bridges-for-ooni-tests': 10, } - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor, and an # UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) # Should have two pseudoRings: - self.assertEqual(len(splitter.pseudoRings), 2) - self.assertIn('pseudo_bridges-for-support-desk', splitter.pseudoRings) - self.assertIn('pseudo_bridges-for-ooni-tests', splitter.pseudoRings) + self.assertEqual(len(hashring.pseudoRings), 2) + self.assertIn('pseudo_bridges-for-support-desk', hashring.pseudoRings) + self.assertIn('pseudo_bridges-for-ooni-tests', hashring.pseudoRings) def test_Main_run(self): """Main.run() should run and then finally raise SystemExit.""" diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 88b858e..e48e20a 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -167,7 +167,7 @@ class BridgeIntegrationTests(unittest.TestCase): test_getConfigLine_vanilla_withoutFingerprint test_integration_getConfigLine_vanilla_withoutFingerprint test_getConfigLine_vanilla_withFingerprint test_integration_getConfigLine_vanilla_withFingerprint test_getConfigLine_scramblesuit_withFingeprint test_integration_getConfigLine_scramblesuit_withFingerprint - test_splitterBridgeInsertion test_integration_splitterBridgeInsertion + test_splitterBridgeInsertion test_integration_hashringBridgeInsertion ============================================== ======================== .. """ @@ -318,9 +318,9 @@ class BridgeIntegrationTests(unittest.TestCase): % (self.fingerprint, ptArgsList), bridgeLine) - def test_integration_splitterBridgeInsertion(self): + def test_integration_hashringBridgeInsertion(self): key = "Testing-Bridges-To-Rings" - splitter = FilteredBridgeSplitter(key) + hashring = FilteredBridgeSplitter(key) bridge1 = bridges.Bridge('unamed1', '1.2.3.5', 9100, 'a1cc8dfef1fa11af9c40af1054df9daf45250550') @@ -335,18 +335,18 @@ class BridgeIntegrationTests(unittest.TestCase): 'b1cc8dfef1fa11af9c40af1054df9daf45250552') bridge4.setStatus(running = True) - self.failUnlessEqual(len(splitter), 0) - splitter.insert(bridge1) - splitter.insert(bridge2) - splitter.insert(bridge3) + self.failUnlessEqual(len(hashring), 0) + hashring.insert(bridge1) + hashring.insert(bridge2) + hashring.insert(bridge3) # Check that all were inserted - self.failUnlessEqual(len(splitter), 3) - splitter.insert(bridge1) + self.failUnlessEqual(len(hashring), 3) + hashring.insert(bridge1) # Check that the same bridge is not inserted twice - self.failUnlessEqual(len(splitter), 3) - splitter.insert(bridge4) + self.failUnlessEqual(len(hashring), 3) + hashring.insert(bridge4) # Check that identical bridges are not inserted twice - self.failUnlessEqual(len(splitter), 3) + self.failUnlessEqual(len(hashring), 3) class FlagsTests(unittest.TestCase):

1 0

[bridgedb/master] Make `areaMapper` built into the IPBasedDistributor class.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 34a97ccc019ca5b9956b8566d81d893293ae5f00 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:12:23 2015 +0000 Make `areaMapper` built into the IPBasedDistributor class. * MOVE Dist.uniformMap() → Dist.IPBasedDistributor.getSubnet(). * REMOVE IPBasedDistributor `areaMapper` parameter. * RENAME IPBasedDistributor.areaOrderHmac → IPBasedDistributor._clientToPositionHMAC. * RENAME IPBasedDistributor.areaClusterHmac → IPBasedDistributor._subnetToSubringHMAC. * ADD IPBasedDistributor.mapClientToHashringPosition() to replace and simplify inline use of IPBasedDistributor._clientToPositionHMAC. * ADD IPBasedDistributor.mapSubnetToSubring() to replace and simplify inline use of IPBasedDistributor._subnetToSubringHMAC. --- lib/bridgedb/Dist.py | 184 ++++++++++++++++++++++++++++++-------------------- lib/bridgedb/Main.py | 1 - 2 files changed, 111 insertions(+), 74 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 1a01543..082f002 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -48,39 +48,6 @@ class EmailRequestedKey(Exception): """Raised when an incoming email requested a copy of our GnuPG keys.""" -def uniformMap(ip): - """Map an IP to an arbitrary 'area' string, such that any two IPv4 - addresses in the same ``/16`` subnet, or any two IPv6 addresses in the - same ``/32`` subnet, get the same string. - - >>> from bridgedb import Dist - >>> Dist.uniformMap('1.2.3.4') - '1.2.0.0/16' - >>> Dist.uniformMap('1.2.211.154') - '1.2.0.0/16' - >>> Dist.uniformMap('2001:f::bc1:b13:2808') - '2001:f::/32' - >>> Dist.uniformMap('2a00:c98:2030:a020:2::42') - '2a00:c98::/32' - - :param str ip: A string representing an IPv4 or IPv6 address. - :rtype: str - :returns: The appropriately sized CIDR subnet representation of the **ip**. - """ - # We aren't using bridgedb.parse.addr.isIPAddress(ip, compressed=False) - # here because adding the string "False" into the map would land any and - # all clients whose IP address appeared to be invalid at the same position - # in a hashring. - address = ipaddr.IPAddress(ip) - if address.version == 6: - truncated = ':'.join(address.exploded.split(':')[:2]) - subnet = str(ipaddr.IPv6Network(truncated + "::/32")) - return subnet - else: - truncated = '.'.join(address.exploded.split('.')[:2]) - subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) - return subnet - def getNumBridgesPerAnswer(ring, max_bridges_per_answer=3): if len(ring) < 20: n_bridges_per_answer = 1 @@ -111,9 +78,7 @@ class Distributor(object): hashrings will also carry that name. >>> from bridgedb import Dist - >>> ipDist = Dist.IPBasedDistributor(Dist.uniformMap, - ... 5, - ... 'fake-hmac-key') + >>> ipDist = Dist.IPBasedDistributor(5, 'fake-hmac-key') >>> ipDist.setDistributorName('HTTPS Distributor') >>> ipDist.prepopulateRings() >>> hashrings = ipDist.splitter.filterRings @@ -130,8 +95,6 @@ class IPBasedDistributor(Distributor): """A Distributor that hands out bridges based on the IP address of an incoming request and the current time period. - :ivar areaOrderHmac: An HMAC function used to order areas within rings. - :ivar areaClusterHmac: An HMAC function used to assign areas to rings. :ivar list rings: A list of :class:`bridgedb.Bridges.BridgeHolder` hashrings, one for each area in the ``areaMapper``. Every inserted bridge will go into one of these rings, and every area is associated @@ -145,19 +108,10 @@ class IPBasedDistributor(Distributor): distributor. """ - def __init__(self, areaMapper, numberOfClusters, key, - proxies=None, answerParameters=None): + def __init__(self, numberOfClusters, key, proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. - :type areaMapper: callable - :param areaMapper: A function that maps IP addresses arbitrarily to - strings, such that IP addresses which map to identical strings are - considered to be in the same "area". The default **areaMapper** - is :func:`bridgedb.Dist.uniformMap`, which maps all IPv4 addresses - within the same /16 and all IPv6 addresses within the same /32 to - the same area. Areas are then grouped into the number of rings - specified by the ``N_IP_CLUSTERS`` configuration option. :param integer numberOfClusters: The number of clusters to group IP addresses into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then the actual number of clusters is one higher than ``numberOfClusters``, @@ -180,7 +134,6 @@ class IPBasedDistributor(Distributor): bridges" or "at least one bridge on port 443", etc. """ self.rings = [] - self.areaMapper = areaMapper self.answerParameters = answerParameters self.numberOfClusters = numberOfClusters @@ -200,14 +153,110 @@ class IPBasedDistributor(Distributor): key3 = getHMAC(key, "Order-Areas-In-Rings") key4 = getHMAC(key, "Assign-Areas-To-Rings") - self.areaOrderHmac = getHMACFunc(key3, hex=False) - self.areaClusterHmac = getHMACFunc(key4, hex=True) + self._clientToPositionHMAC = getHMACFunc(key3, hex=False) + self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) self.splitter = FilteredBridgeSplitter(key2, self.ringCacheSize) logging.debug("Added %s to HTTPS distributor." % self.splitter.__class__.__name__) self.setDistributorName('HTTPS') + @classmethod + def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): + """Map all clients whose **ip**s are within the same subnet to the same + arbitrary string. + + .. hint:: For non-proxy IP addresses, any two IPv4 addresses within + the same ``/16`` subnet, or any two IPv6 addresses in the same + ``/32`` subnet, will get the same string. + + Subnets for this distributor are grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option, such that + Alice (with the address ``1.2.3.4`` and Bob (with the address + ``1.2.178.234``) are placed within the same cluster, but Carol (with + address ``1.3.11.33``) *might* end up in a different cluster. + + >>> from bridgedb.Dist import IPBasedDistributor + >>> IPBasedDistributor.getSubnet('1.2.3.4') + '1.2.0.0/16' + >>> IPBasedDistributor.getSubnet('1.2.211.154') + '1.2.0.0/16' + >>> IPBasedDistributor.getSubnet('2001:f::bc1:b13:2808') + '2001:f::/32' + >>> IPBasedDistributor.getSubnet('2a00:c98:2030:a020:2::42') + '2a00:c98::/32' + + :param str ip: A string representing an IPv4 or IPv6 address. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + :param int proxySubnets: Place Tor/proxy users into this number of + "subnet" groups. This means that no matter how many different Tor + Exits or proxies a client uses, the most they can ever get is + **proxySubnets** different sets of bridge lines (per interval). + This parameter only has any effect when **usingProxy** is ``True``. + :rtype: str + :returns: The appropriately sized CIDR subnet representation of the **ip**. + """ + if not usingProxy: + # We aren't using bridgedb.parse.addr.isIPAddress(ip, + # compressed=False) here because adding the string "False" into + # the map would land any and all clients whose IP address appeared + # to be invalid at the same position in a hashring. + address = ipaddr.IPAddress(ip) + if address.version == 6: + truncated = ':'.join(address.exploded.split(':')[:2]) + subnet = str(ipaddr.IPv6Network(truncated + "::/32")) + else: + truncated = '.'.join(address.exploded.split('.')[:2]) + subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) + else: + group = (int(ipaddr.IPAddress(ip)) % 4) + 1 + subnet = "proxy-group-%d" % group + + logging.debug("Client IP was within area: %s" % subnet) + return subnet + + def mapSubnetToSubring(self, subnet, usingProxy=False): + """Determine the correct subhashring for a client, based upon the + **subnet**. + + :param str subnet: The subnet which contains the client's IP. See + :staticmethod:`getSubnet`. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + """ + # If the client wasn't using a proxy, select the client's subring + # based upon the client's subnet (modulo the total subrings): + if not usingProxy: + mod = self.numberOfClusters + # If there is a proxy subring, don't count it for the modulus: + if self.proxyCluster: + mod -= 1 + return int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod + else: + return self.proxyCluster + + def mapClientToHashringPosition(self, interval, subnet): + """Map the client to a position on a (sub)hashring, based upon the + **interval** which the client's request occurred within, as well as + the **subnet** of the client's IP address. + + .. note:: For an explanation of how **subnet** is determined, see + :staticmethod:`getSubnet`. + + :param str interval: The interval which this client's request for + bridges took place within. + :param str subnet: A string representing the subnet containing the + client's IP address. + :rtype: int + :returns: The results of keyed HMAC, which should determine the + client's position in a (sub)hashring of bridges (and thus + determine which bridges they receive). + """ + position = "<%s>%s" % (interval, subnet) + mapping = self._clientToPositionHMAC(position) + return mapping + def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. @@ -304,6 +353,8 @@ class IPBasedDistributor(Distributor): logging.warn("Bailing! Splitter has zero bridges!") return [] + usingProxy = False + # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: cluster = self.proxyCluster @@ -315,30 +366,17 @@ class IPBasedDistributor(Distributor): tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) - # Place Tor/proxy users into four groups. This means that no - # matter how many different Tor Exits or proxies a client - # uses, the most they can ever get is four different sets of - # bridge lines (per period). - group = (int(ipaddr.IPAddress(bridgeRequest.client)) % 4) + 1 - area = "known-proxy-group-%d" % group - # If the client wasn't using a proxy, select the client's cluster - # based upon the client's area (i.e. the /16 of the client's IP - # address): - else: - area = self.areaMapper(bridgeRequest.client) - cluster = (int(self.areaClusterHmac(area)[:8], 16) - % (self.numberOfClusters - 1)) - pos = self.areaOrderHmac("<%s>%s" % (interval, area)) + subnet = self.getSubnet(bridgeRequest.client, usingProxy) + cluster = self.mapSubnetToSubring(subnet, usingProxy) + position = self.mapClientToHashringPosition(interval, subnet) filters = self._buildHashringFilters(bridgeRequest.filters, cluster) - logging.debug("Assigned client to cluster %d/%d" % - (cluster, self.numberOfClusters)) - logging.debug("Assigned client hashring position based on: <%s>%s" % - (interval, area)) - logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) - logging.debug("Active bridge filters:\t%s" % - ' '.join([x.func_name for x in filters])) + logging.debug("Client request within time interval: %s" % interval) + logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) + logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Total bridges: %d" % len(self.hashring)) + logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.splitter.filterRings.keys(): @@ -354,7 +392,7 @@ class IPBasedDistributor(Distributor): # Determine the appropriate number of bridges to give to the client: returnNum = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) - answer = ring.getBridges(pos, returnNum) + answer = ring.getBridges(position, returnNum) return answer diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 23ed262..d2fd2da 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -211,7 +211,6 @@ def createBridgeRings(cfg, proxyList, key): if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") ipDistributor = Dist.IPBasedDistributor( - Dist.uniformMap, cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList,

1 0

[bridgedb/master] Remove IPBasedDistributor.rings.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 809f447f3c8a7e0023a399de469b77a8b4c2735f Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:25:45 2015 +0000 Remove IPBasedDistributor.rings. It was never used. --- lib/bridgedb/Dist.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 082f002..c9d82c1 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -95,10 +95,6 @@ class IPBasedDistributor(Distributor): """A Distributor that hands out bridges based on the IP address of an incoming request and the current time period. - :ivar list rings: A list of :class:`bridgedb.Bridges.BridgeHolder` - hashrings, one for each area in the ``areaMapper``. Every inserted - bridge will go into one of these rings, and every area is associated - with one. :type proxies: :class:`~bridgedb.proxies.ProxySet` :ivar proxies: All known proxies, which we treat differently. See :param:`proxies`. @@ -133,7 +129,6 @@ class IPBasedDistributor(Distributor): parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ - self.rings = [] self.answerParameters = answerParameters self.numberOfClusters = numberOfClusters

1 0

[bridgedb/master] Remove `countryCode` parameter from BridgeRing.getBridges().
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 509fb2094d6cc02ba889bbc0a6883447e3131e7e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 04:14:52 2015 +0000 Remove `countryCode` parameter from BridgeRing.getBridges(). It was never used. And it's unnecessary. --- lib/bridgedb/Bridges.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 988d464..a13a73e 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -283,28 +283,17 @@ class BridgeRing(BridgeHolder): assert len(r) == N return r - def getBridges(self, pos, N=1, countryCode=None): + def getBridges(self, pos, N=1): """Return **N** bridges appearing in this hashring after a position. :param bytes pos: The position to jump to. Any bridges returned will - start at this position in the hashring, if there is - a bridge assigned to that position. Otherwise, - indexing will start at the first position after this - one which has a bridge assigned to it. + start at this position in the hashring, if there is a bridge + assigned to that position. Otherwise, indexing will start at the + first position after this one which has a bridge assigned to it. :param int N: The number of bridges to return. - :type countryCode: str or None - :param countryCode: DOCDOC :rtype: list - :returns: A list of :class:`~bridgedb.Bridges.Bridge`s. + :returns: A list of :class:`~bridgedb.bridges.Bridge`s. """ - # XXX This can be removed after we determine if countryCode is ever - # actually being used. It seems the countryCode should be passed in - # from bridgedb.https.server.WebResource.getBridgeRequestAnswer() in - # order to hand out bridges which are believed to not be blocked in a - # given country. - if countryCode: - logging.debug("getBridges: countryCode=%r" % countryCode) - forced = [] for _, _, count, subring in self.subrings: if len(subring) < count:

1 0

[bridgedb/master] Whitespace fixes in bridgedb.https.server module.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit e62191b6d484074d3d1fa30ef0cd4906822eaf2c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 05:10:47 2015 +0000 Whitespace fixes in bridgedb.https.server module. --- lib/bridgedb/https/server.py | 106 +++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index ab15ef3..6010218 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -6,8 +6,8 @@ # :copyright: (c) 2007-2015, The Tor Project, Inc. # (c) 2013-2015, Isis Lovecruft # :license: see LICENSE for licensing information - -""" + +""" .. py:module:: bridgedb.https.server :synopsis: Servers which interface with clients and distribute bridges over HTTP(S). @@ -16,16 +16,16 @@ bridgedb.https.server ===================== Servers which interface with clients and distribute bridges over HTTP(S). -""" - -import base64 -import gettext -import logging +""" + +import base64 +import gettext +import logging import random -import re -import time -import os - +import re +import time +import os + from functools import partial from ipaddr import IPv4Address @@ -34,14 +34,14 @@ import mako.exceptions from mako.template import Template from mako.lookup import TemplateLookup -from twisted.internet import reactor +from twisted.internet import reactor from twisted.internet.error import CannotListenError from twisted.web import resource -from twisted.web import static +from twisted.web import static from twisted.web.server import NOT_DONE_YET from twisted.web.server import Site -from twisted.web.util import redirectTo - +from twisted.web.util import redirectTo + from bridgedb import captcha from bridgedb import crypto from bridgedb import strings @@ -49,8 +49,8 @@ from bridgedb import translations from bridgedb import txrecaptcha from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.Filters import filterBridgesByTransport +from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress @@ -63,7 +63,7 @@ from bridgedb.util import replaceControlChars TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') - + # Setting `filesystem_checks` to False is recommended for production servers, # due to potential speed increases. This means that the atimes of the Mako # template files aren't rechecked every time the template is requested @@ -214,10 +214,10 @@ class CaptchaProtectedResource(resource.Resource): resource.Resource.__init__(self) self.publicKey = publicKey self.secretKey = secretKey - self.useForwardedHeader = useForwardedHeader + self.useForwardedHeader = useForwardedHeader self.resource = protectedResource - - def getClientIP(self, request): + + def getClientIP(self, request): """Get the client's IP address from the :header:`X-Forwarded-For` header, or from the :api:`request <twisted.web.server.Request>`. @@ -228,7 +228,7 @@ class CaptchaProtectedResource(resource.Resource): :returns: The client's IP address, if it was obtainable. """ return getClientIP(request, self.useForwardedHeader) - + def getCaptchaImage(self, request=None): """Get a CAPTCHA image. @@ -269,7 +269,7 @@ class CaptchaProtectedResource(resource.Resource): """ return False - def render_GET(self, request): + def render_GET(self, request): """Retrieve a ReCaptcha from the API server and serve it to the client. :type request: :api:`twisted.web.http.Request` @@ -298,8 +298,8 @@ class CaptchaProtectedResource(resource.Resource): request.setHeader("Content-Type", "text/html; charset=utf-8") return rendered - - def render_POST(self, request): + + def render_POST(self, request): """Process a client's CAPTCHA solution. If the client's CAPTCHA solution is valid (according to @@ -328,8 +328,8 @@ class CaptchaProtectedResource(resource.Resource): logging.debug("Client failed a CAPTCHA; returning redirect to %s" % request.uri) return redirectTo(request.uri, request) - - + + class GimpCaptchaProtectedResource(CaptchaProtectedResource): """A web resource which uses a local cache of CAPTCHAs, generated with gimp-captcha_, to protect another resource. @@ -626,9 +626,9 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): class BridgesResource(resource.Resource): """This resource displays bridge lines in response to a request.""" - isLeaf = True - - def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, + isLeaf = True + + def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, includeFingerprints=True): """Create a new resource for displaying bridges to a client. @@ -643,16 +643,16 @@ class BridgesResource(resource.Resource): X-Forwarded-For header instead of the source IP address. :param bool includeFingerprints: Do we include the bridge's fingerprint in the response? - """ - gettext.install("bridgedb", unicode=True) + """ + gettext.install("bridgedb", unicode=True) resource.Resource.__init__(self) - self.distributor = distributor - self.schedule = schedule - self.nBridgesToGive = N - self.useForwardedHeader = useForwardedHeader - self.includeFingerprints = includeFingerprints - - def render(self, request): + self.distributor = distributor + self.schedule = schedule + self.nBridgesToGive = N + self.useForwardedHeader = useForwardedHeader + self.includeFingerprints = includeFingerprints + + def render(self, request): """Render a response for a client HTTP request. Presently, this method merely wraps :meth:`getBridgeRequestAnswer` to @@ -674,7 +674,7 @@ class BridgesResource(resource.Resource): response = self.renderAnswer(request) return response - + def getClientIP(self, request): """Get the client's IP address from the :header:`X-Forwarded-For` header, or from the :api:`request <twisted.web.server.Request>`. @@ -687,7 +687,7 @@ class BridgesResource(resource.Resource): """ return getClientIP(request, self.useForwardedHeader) - def getBridgeRequestAnswer(self, request): + def getBridgeRequestAnswer(self, request): """Respond to a client HTTP request for bridges. :type request: :api:`twisted.web.http.Request` @@ -702,7 +702,7 @@ class BridgesResource(resource.Resource): logging.info("Replying to web request from %s. Parameters were %r" % (ip, request.args)) - + if ip: bridgeRequest = HTTPSBridgeRequest() bridgeRequest.client = ip @@ -717,10 +717,10 @@ class BridgesResource(resource.Resource): bridgeRequest, self.includeFingerprints)) for bridge in bridges] return self.renderAnswer(request, bridgeLines) - + def getResponseFormat(self, request): """Determine the requested format for the response. - + :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object containing the HTTP method, full URI, and any URL/POST arguments and headers present. @@ -734,16 +734,16 @@ class BridgesResource(resource.Resource): if format and len(format): format = format[0] # Choose the first arg return format - + def renderAnswer(self, request, bridgeLines=None): """Generate a response for a client which includes **bridgesLines**. - + .. note: The generated response can be plain or HTML. A plain response looks like:: voltron 1.2.3.4:1234 ABCDEF01234567890ABCDEF01234567890ABCDEF voltron 5.5.5.5:5555 0123456789ABCDEF0123456789ABCDEF01234567 - + That is, there is no HTML, what you see is what you get, and what you get is suitable for pasting directly into Tor Launcher (or into a torrc, if you prepend ``"Bridge "`` to each line). The @@ -824,7 +824,7 @@ def addWebServer(config, distributor): :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` :returns: A webserver. - """ + """ captcha = None fwdHeaders = config.HTTP_USE_IP_FROM_FORWARDED_HEADER numBridges = config.HTTPS_N_BRIDGES_PER_ANSWER @@ -879,12 +879,12 @@ def addWebServer(config, distributor): protectedResource=bridges) root.putChild('bridges', protected) logging.info("Protecting resources with %s." % captcha.func.__name__) - else: + else: root.putChild('bridges', bridges) site = Site(root) site.displayTracebacks = False - + if config.HTTP_UNENCRYPTED_PORT: # pragma: no cover ip = config.HTTP_UNENCRYPTED_BIND_IP or "" port = config.HTTP_UNENCRYPTED_PORT or 80 @@ -893,7 +893,7 @@ def addWebServer(config, distributor): except CannotListenError as error: raise SystemExit(error) logging.info("Started HTTP server on %s:%d" % (str(ip), int(port))) - + if config.HTTPS_PORT: # pragma: no cover ip = config.HTTPS_BIND_IP or "" port = config.HTTPS_PORT or 443 @@ -905,5 +905,5 @@ def addWebServer(config, distributor): except CannotListenError as error: raise SystemExit(error) logging.info("Started HTTPS server on %s:%d" % (str(ip), int(port))) - - return site + + return site

1 0

[bridgedb/master] Remove the idea of clusters. Clusters are just subhashrings.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 1a676f343a7a5a854106c9e90c8418d1f7b2c09e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:33:38 2015 +0000 Remove the idea of clusters. Clusters are just subhashrings. --- lib/bridgedb/Dist.py | 60 +++++++++++++++++++++++------------------------ lib/bridgedb/Filters.py | 2 +- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index c9d82c1..58f0a2d 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -104,16 +104,15 @@ class IPBasedDistributor(Distributor): distributor. """ - def __init__(self, numberOfClusters, key, proxies=None, answerParameters=None): + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. - :param integer numberOfClusters: The number of clusters to group IP addresses - into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then - the actual number of clusters is one higher than ``numberOfClusters``, - because the set of known open proxies constitutes its own - category. - DOCDOC What exactly does a cluster *do*? + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. :param bytes key: The master HMAC key for this distributor. All added bridges are HMACed with this key in order to place them into the hashrings. @@ -129,20 +128,20 @@ class IPBasedDistributor(Distributor): parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ + self.totalSubrings = totalSubrings self.answerParameters = answerParameters - self.numberOfClusters = numberOfClusters if proxies: logging.info("Added known proxies to HTTPS distributor...") self.proxies = proxies - self.numberOfClusters += 1 - self.proxyCluster = self.numberOfClusters + self.totalSubrings += 1 + self.proxySubring = self.totalSubrings else: logging.warn("No known proxies were added to HTTPS distributor!") self.proxies = proxy.ProxySet() - self.proxyCluster = 0 + self.proxySubring = 0 - self.ringCacheSize = self.numberOfClusters * 3 + self.ringCacheSize = self.totalSubrings * 3 key2 = getHMAC(key, "Assign-Bridges-To-Rings") key3 = getHMAC(key, "Order-Areas-In-Rings") @@ -223,13 +222,13 @@ class IPBasedDistributor(Distributor): # If the client wasn't using a proxy, select the client's subring # based upon the client's subnet (modulo the total subrings): if not usingProxy: - mod = self.numberOfClusters + mod = self.totalSubrings # If there is a proxy subring, don't count it for the modulus: - if self.proxyCluster: + if self.proxySubring: mod -= 1 - return int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod + return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 else: - return self.proxyCluster + return self.proxySubring def mapClientToHashringPosition(self, interval, subnet): """Map the client to a position on a (sub)hashring, based upon the @@ -260,14 +259,14 @@ class IPBasedDistributor(Distributor): ``N_IP_CLUSTERS`` configuration option, as well as the number of ``PROXY_LIST_FILES``. - Essentially, :data:`numberOfClusters` is set to the specified + Essentially, :data:`totalSubrings` is set to the specified ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), are stored in :data:`proxies`, and the latter is added as an - additional cluster (such that :data:`numberOfClusters` becomes + additional cluster (such that :data:`totalSubrings` becomes ``N_IP_CLUSTERS + 1``). The number of subhashrings which this :class:`Distributor` has active in its hashring is then - :data:`numberOfClusters`, where the last cluster is reserved for all + :data:`totalSubrings`, where the last cluster is reserved for all :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and @@ -302,14 +301,14 @@ class IPBasedDistributor(Distributor): logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: - for cluster in range(1, self.numberOfClusters): - filters = self._buildHashringFilters([filterFn,], cluster) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) + for subring in range(1, self.totalSubrings + 1): + filters = self._buildHashringFilters([filterFn,], subring) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this # distributor's proxies: - if cluster == self.proxyCluster: + if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), @@ -319,10 +318,9 @@ class IPBasedDistributor(Distributor): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def _buildHashringFilters(self, previousFilters, clientCluster): - g = filterAssignBridgesToRing(self.splitter.hmac, - self.numberOfClusters, clientCluster) - previousFilters.append(g) + def _buildHashringFilters(self, previousFilters, subring): + f = filterAssignBridgesToRing(self.splitter.hmac, self.totalSubrings, subring) + previousFilters.append(f) return frozenset(previousFilters) def getBridges(self, bridgeRequest, interval, N=1): @@ -352,20 +350,20 @@ class IPBasedDistributor(Distributor): # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: - cluster = self.proxyCluster # The tag is a tag applied to a proxy IP address when it is added # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default # is 'exit_relay'. For other proxies loaded from the # PROXY_LIST_FILES config option, the default tag is the full # filename that the IP address originally came from. + usingProxy = True tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) subnet = self.getSubnet(bridgeRequest.client, usingProxy) - cluster = self.mapSubnetToSubring(subnet, usingProxy) + subring = self.mapSubnetToSubring(subnet, usingProxy) position = self.mapClientToHashringPosition(interval, subnet) - filters = self._buildHashringFilters(bridgeRequest.filters, cluster) + filters = self._buildHashringFilters(bridgeRequest.filters, subring) logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) @@ -380,7 +378,7 @@ class IPBasedDistributor(Distributor): # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index aade069..94d4325 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -19,7 +19,7 @@ def filterAssignBridgesToRing(hmac, numRings, assignedRing): def _assignBridgesToRing(bridge): digest = hmac(bridge.getID()) pos = long( digest[:8], 16 ) - which = pos % numRings + which = pos % numRings + 1 if which == assignedRing: return True

1 0