June 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Make subhashring cache size relative to numberOfClusters.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 397cc7d316a618bcadfca6663f318d3b798f4405 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Apr 11 02:53:20 2015 +0000 Make subhashring cache size relative to numberOfClusters. * RENAME ring_cache_size → IPBasedDistributor.ringCacheSize. * CHANGE IPBasedDistributor.ringCacheSize to be relative to the total number of clusters, rather than equal to the total number of clusters plus five. --- lib/bridgedb/Dist.py | 19 +++++++-----------… [View More]

1 0

[bridgedb/develop] Remove most "splitter"s — they're just crappy consistent hashrings.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit d83438d24104c5834cd0230a757b17a8e1941edb Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 09:23:40 2015 +0000 Remove most "splitter"s — they're just crappy consistent hashrings. --- lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 60 ++++++++++++++-------------- lib/bridgedb/Main.py | 66 +++++++++++++++--------------- lib/bridgedb/test/legacy_Tests.py | 4 +- lib/bridgedb/test/test_Main.py | 80 +++++++… [View More]+++++++++++------------------- lib/bridgedb/test/test_bridges.py | 24 +++++------ 6 files changed, 118 insertions(+), 118 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index c727c9a..b06ffc4 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -569,7 +569,7 @@ class FilteredBridgeSplitter(BridgeHolder): return index = 0 - logging.debug("Inserting %s into splitter" + logging.debug("Inserting %s into hashring" % (logSafely(bridge.fingerprint))) for old_bridge in self.bridges[:]: if bridge.fingerprint == old_bridge.fingerprint: diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 445d2d8..44cb526 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -81,14 +81,14 @@ class Distributor(object): >>> ipDist = Dist.IPBasedDistributor(5, 'fake-hmac-key') >>> ipDist.setDistributorName('HTTPS Distributor') >>> ipDist.prepopulateRings() - >>> hashrings = ipDist.splitter.filterRings + >>> hashrings = ipDist.hashring.filterRings >>> firstSubring = hashrings.items()[0][1][1] >>> assert firstSubring.name :param str name: A name for this distributor. """ self.name = name - self.splitter.distributorName = name + self.hashring.distributorName = name class IPBasedDistributor(Distributor): @@ -98,8 +98,8 @@ class IPBasedDistributor(Distributor): :type proxies: :class:`~bridgedb.proxies.ProxySet` :ivar proxies: All known proxies, which we treat differently. See :param:`proxies`. - :type splitter: :class:`bridgedb.Bridges.FixedBridgeSplitter` - :ivar splitter: A hashring that assigns bridges to subrings with fixed + :type hashring: :class:`bridgedb.Bridges.FixedBridgeSplitter` + :ivar hashring: A hashring that assigns bridges to subrings with fixed proportions. Used to assign bridges into the subrings of this distributor. """ @@ -150,9 +150,9 @@ class IPBasedDistributor(Distributor): self._clientToPositionHMAC = getHMACFunc(key3, hex=False) self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) - self.splitter = FilteredBridgeSplitter(key2, self.ringCacheSize) + self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) logging.debug("Added %s to HTTPS distributor." % - self.splitter.__class__.__name__) + self.hashring.__class__.__name__) self.setDistributorName('HTTPS') @@ -311,16 +311,16 @@ class IPBasedDistributor(Distributor): # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) - self.splitter.addRing(ring, filters, + self.hashring.addRing(ring, filters, filterBridgesByRules(filters), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) def insert(self, bridge): """Assign a bridge to this distributor.""" - self.splitter.insert(bridge) + self.hashring.insert(bridge) def _buildHashringFilters(self, previousFilters, subring): - f = filterAssignBridgesToRing(self.splitter.hmac, self.totalSubrings, subring) + f = filterAssignBridgesToRing(self.hashring.hmac, self.totalSubrings, subring) previousFilters.append(f) return frozenset(previousFilters) @@ -343,8 +343,8 @@ class IPBasedDistributor(Distributor): logging.info("Attempting to return %d bridges to client %s..." % (N, bridgeRequest.client)) - if not len(self.splitter): - logging.warn("Bailing! Splitter has zero bridges!") + if not len(self.hashring): + logging.warn("Bailing! Hashring has zero bridges!") return [] usingProxy = False @@ -373,16 +373,16 @@ class IPBasedDistributor(Distributor): logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: - if filters in self.splitter.filterRings.keys(): + if filters in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % filters) - _, ring = self.splitter.filterRings[filters] + _, ring = self.hashring.filterRings[filters] # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, filters, filterBridgesByRules(filters), - populate_from=self.splitter.bridges) + self.hashring.addRing(ring, filters, filterBridgesByRules(filters), + populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: returnNum = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) @@ -391,18 +391,18 @@ class IPBasedDistributor(Distributor): return answer def __len__(self): - return len(self.splitter) + return len(self.hashring) def dumpAssignments(self, f, description=""): - self.splitter.dumpAssignments(f, description) + self.hashring.dumpAssignments(f, description) class EmailBasedDistributor(Distributor): """Object that hands out bridges based on the email address of an incoming request and the current time period. - :type splitter: :class:`~bridgedb.Bridges.BridgeRing` - :ivar splitter: A hashring to hold all the bridges we hand out. + :type hashring: :class:`~bridgedb.Bridges.BridgeRing` + :ivar hashring: A hashring to hold all the bridges we hand out. """ def __init__(self, key, domainmap, domainrules, @@ -434,14 +434,14 @@ class EmailBasedDistributor(Distributor): self.answerParameters = answerParameters #XXX cache options not implemented - self.splitter = bridgedb.Bridges.FilteredBridgeSplitter( + self.hashring = bridgedb.Bridges.FilteredBridgeSplitter( key2, max_cached_rings=5) self.setDistributorName('Email') def insert(self, bridge): """Assign a bridge to this distributor.""" - self.splitter.insert(bridge) + self.hashring.insert(bridge) def getBridges(self, bridgeRequest, interval, N=1): """Return a list of bridges to give to a user. @@ -500,9 +500,9 @@ class EmailBasedDistributor(Distributor): ring = None ruleset = frozenset(bridgeRequest.filters) - if ruleset in self.splitter.filterRings.keys(): + if ruleset in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) - _, ring = self.splitter.filterRings[ruleset] + _, ring = self.hashring.filterRings[ruleset] else: # cache miss, add new ring logging.debug("Cache miss %s" % ruleset) @@ -510,9 +510,9 @@ class EmailBasedDistributor(Distributor): # add new ring key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, + self.hashring.addRing(ring, ruleset, filterBridgesByRules(ruleset), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) numBridgesToReturn = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) @@ -524,7 +524,7 @@ class EmailBasedDistributor(Distributor): return result def __len__(self): - return len(self.splitter) + return len(self.hashring) def cleanDatabase(self): with bridgedb.Storage.getDB() as db: @@ -538,7 +538,7 @@ class EmailBasedDistributor(Distributor): db.commit() def dumpAssignments(self, f, description=""): - self.splitter.dumpAssignments(f, description) + self.hashring.dumpAssignments(f, description) def prepopulateRings(self): # populate all rings (for dumping assignments and testing) @@ -546,6 +546,6 @@ class EmailBasedDistributor(Distributor): ruleset = frozenset([filterFn]) key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) - self.splitter.addRing(ring, ruleset, + self.hashring.addRing(ring, ruleset, filterBridgesByRules([filterFn]), - populate_from=self.splitter.bridges) + populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index d2fd2da..53189fd 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -39,19 +39,19 @@ from bridgedb import Dist from bridgedb.Stability import updateBridgeHistory -def load(state, splitter, clear=False): - """Read and parse all descriptors, and load into a bridge splitter. +def load(state, hashring, clear=False): + """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then - store them into our ``state.splitter`` instance. The ``state`` will be + store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. - :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` - :param splitter: A class which provides a mechanism for HMACing + :type hashring: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` + :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the - splitter before parsing for new ones. + hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.Main.load() could not retrieve state!") @@ -59,7 +59,7 @@ def load(state, splitter, clear=False): if clear: logging.info("Clearing old bridges...") - splitter.clear() + hashring.clear() logging.info("Loading bridges...") @@ -137,7 +137,7 @@ def load(state, splitter, clear=False): % router.fingerprint) inserted = 0 - logging.info("Inserting %d bridges into splitter..." % len(bridges)) + logging.info("Inserting %d bridges into hashring..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute @@ -148,9 +148,9 @@ def load(state, splitter, clear=False): else: # If the bridge is not running, then it is skipped during the # insertion process. - splitter.insert(bridge) + hashring.insert(bridge) inserted += 1 - logging.info("Done inserting %d bridges into splitter." % inserted) + logging.info("Done inserting %d bridges into hashring." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) @@ -180,7 +180,7 @@ def _handleSIGUSR1(*args): def replaceBridgeRings(current, replacement): """Replace the current thing with the new one""" - current.splitter = replacement.splitter + current.hashring = replacement.hashring def createBridgeRings(cfg, proxyList, key): """Create the bridge distributors defined by the config file @@ -192,15 +192,15 @@ def createBridgeRings(cfg, proxyList, key): :type proxyList: :class:`~bridgedb.proxy.ProxySet` :param proxyList: The container for the IP addresses of any currently known open proxies. - :param bytes key: Splitter master key + :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter splitter, an IPBasedDistributor or None, + :returns: A BridgeSplitter hashring, an IPBasedDistributor or None, and an EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. - splitter = Bridges.BridgeSplitter(crypto.getHMAC(key, "Splitter-Key")) - logging.debug("Created splitter: %r" % splitter) + hashring = Bridges.BridgeSplitter(crypto.getHMAC(key, "Hashring-Key")) + logging.debug("Created hashring: %r" % hashring) # Create ring parameters. ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS, @@ -215,7 +215,7 @@ def createBridgeRings(cfg, proxyList, key): crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, answerParameters=ringParams) - splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) + hashring.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) # As appropriate, create an email-based distributor. if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: @@ -226,19 +226,19 @@ def createBridgeRings(cfg, proxyList, key): cfg.EMAIL_DOMAIN_RULES.copy(), answerParameters=ringParams, whitelist=cfg.EMAIL_WHITELIST.copy()) - splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) + hashring.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) - # As appropriate, tell the splitter to leave some bridges unallocated. + # As appropriate, tell the hashring to leave some bridges unallocated. if cfg.RESERVED_SHARE: - splitter.addRing(Bridges.UnallocatedHolder(), + hashring.addRing(Bridges.UnallocatedHolder(), "unallocated", cfg.RESERVED_SHARE) - # Add pseudo distributors to splitter + # Add pseudo distributors to hashring for pseudoRing in cfg.FILE_BUCKETS.keys(): - splitter.addPseudoRing(pseudoRing) + hashring.addPseudoRing(pseudoRing) - return splitter, emailDistributor, ipDistributor + return hashring, emailDistributor, ipDistributor def run(options, reactor=reactor): """This is BridgeDB's main entry point and main runtime loop. @@ -314,7 +314,7 @@ def run(options, reactor=reactor): State should be saved before calling this method, and will be saved again at the end of it. - The internal variables, ``cfg``, ``splitter``, ``proxyList``, + The internal variables, ``cfg``, ``hashring``, ``proxyList``, ``ipDistributor``, and ``emailDistributor`` are all taken from a :class:`~bridgedb.persistent.State` instance, which has been saved to a statefile with :meth:`bridgedb.persistent.State.save`. @@ -323,8 +323,8 @@ def run(options, reactor=reactor): :ivar cfg: The current configuration, including any in-memory settings (i.e. settings whose values were not obtained from the config file, but were set via a function somewhere) - :type splitter: A :class:`bridgedb.Bridges.BridgeHolder` - :ivar splitter: A class which takes an HMAC key and splits bridges + :type hashring: A :class:`bridgedb.Bridges.BridgeHolder` + :ivar hashring: A class which takes an HMAC key and splits bridges into their hashring assignments. :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently @@ -357,21 +357,21 @@ def run(options, reactor=reactor): proxy.loadProxiesFromFile(proxyfile, state.proxies, removeStale=True) logging.info("Reparsing bridge descriptors...") - (splitter, + (hashring, emailDistributorTmp, ipDistributorTmp) = createBridgeRings(cfg, state.proxies, key) - logging.info("Bridges loaded: %d" % len(splitter)) + logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. bridgedb.Storage.initializeDBLock() bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") - load(state, splitter, clear=False) + load(state, hashring, clear=False) if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings logging.info("Bridges allotted for %s distribution: %d" % (emailDistributorTmp.name, - len(emailDistributorTmp.splitter))) + len(emailDistributorTmp.hashring))) else: logging.warn("No email distributor created!") @@ -380,15 +380,15 @@ def run(options, reactor=reactor): logging.info("Bridges allotted for %s distribution: %d" % (ipDistributorTmp.name, - len(ipDistributorTmp.splitter))) + len(ipDistributorTmp.hashring))) logging.info("\tNum bridges:\tFilter set:") nSubrings = 0 - ipSubrings = ipDistributorTmp.splitter.filterRings + ipSubrings = ipDistributorTmp.hashring.filterRings for (ringname, (filterFn, subring)) in ipSubrings.items(): nSubrings += 1 filterSet = ' '.join( - ipDistributorTmp.splitter.extractFilterNames(ringname)) + ipDistributorTmp.hashring.extractFilterNames(ringname)) logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) logging.info("Total subrings for %s: %d" @@ -403,7 +403,7 @@ def run(options, reactor=reactor): fh = open(state.ASSIGNMENTS_FILE, 'a') fh.write("bridge-pool-assignment %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")) - splitter.dumpAssignments(fh) + hashring.dumpAssignments(fh) fh.flush() fh.close() except IOError: diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 7d7d236..dc91c9b 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -311,7 +311,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) - for b in d.splitter.bridges: + for b in d.hashring.bridges: # china blocks all :-( for pt in b.transports: key = "%s:%s" % (pt.address, pt.port) @@ -337,7 +337,7 @@ class IPBridgeDistTests(unittest.TestCase): d.insert(fakeBridge6(or_addresses=True, transports=True)) d.insert(fakeBridge(or_addresses=True, transports=True)) - for b in d.splitter.bridges: + for b in d.hashring.bridges: # china blocks some transports for pt in b.transports: if random.choice(xrange(2)) > 0: diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 43e008c..ec3fdd2 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -111,18 +111,18 @@ class MainTests(unittest.TestCase): return updatedPaths def _cbAssertFingerprints(self, d): - """Assert that there are some bridges in the splitter.""" - self.assertGreater(len(self.splitter), 0) + """Assert that there are some bridges in the hashring.""" + self.assertGreater(len(self.hashring), 0) return d - def _cbCallUpdateBridgeHistory(self, d, splitter): - """Fake some timestamps for the bridges in the splitter, and then call + def _cbCallUpdateBridgeHistory(self, d, hashring): + """Fake some timestamps for the bridges in the hashring, and then call Main.updateBridgeHistory(). """ def timestamp(): return datetime.fromtimestamp(random.randint(1324285117, 1524285117)) - bridges = splitter._bridges + bridges = hashring._bridges timestamps = {} for fingerprint, _ in bridges.items(): @@ -171,7 +171,7 @@ class MainTests(unittest.TestCase): self.key = base64.b64decode('TvPS1y36BFguBmSOvhChgtXB2Lt+BOw0mGfz9SZe12Y=') # Create a BridgeSplitter - self.splitter = MockBridgeHolder() + self.hashring = MockBridgeHolder() # Functions which some tests mock, which we'll need to re-replace # later in tearDown(): @@ -193,30 +193,30 @@ class MainTests(unittest.TestCase): # access the database: Main.updateBridgeHistory = mockUpdateBridgeHistory - # Get the bridges into the mocked splitter - d = deferToThread(Main.load, self.state, self.splitter) + # Get the bridges into the mocked hashring + d = deferToThread(Main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) - d.addCallback(self._cbCallUpdateBridgeHistory, self.splitter) + d.addCallback(self._cbCallUpdateBridgeHistory, self.hashring) d.addErrback(self._eb_Failure) return d def test_Main_load(self): """Main.load() should run without error.""" - d = deferToThread(Main.load, self.state, self.splitter) + d = deferToThread(Main.load, self.state, self.hashring) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d def test_Main_load_no_state(self): """Main.load() should raise SystemExit without a state object.""" - self.assertRaises(SystemExit, Main.load, None, self.splitter) + self.assertRaises(SystemExit, Main.load, None, self.hashring) def test_Main_load_clear(self): """When called with clear=True, load() should run and clear the hashrings. """ - d = deferToThread(Main.load, self.state, self.splitter, clear=True) + d = deferToThread(Main.load, self.state, self.hashring, clear=True) d.addCallback(self._cbAssertFingerprints) d.addErrback(self._eb_Failure) return d @@ -234,14 +234,14 @@ class MainTests(unittest.TestCase): # The reactor is deferring this to a thread, so the test execution # here isn't actually covering the Storage.updateBridgeHistory() # function: - Main.load(state, self.splitter) + Main.load(state, self.hashring) def test_Main_load_malformed_networkstatus(self): """When called with a networkstatus file with an invalid descriptor, Main.load() should raise a ValueError. """ self._appendToFile(self.state.STATUS_FILE, NETWORKSTATUS_MALFORMED) - self.assertRaises(ValueError, Main.load, self.state, self.splitter) + self.assertRaises(ValueError, Main.load, self.state, self.hashring) def test_Main_reloadFn(self): """Main._reloadFn() should return True.""" @@ -255,79 +255,79 @@ class MainTests(unittest.TestCase): def test_Main_createBridgeRings(self): """Main.createBridgeRings() should add three hashrings to the - splitter. + hashring. """ proxyList = None - (splitter, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) def test_Main_createBridgeRings_with_proxyList(self): """Main.createBridgeRings() should add three hashrings to the - splitter and add the proxyList to the IPBasedDistibutor. + hashring and add the proxyList to the IPBasedDistibutor. """ exitRelays = ['1.1.1.1', '2.2.2.2', '3.3.3.3'] proxyList = Main.proxy.ProxySet() proxyList.addExitRelays(exitRelays) - (splitter, emailDist, httpsDist) = Main.createBridgeRings(self.config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertGreater(len(httpsDist.proxies), 0) self.assertItemsEqual(exitRelays, httpsDist.proxies) def test_Main_createBridgeRings_no_https_dist(self): """When HTTPS_DIST=False, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.HTTPS_DIST = False - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an EmailDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('https', splitter.rings) - self.assertNotIn(httpsDist, splitter.ringsByName.values()) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('https', hashring.rings) + self.assertNotIn(httpsDist, hashring.ringsByName.values()) def test_Main_createBridgeRings_no_email_dist(self): """When EMAIL_DIST=False, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.EMAIL_DIST = False - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, and an UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('email', splitter.rings) - self.assertNotIn(emailDist, splitter.ringsByName.values()) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('email', hashring.rings) + self.assertNotIn(emailDist, hashring.ringsByName.values()) def test_Main_createBridgeRings_no_reserved_share(self): """When RESERVED_SHARE=0, Main.createBridgeRings() should add only - two hashrings to the splitter. + two hashrings to the hashring. """ proxyList = Main.proxy.ProxySet() config = self.config config.RESERVED_SHARE = 0 - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, and an EmailDistributor ring: - self.assertEqual(len(splitter.ringsByName.keys()), 2) - self.assertNotIn('unallocated', splitter.rings) + self.assertEqual(len(hashring.ringsByName.keys()), 2) + self.assertNotIn('unallocated', hashring.rings) def test_Main_createBridgeRings_two_file_buckets(self): """When FILE_BUCKETS has two filenames in it, Main.createBridgeRings() - should add three hashrings to the splitter, then add two + should add three hashrings to the hashring, then add two "pseudo-rings". """ proxyList = Main.proxy.ProxySet() @@ -336,17 +336,17 @@ class MainTests(unittest.TestCase): 'bridges-for-support-desk': 10, 'bridges-for-ooni-tests': 10, } - (splitter, emailDist, httpsDist) = Main.createBridgeRings(config, + (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) # Should have an IPBasedDistributor ring, an EmailDistributor, and an # UnallocatedHolder ring: - self.assertEqual(len(splitter.ringsByName.keys()), 3) + self.assertEqual(len(hashring.ringsByName.keys()), 3) # Should have two pseudoRings: - self.assertEqual(len(splitter.pseudoRings), 2) - self.assertIn('pseudo_bridges-for-support-desk', splitter.pseudoRings) - self.assertIn('pseudo_bridges-for-ooni-tests', splitter.pseudoRings) + self.assertEqual(len(hashring.pseudoRings), 2) + self.assertIn('pseudo_bridges-for-support-desk', hashring.pseudoRings) + self.assertIn('pseudo_bridges-for-ooni-tests', hashring.pseudoRings) def test_Main_run(self): """Main.run() should run and then finally raise SystemExit.""" diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 88b858e..e48e20a 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -167,7 +167,7 @@ class BridgeIntegrationTests(unittest.TestCase): test_getConfigLine_vanilla_withoutFingerprint test_integration_getConfigLine_vanilla_withoutFingerprint test_getConfigLine_vanilla_withFingerprint test_integration_getConfigLine_vanilla_withFingerprint test_getConfigLine_scramblesuit_withFingeprint test_integration_getConfigLine_scramblesuit_withFingerprint - test_splitterBridgeInsertion test_integration_splitterBridgeInsertion + test_splitterBridgeInsertion test_integration_hashringBridgeInsertion ============================================== ======================== .. """ @@ -318,9 +318,9 @@ class BridgeIntegrationTests(unittest.TestCase): % (self.fingerprint, ptArgsList), bridgeLine) - def test_integration_splitterBridgeInsertion(self): + def test_integration_hashringBridgeInsertion(self): key = "Testing-Bridges-To-Rings" - splitter = FilteredBridgeSplitter(key) + hashring = FilteredBridgeSplitter(key) bridge1 = bridges.Bridge('unamed1', '1.2.3.5', 9100, 'a1cc8dfef1fa11af9c40af1054df9daf45250550') @@ -335,18 +335,18 @@ class BridgeIntegrationTests(unittest.TestCase): 'b1cc8dfef1fa11af9c40af1054df9daf45250552') bridge4.setStatus(running = True) - self.failUnlessEqual(len(splitter), 0) - splitter.insert(bridge1) - splitter.insert(bridge2) - splitter.insert(bridge3) + self.failUnlessEqual(len(hashring), 0) + hashring.insert(bridge1) + hashring.insert(bridge2) + hashring.insert(bridge3) # Check that all were inserted - self.failUnlessEqual(len(splitter), 3) - splitter.insert(bridge1) + self.failUnlessEqual(len(hashring), 3) + hashring.insert(bridge1) # Check that the same bridge is not inserted twice - self.failUnlessEqual(len(splitter), 3) - splitter.insert(bridge4) + self.failUnlessEqual(len(hashring), 3) + hashring.insert(bridge4) # Check that identical bridges are not inserted twice - self.failUnlessEqual(len(splitter), 3) + self.failUnlessEqual(len(hashring), 3) class FlagsTests(unittest.TestCase): [View Less]

1 0

[bridgedb/develop] Remove the idea of clusters. Clusters are just subhashrings.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 1a676f343a7a5a854106c9e90c8418d1f7b2c09e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:33:38 2015 +0000 Remove the idea of clusters. Clusters are just subhashrings. --- lib/bridgedb/Dist.py | 60 +++++++++++++++++++++++------------------------ lib/bridgedb/Filters.py | 2 +- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index c9d82c1..58f0a2d 100644 --- a/lib/bridgedb/Dist.py +++ b/… [View More]lib/bridgedb/Dist.py @@ -104,16 +104,15 @@ class IPBasedDistributor(Distributor): distributor. """ - def __init__(self, numberOfClusters, key, proxies=None, answerParameters=None): + def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. - :param integer numberOfClusters: The number of clusters to group IP addresses - into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then - the actual number of clusters is one higher than ``numberOfClusters``, - because the set of known open proxies constitutes its own - category. - DOCDOC What exactly does a cluster *do*? + :param int totalSubrings: The number of subhashrings to group clients + into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, + then the actual number of clusters is one higher than + ``totalSubrings``, because the set of all known open proxies is + given its own subhashring. :param bytes key: The master HMAC key for this distributor. All added bridges are HMACed with this key in order to place them into the hashrings. @@ -129,20 +128,20 @@ class IPBasedDistributor(Distributor): parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ + self.totalSubrings = totalSubrings self.answerParameters = answerParameters - self.numberOfClusters = numberOfClusters if proxies: logging.info("Added known proxies to HTTPS distributor...") self.proxies = proxies - self.numberOfClusters += 1 - self.proxyCluster = self.numberOfClusters + self.totalSubrings += 1 + self.proxySubring = self.totalSubrings else: logging.warn("No known proxies were added to HTTPS distributor!") self.proxies = proxy.ProxySet() - self.proxyCluster = 0 + self.proxySubring = 0 - self.ringCacheSize = self.numberOfClusters * 3 + self.ringCacheSize = self.totalSubrings * 3 key2 = getHMAC(key, "Assign-Bridges-To-Rings") key3 = getHMAC(key, "Order-Areas-In-Rings") @@ -223,13 +222,13 @@ class IPBasedDistributor(Distributor): # If the client wasn't using a proxy, select the client's subring # based upon the client's subnet (modulo the total subrings): if not usingProxy: - mod = self.numberOfClusters + mod = self.totalSubrings # If there is a proxy subring, don't count it for the modulus: - if self.proxyCluster: + if self.proxySubring: mod -= 1 - return int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod + return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 else: - return self.proxyCluster + return self.proxySubring def mapClientToHashringPosition(self, interval, subnet): """Map the client to a position on a (sub)hashring, based upon the @@ -260,14 +259,14 @@ class IPBasedDistributor(Distributor): ``N_IP_CLUSTERS`` configuration option, as well as the number of ``PROXY_LIST_FILES``. - Essentially, :data:`numberOfClusters` is set to the specified + Essentially, :data:`totalSubrings` is set to the specified ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), are stored in :data:`proxies`, and the latter is added as an - additional cluster (such that :data:`numberOfClusters` becomes + additional cluster (such that :data:`totalSubrings` becomes ``N_IP_CLUSTERS + 1``). The number of subhashrings which this :class:`Distributor` has active in its hashring is then - :data:`numberOfClusters`, where the last cluster is reserved for all + :data:`totalSubrings`, where the last cluster is reserved for all :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and @@ -302,14 +301,14 @@ class IPBasedDistributor(Distributor): logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: - for cluster in range(1, self.numberOfClusters): - filters = self._buildHashringFilters([filterFn,], cluster) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) + for subring in range(1, self.totalSubrings + 1): + filters = self._buildHashringFilters([filterFn,], subring) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this # distributor's proxies: - if cluster == self.proxyCluster: + if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), @@ -319,10 +318,9 @@ class IPBasedDistributor(Distributor): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def _buildHashringFilters(self, previousFilters, clientCluster): - g = filterAssignBridgesToRing(self.splitter.hmac, - self.numberOfClusters, clientCluster) - previousFilters.append(g) + def _buildHashringFilters(self, previousFilters, subring): + f = filterAssignBridgesToRing(self.splitter.hmac, self.totalSubrings, subring) + previousFilters.append(f) return frozenset(previousFilters) def getBridges(self, bridgeRequest, interval, N=1): @@ -352,20 +350,20 @@ class IPBasedDistributor(Distributor): # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: - cluster = self.proxyCluster # The tag is a tag applied to a proxy IP address when it is added # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default # is 'exit_relay'. For other proxies loaded from the # PROXY_LIST_FILES config option, the default tag is the full # filename that the IP address originally came from. + usingProxy = True tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) subnet = self.getSubnet(bridgeRequest.client, usingProxy) - cluster = self.mapSubnetToSubring(subnet, usingProxy) + subring = self.mapSubnetToSubring(subnet, usingProxy) position = self.mapClientToHashringPosition(interval, subnet) - filters = self._buildHashringFilters(bridgeRequest.filters, cluster) + filters = self._buildHashringFilters(bridgeRequest.filters, subring) logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) @@ -380,7 +378,7 @@ class IPBasedDistributor(Distributor): # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % cluster) + key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index aade069..94d4325 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -19,7 +19,7 @@ def filterAssignBridgesToRing(hmac, numRings, assignedRing): def _assignBridgesToRing(bridge): digest = hmac(bridge.getID()) pos = long( digest[:8], 16 ) - which = pos % numRings + which = pos % numRings + 1 if which == assignedRing: return True [View Less]

1 0

[bridgedb/develop] Make `areaMapper` built into the IPBasedDistributor class.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 34a97ccc019ca5b9956b8566d81d893293ae5f00 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:12:23 2015 +0000 Make `areaMapper` built into the IPBasedDistributor class. * MOVE Dist.uniformMap() → Dist.IPBasedDistributor.getSubnet(). * REMOVE IPBasedDistributor `areaMapper` parameter. * RENAME IPBasedDistributor.areaOrderHmac → IPBasedDistributor._clientToPositionHMAC. * RENAME IPBasedDistributor.… [View More]areaClusterHmac → IPBasedDistributor._subnetToSubringHMAC. * ADD IPBasedDistributor.mapClientToHashringPosition() to replace and simplify inline use of IPBasedDistributor._clientToPositionHMAC. * ADD IPBasedDistributor.mapSubnetToSubring() to replace and simplify inline use of IPBasedDistributor._subnetToSubringHMAC. --- lib/bridgedb/Dist.py | 184 ++++++++++++++++++++++++++++++-------------------- lib/bridgedb/Main.py | 1 - 2 files changed, 111 insertions(+), 74 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 1a01543..082f002 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -48,39 +48,6 @@ class EmailRequestedKey(Exception): """Raised when an incoming email requested a copy of our GnuPG keys.""" -def uniformMap(ip): - """Map an IP to an arbitrary 'area' string, such that any two IPv4 - addresses in the same ``/16`` subnet, or any two IPv6 addresses in the - same ``/32`` subnet, get the same string. - - >>> from bridgedb import Dist - >>> Dist.uniformMap('1.2.3.4') - '1.2.0.0/16' - >>> Dist.uniformMap('1.2.211.154') - '1.2.0.0/16' - >>> Dist.uniformMap('2001:f::bc1:b13:2808') - '2001:f::/32' - >>> Dist.uniformMap('2a00:c98:2030:a020:2::42') - '2a00:c98::/32' - - :param str ip: A string representing an IPv4 or IPv6 address. - :rtype: str - :returns: The appropriately sized CIDR subnet representation of the **ip**. - """ - # We aren't using bridgedb.parse.addr.isIPAddress(ip, compressed=False) - # here because adding the string "False" into the map would land any and - # all clients whose IP address appeared to be invalid at the same position - # in a hashring. - address = ipaddr.IPAddress(ip) - if address.version == 6: - truncated = ':'.join(address.exploded.split(':')[:2]) - subnet = str(ipaddr.IPv6Network(truncated + "::/32")) - return subnet - else: - truncated = '.'.join(address.exploded.split('.')[:2]) - subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) - return subnet - def getNumBridgesPerAnswer(ring, max_bridges_per_answer=3): if len(ring) < 20: n_bridges_per_answer = 1 @@ -111,9 +78,7 @@ class Distributor(object): hashrings will also carry that name. >>> from bridgedb import Dist - >>> ipDist = Dist.IPBasedDistributor(Dist.uniformMap, - ... 5, - ... 'fake-hmac-key') + >>> ipDist = Dist.IPBasedDistributor(5, 'fake-hmac-key') >>> ipDist.setDistributorName('HTTPS Distributor') >>> ipDist.prepopulateRings() >>> hashrings = ipDist.splitter.filterRings @@ -130,8 +95,6 @@ class IPBasedDistributor(Distributor): """A Distributor that hands out bridges based on the IP address of an incoming request and the current time period. - :ivar areaOrderHmac: An HMAC function used to order areas within rings. - :ivar areaClusterHmac: An HMAC function used to assign areas to rings. :ivar list rings: A list of :class:`bridgedb.Bridges.BridgeHolder` hashrings, one for each area in the ``areaMapper``. Every inserted bridge will go into one of these rings, and every area is associated @@ -145,19 +108,10 @@ class IPBasedDistributor(Distributor): distributor. """ - def __init__(self, areaMapper, numberOfClusters, key, - proxies=None, answerParameters=None): + def __init__(self, numberOfClusters, key, proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. - :type areaMapper: callable - :param areaMapper: A function that maps IP addresses arbitrarily to - strings, such that IP addresses which map to identical strings are - considered to be in the same "area". The default **areaMapper** - is :func:`bridgedb.Dist.uniformMap`, which maps all IPv4 addresses - within the same /16 and all IPv6 addresses within the same /32 to - the same area. Areas are then grouped into the number of rings - specified by the ``N_IP_CLUSTERS`` configuration option. :param integer numberOfClusters: The number of clusters to group IP addresses into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then the actual number of clusters is one higher than ``numberOfClusters``, @@ -180,7 +134,6 @@ class IPBasedDistributor(Distributor): bridges" or "at least one bridge on port 443", etc. """ self.rings = [] - self.areaMapper = areaMapper self.answerParameters = answerParameters self.numberOfClusters = numberOfClusters @@ -200,14 +153,110 @@ class IPBasedDistributor(Distributor): key3 = getHMAC(key, "Order-Areas-In-Rings") key4 = getHMAC(key, "Assign-Areas-To-Rings") - self.areaOrderHmac = getHMACFunc(key3, hex=False) - self.areaClusterHmac = getHMACFunc(key4, hex=True) + self._clientToPositionHMAC = getHMACFunc(key3, hex=False) + self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) self.splitter = FilteredBridgeSplitter(key2, self.ringCacheSize) logging.debug("Added %s to HTTPS distributor." % self.splitter.__class__.__name__) self.setDistributorName('HTTPS') + @classmethod + def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): + """Map all clients whose **ip**s are within the same subnet to the same + arbitrary string. + + .. hint:: For non-proxy IP addresses, any two IPv4 addresses within + the same ``/16`` subnet, or any two IPv6 addresses in the same + ``/32`` subnet, will get the same string. + + Subnets for this distributor are grouped into the number of rings + specified by the ``N_IP_CLUSTERS`` configuration option, such that + Alice (with the address ``1.2.3.4`` and Bob (with the address + ``1.2.178.234``) are placed within the same cluster, but Carol (with + address ``1.3.11.33``) *might* end up in a different cluster. + + >>> from bridgedb.Dist import IPBasedDistributor + >>> IPBasedDistributor.getSubnet('1.2.3.4') + '1.2.0.0/16' + >>> IPBasedDistributor.getSubnet('1.2.211.154') + '1.2.0.0/16' + >>> IPBasedDistributor.getSubnet('2001:f::bc1:b13:2808') + '2001:f::/32' + >>> IPBasedDistributor.getSubnet('2a00:c98:2030:a020:2::42') + '2a00:c98::/32' + + :param str ip: A string representing an IPv4 or IPv6 address. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + :param int proxySubnets: Place Tor/proxy users into this number of + "subnet" groups. This means that no matter how many different Tor + Exits or proxies a client uses, the most they can ever get is + **proxySubnets** different sets of bridge lines (per interval). + This parameter only has any effect when **usingProxy** is ``True``. + :rtype: str + :returns: The appropriately sized CIDR subnet representation of the **ip**. + """ + if not usingProxy: + # We aren't using bridgedb.parse.addr.isIPAddress(ip, + # compressed=False) here because adding the string "False" into + # the map would land any and all clients whose IP address appeared + # to be invalid at the same position in a hashring. + address = ipaddr.IPAddress(ip) + if address.version == 6: + truncated = ':'.join(address.exploded.split(':')[:2]) + subnet = str(ipaddr.IPv6Network(truncated + "::/32")) + else: + truncated = '.'.join(address.exploded.split('.')[:2]) + subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) + else: + group = (int(ipaddr.IPAddress(ip)) % 4) + 1 + subnet = "proxy-group-%d" % group + + logging.debug("Client IP was within area: %s" % subnet) + return subnet + + def mapSubnetToSubring(self, subnet, usingProxy=False): + """Determine the correct subhashring for a client, based upon the + **subnet**. + + :param str subnet: The subnet which contains the client's IP. See + :staticmethod:`getSubnet`. + :param bool usingProxy: Set to ``True`` if the client was using one of + the known :data:`proxies`. + """ + # If the client wasn't using a proxy, select the client's subring + # based upon the client's subnet (modulo the total subrings): + if not usingProxy: + mod = self.numberOfClusters + # If there is a proxy subring, don't count it for the modulus: + if self.proxyCluster: + mod -= 1 + return int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod + else: + return self.proxyCluster + + def mapClientToHashringPosition(self, interval, subnet): + """Map the client to a position on a (sub)hashring, based upon the + **interval** which the client's request occurred within, as well as + the **subnet** of the client's IP address. + + .. note:: For an explanation of how **subnet** is determined, see + :staticmethod:`getSubnet`. + + :param str interval: The interval which this client's request for + bridges took place within. + :param str subnet: A string representing the subnet containing the + client's IP address. + :rtype: int + :returns: The results of keyed HMAC, which should determine the + client's position in a (sub)hashring of bridges (and thus + determine which bridges they receive). + """ + position = "<%s>%s" % (interval, subnet) + mapping = self._clientToPositionHMAC(position) + return mapping + def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. @@ -304,6 +353,8 @@ class IPBasedDistributor(Distributor): logging.warn("Bailing! Splitter has zero bridges!") return [] + usingProxy = False + # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: cluster = self.proxyCluster @@ -315,30 +366,17 @@ class IPBasedDistributor(Distributor): tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) - # Place Tor/proxy users into four groups. This means that no - # matter how many different Tor Exits or proxies a client - # uses, the most they can ever get is four different sets of - # bridge lines (per period). - group = (int(ipaddr.IPAddress(bridgeRequest.client)) % 4) + 1 - area = "known-proxy-group-%d" % group - # If the client wasn't using a proxy, select the client's cluster - # based upon the client's area (i.e. the /16 of the client's IP - # address): - else: - area = self.areaMapper(bridgeRequest.client) - cluster = (int(self.areaClusterHmac(area)[:8], 16) - % (self.numberOfClusters - 1)) - pos = self.areaOrderHmac("<%s>%s" % (interval, area)) + subnet = self.getSubnet(bridgeRequest.client, usingProxy) + cluster = self.mapSubnetToSubring(subnet, usingProxy) + position = self.mapClientToHashringPosition(interval, subnet) filters = self._buildHashringFilters(bridgeRequest.filters, cluster) - logging.debug("Assigned client to cluster %d/%d" % - (cluster, self.numberOfClusters)) - logging.debug("Assigned client hashring position based on: <%s>%s" % - (interval, area)) - logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) - logging.debug("Active bridge filters:\t%s" % - ' '.join([x.func_name for x in filters])) + logging.debug("Client request within time interval: %s" % interval) + logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) + logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Total bridges: %d" % len(self.hashring)) + logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.splitter.filterRings.keys(): @@ -354,7 +392,7 @@ class IPBasedDistributor(Distributor): # Determine the appropriate number of bridges to give to the client: returnNum = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) - answer = ring.getBridges(pos, returnNum) + answer = ring.getBridges(position, returnNum) return answer diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 23ed262..d2fd2da 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -211,7 +211,6 @@ def createBridgeRings(cfg, proxyList, key): if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") ipDistributor = Dist.IPBasedDistributor( - Dist.uniformMap, cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, [View Less]

1 0

[bridgedb/develop] Change all distributors to store their own keys.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit d5628d5bb6244e7f8a0cbd30fa1e52941eb91867 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 09:11:21 2015 +0000 Change all distributors to store their own keys. --- lib/bridgedb/Dist.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 58f0a2d..445d2d8 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -128,6 +128,7 @@ class IPBasedDistributor(Distributor): … [View More] parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ + self.key = key self.totalSubrings = totalSubrings self.answerParameters = answerParameters @@ -303,7 +304,7 @@ class IPBasedDistributor(Distributor): for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this @@ -378,7 +379,7 @@ class IPBasedDistributor(Distributor): # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % subring) + key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, filters, filterBridgesByRules(filters), populate_from=self.splitter.bridges) @@ -420,6 +421,8 @@ class EmailBasedDistributor(Distributor): :param whitelist: A dictionary that maps whitelisted email addresses to GnuPG fingerprints. """ + self.key = key + key1 = getHMAC(key, "Map-Addresses-To-Ring") self.emailHmac = getHMACFunc(key1, hex=False) @@ -505,7 +508,7 @@ class EmailBasedDistributor(Distributor): logging.debug("Cache miss %s" % ruleset) # add new ring - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring") + key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, ruleset, filterBridgesByRules(ruleset), @@ -541,7 +544,7 @@ class EmailBasedDistributor(Distributor): # populate all rings (for dumping assignments and testing) for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: ruleset = frozenset([filterFn]) - key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring") + key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, ruleset, filterBridgesByRules([filterFn]), [View Less]

1 0

[bridgedb/develop] Whitespace fixes in bridgedb.https.server module.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit e62191b6d484074d3d1fa30ef0cd4906822eaf2c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 05:10:47 2015 +0000 Whitespace fixes in bridgedb.https.server module. --- lib/bridgedb/https/server.py | 106 +++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index ab15ef3..6010218 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/… [View More]server.py @@ -6,8 +6,8 @@ # :copyright: (c) 2007-2015, The Tor Project, Inc. # (c) 2013-2015, Isis Lovecruft # :license: see LICENSE for licensing information - -""" + +""" .. py:module:: bridgedb.https.server :synopsis: Servers which interface with clients and distribute bridges over HTTP(S). @@ -16,16 +16,16 @@ bridgedb.https.server ===================== Servers which interface with clients and distribute bridges over HTTP(S). -""" - -import base64 -import gettext -import logging +""" + +import base64 +import gettext +import logging import random -import re -import time -import os - +import re +import time +import os + from functools import partial from ipaddr import IPv4Address @@ -34,14 +34,14 @@ import mako.exceptions from mako.template import Template from mako.lookup import TemplateLookup -from twisted.internet import reactor +from twisted.internet import reactor from twisted.internet.error import CannotListenError from twisted.web import resource -from twisted.web import static +from twisted.web import static from twisted.web.server import NOT_DONE_YET from twisted.web.server import Site -from twisted.web.util import redirectTo - +from twisted.web.util import redirectTo + from bridgedb import captcha from bridgedb import crypto from bridgedb import strings @@ -49,8 +49,8 @@ from bridgedb import translations from bridgedb import txrecaptcha from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.Filters import filterBridgesByTransport +from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress @@ -63,7 +63,7 @@ from bridgedb.util import replaceControlChars TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') - + # Setting `filesystem_checks` to False is recommended for production servers, # due to potential speed increases. This means that the atimes of the Mako # template files aren't rechecked every time the template is requested @@ -214,10 +214,10 @@ class CaptchaProtectedResource(resource.Resource): resource.Resource.__init__(self) self.publicKey = publicKey self.secretKey = secretKey - self.useForwardedHeader = useForwardedHeader + self.useForwardedHeader = useForwardedHeader self.resource = protectedResource - - def getClientIP(self, request): + + def getClientIP(self, request): """Get the client's IP address from the :header:`X-Forwarded-For` header, or from the :api:`request <twisted.web.server.Request>`. @@ -228,7 +228,7 @@ class CaptchaProtectedResource(resource.Resource): :returns: The client's IP address, if it was obtainable. """ return getClientIP(request, self.useForwardedHeader) - + def getCaptchaImage(self, request=None): """Get a CAPTCHA image. @@ -269,7 +269,7 @@ class CaptchaProtectedResource(resource.Resource): """ return False - def render_GET(self, request): + def render_GET(self, request): """Retrieve a ReCaptcha from the API server and serve it to the client. :type request: :api:`twisted.web.http.Request` @@ -298,8 +298,8 @@ class CaptchaProtectedResource(resource.Resource): request.setHeader("Content-Type", "text/html; charset=utf-8") return rendered - - def render_POST(self, request): + + def render_POST(self, request): """Process a client's CAPTCHA solution. If the client's CAPTCHA solution is valid (according to @@ -328,8 +328,8 @@ class CaptchaProtectedResource(resource.Resource): logging.debug("Client failed a CAPTCHA; returning redirect to %s" % request.uri) return redirectTo(request.uri, request) - - + + class GimpCaptchaProtectedResource(CaptchaProtectedResource): """A web resource which uses a local cache of CAPTCHAs, generated with gimp-captcha_, to protect another resource. @@ -626,9 +626,9 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): class BridgesResource(resource.Resource): """This resource displays bridge lines in response to a request.""" - isLeaf = True - - def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, + isLeaf = True + + def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, includeFingerprints=True): """Create a new resource for displaying bridges to a client. @@ -643,16 +643,16 @@ class BridgesResource(resource.Resource): X-Forwarded-For header instead of the source IP address. :param bool includeFingerprints: Do we include the bridge's fingerprint in the response? - """ - gettext.install("bridgedb", unicode=True) + """ + gettext.install("bridgedb", unicode=True) resource.Resource.__init__(self) - self.distributor = distributor - self.schedule = schedule - self.nBridgesToGive = N - self.useForwardedHeader = useForwardedHeader - self.includeFingerprints = includeFingerprints - - def render(self, request): + self.distributor = distributor + self.schedule = schedule + self.nBridgesToGive = N + self.useForwardedHeader = useForwardedHeader + self.includeFingerprints = includeFingerprints + + def render(self, request): """Render a response for a client HTTP request. Presently, this method merely wraps :meth:`getBridgeRequestAnswer` to @@ -674,7 +674,7 @@ class BridgesResource(resource.Resource): response = self.renderAnswer(request) return response - + def getClientIP(self, request): """Get the client's IP address from the :header:`X-Forwarded-For` header, or from the :api:`request <twisted.web.server.Request>`. @@ -687,7 +687,7 @@ class BridgesResource(resource.Resource): """ return getClientIP(request, self.useForwardedHeader) - def getBridgeRequestAnswer(self, request): + def getBridgeRequestAnswer(self, request): """Respond to a client HTTP request for bridges. :type request: :api:`twisted.web.http.Request` @@ -702,7 +702,7 @@ class BridgesResource(resource.Resource): logging.info("Replying to web request from %s. Parameters were %r" % (ip, request.args)) - + if ip: bridgeRequest = HTTPSBridgeRequest() bridgeRequest.client = ip @@ -717,10 +717,10 @@ class BridgesResource(resource.Resource): bridgeRequest, self.includeFingerprints)) for bridge in bridges] return self.renderAnswer(request, bridgeLines) - + def getResponseFormat(self, request): """Determine the requested format for the response. - + :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object containing the HTTP method, full URI, and any URL/POST arguments and headers present. @@ -734,16 +734,16 @@ class BridgesResource(resource.Resource): if format and len(format): format = format[0] # Choose the first arg return format - + def renderAnswer(self, request, bridgeLines=None): """Generate a response for a client which includes **bridgesLines**. - + .. note: The generated response can be plain or HTML. A plain response looks like:: voltron 1.2.3.4:1234 ABCDEF01234567890ABCDEF01234567890ABCDEF voltron 5.5.5.5:5555 0123456789ABCDEF0123456789ABCDEF01234567 - + That is, there is no HTML, what you see is what you get, and what you get is suitable for pasting directly into Tor Launcher (or into a torrc, if you prepend ``"Bridge "`` to each line). The @@ -824,7 +824,7 @@ def addWebServer(config, distributor): :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` :returns: A webserver. - """ + """ captcha = None fwdHeaders = config.HTTP_USE_IP_FROM_FORWARDED_HEADER numBridges = config.HTTPS_N_BRIDGES_PER_ANSWER @@ -879,12 +879,12 @@ def addWebServer(config, distributor): protectedResource=bridges) root.putChild('bridges', protected) logging.info("Protecting resources with %s." % captcha.func.__name__) - else: + else: root.putChild('bridges', bridges) site = Site(root) site.displayTracebacks = False - + if config.HTTP_UNENCRYPTED_PORT: # pragma: no cover ip = config.HTTP_UNENCRYPTED_BIND_IP or "" port = config.HTTP_UNENCRYPTED_PORT or 80 @@ -893,7 +893,7 @@ def addWebServer(config, distributor): except CannotListenError as error: raise SystemExit(error) logging.info("Started HTTP server on %s:%d" % (str(ip), int(port))) - + if config.HTTPS_PORT: # pragma: no cover ip = config.HTTPS_BIND_IP or "" port = config.HTTPS_PORT or 443 @@ -905,5 +905,5 @@ def addWebServer(config, distributor): except CannotListenError as error: raise SystemExit(error) logging.info("Started HTTPS server on %s:%d" % (str(ip), int(port))) - - return site + + return site [View Less]

1 0

[bridgedb/develop] Remove `countryCode` parameter from BridgeRing.getBridges().
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 509fb2094d6cc02ba889bbc0a6883447e3131e7e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 04:14:52 2015 +0000 Remove `countryCode` parameter from BridgeRing.getBridges(). It was never used. And it's unnecessary. --- lib/bridgedb/Bridges.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 988d464..a13a73e 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/… [View More]

1 0

[bridgedb/develop] Rename IPBasedDistributor → HTTPSDistributor.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit ae7acb11669b380de4f7873857ca30167cd493b9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 05:44:32 2015 +0000 Rename IPBasedDistributor → HTTPSDistributor. --- lib/bridgedb/Bridges.py | 2 +- lib/bridgedb/Dist.py | 25 +++++++++++++------------ lib/bridgedb/Main.py | 6 +++--- lib/bridgedb/bridges.py | 2 +- lib/bridgedb/https/request.py | 2 +- lib/bridgedb/https/server.… [View More]py | 4 ++-- lib/bridgedb/test/https_helpers.py | 4 ++-- lib/bridgedb/test/legacy_Tests.py | 20 ++++++++++---------- lib/bridgedb/test/test_Main.py | 10 +++++----- lib/bridgedb/test/test_https_server.py | 6 +++--- 10 files changed, 41 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index a13a73e..b8fdd2e 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -523,7 +523,7 @@ class FilteredBridgeSplitter(BridgeHolder): :ivar bridges: DOCDOC :type distributorName: str :ivar distributorName: The name of this splitter's distributor. See - :meth:`bridgedb.Dist.IPBasedDistributor.setDistributorName`. + :meth:`bridgedb.Dist.HTTPSDistributor.setDistributorName`. """ self.key = key self.filterRings = {} diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index ff9a85d..e66c1e6 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -67,6 +67,7 @@ class Distributor(object): def __init__(self): super(Distributor, self).__init__() + self.name = None def setDistributorName(self, name): """Set a **name** for identifying this distributor. @@ -78,12 +79,10 @@ class Distributor(object): hashrings will also carry that name. >>> from bridgedb import Dist - >>> ipDist = Dist.IPBasedDistributor(5, 'fake-hmac-key') - >>> ipDist.setDistributorName('HTTPS Distributor') - >>> ipDist.prepopulateRings() - >>> hashrings = ipDist.hashring.filterRings - >>> firstSubring = hashrings.items()[0][1][1] - >>> assert firstSubring.name + >>> dist = Dist.HTTPSDistributor(2, 'masterkey') + >>> dist.setDistributorName('Excellent Distributor') + >>> dist.name + 'Excellent Distributor' :param str name: A name for this distributor. """ @@ -91,7 +90,7 @@ class Distributor(object): self.hashring.distributorName = name -class IPBasedDistributor(Distributor): +class HTTPSDistributor(Distributor): """A Distributor that hands out bridges based on the IP address of an incoming request and the current time period. @@ -128,6 +127,8 @@ class IPBasedDistributor(Distributor): parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ + super(HTTPSDistributor, self).__init__() + self.key = key self.totalSubrings = totalSubrings self.answerParameters = answerParameters @@ -171,14 +172,14 @@ class IPBasedDistributor(Distributor): ``1.2.178.234``) are placed within the same cluster, but Carol (with address ``1.3.11.33``) *might* end up in a different cluster. - >>> from bridgedb.Dist import IPBasedDistributor - >>> IPBasedDistributor.getSubnet('1.2.3.4') + >>> from bridgedb.Dist import HTTPSDistributor + >>> HTTPSDistributor.getSubnet('1.2.3.4') '1.2.0.0/16' - >>> IPBasedDistributor.getSubnet('1.2.211.154') + >>> HTTPSDistributor.getSubnet('1.2.211.154') '1.2.0.0/16' - >>> IPBasedDistributor.getSubnet('2001:f::bc1:b13:2808') + >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') '2001:f::/32' - >>> IPBasedDistributor.getSubnet('2a00:c98:2030:a020:2::42') + >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') '2a00:c98::/32' :param str ip: A string representing an IPv4 or IPv6 address. diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 1262122..0bb319d 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -194,7 +194,7 @@ def createBridgeRings(cfg, proxyList, key): known open proxies. :param bytes key: Hashring master key :rtype: tuple - :returns: A BridgeSplitter hashring, an IPBasedDistributor or None, + :returns: A BridgeSplitter hashring, an HTTPSDistributor or None, and an EmailBasedDistributor or None. """ # Create a BridgeSplitter to assign the bridges to the different @@ -210,7 +210,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") - ipDistributor = Dist.IPBasedDistributor( + ipDistributor = Dist.HTTPSDistributor( cfg.N_IP_CLUSTERS, crypto.getHMAC(key, "HTTPS-IP-Dist-Key"), proxyList, @@ -329,7 +329,7 @@ def run(options, reactor=reactor): :type proxyList: :class:`~bridgedb.proxy.ProxySet` :ivar proxyList: The container for the IP addresses of any currently known open proxies. - :ivar ipDistributor: A :class:`Dist.IPBasedDistributor`. + :ivar ipDistributor: A :class:`Dist.HTTPSDistributor`. :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 91b62fa..541e735 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -727,7 +727,7 @@ class BridgeBackwardsCompatibility(BridgeBase): :class:`ipaddr.IPv6Address`. :param addressClass: Type of address to choose. :param str request: A string (somewhat) unique to this request, - e.g. email-address or ``IPBasedDistributor.getSubnet(ip)``. In + e.g. email-address or ``HTTPSDistributor.getSubnet(ip)``. In this case, this is not a :class:`~bridgerequest.BridgeRequestBase` (as might be expected) but the equivalent of :data:`bridgerequest.BridgeRequestBase.client`. diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py index a8f6920..37b3ed6 100644 --- a/lib/bridgedb/https/request.py +++ b/lib/bridgedb/https/request.py @@ -55,7 +55,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self, addClientCountryCode=True): """Process a new bridge request received through the - :class:`~bridgedb.Dist.IPBasedDistributor`. + :class:`~bridgedb.Dist.HTTPSDistributor`. :param bool addClientCountryCode: If ``True``, then calling :meth:`withoutBlockInCountry` will attempt to add the client's own diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index 6010218..e8cdd62 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -632,7 +632,7 @@ class BridgesResource(resource.Resource): includeFingerprints=True): """Create a new resource for displaying bridges to a client. - :type distributor: :class:`IPBasedDistributor` + :type distributor: :class:`HTTPSDistributor` :param distributor: The mechanism to retrieve bridges for this distributor. :type schedule: :class:`~bridgedb.schedule.ScheduledInterval` @@ -819,7 +819,7 @@ def addWebServer(config, distributor): GIMP_CAPTCHA_DIR GIMP_CAPTCHA_HMAC_KEYFILE GIMP_CAPTCHA_RSA_KEYFILE - :type distributor: :class:`bridgedb.Dist.IPBasedDistributor` + :type distributor: :class:`bridgedb.Dist.HTTPSDistributor` :param distributor: A bridge distributor. :raises SystemExit: if the servers cannot be started. :rtype: :api:`twisted.web.server.Site` diff --git a/lib/bridgedb/test/https_helpers.py b/lib/bridgedb/test/https_helpers.py index f9232bc..00ccbd6 100644 --- a/lib/bridgedb/test/https_helpers.py +++ b/lib/bridgedb/test/https_helpers.py @@ -98,8 +98,8 @@ def _createConfig(configFile=TEST_CONFIG_FILE): return config -class DummyIPBasedDistributor(object): - """A mocked :class:`bridgedb.Dist.IPBasedDistributor` which is used to test +class DummyHTTPSDistributor(object): + """A mocked :class:`bridgedb.Dist.HTTPSDistributor` which is used to test :class:`bridgedb.https.server.BridgesResource`. """ _bridge_class = util.DummyBridge diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index dc91c9b..cc1ba6d 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -187,7 +187,7 @@ class IPBridgeDistTests(unittest.TestCase): def dumbAreaMapper(self, ip): return ip def testBasicDist(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(256): d.insert(fakeBridge()) n = d.getBridges("1.2.3.4", "x", 2) @@ -195,7 +195,7 @@ class IPBridgeDistTests(unittest.TestCase): self.assertEquals(n, n2) def testDistWithProxies(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo", + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo", [RhymesWith255ProxySet()]) for _ in xrange(256): d.insert(fakeBridge()) @@ -219,8 +219,8 @@ class IPBridgeDistTests(unittest.TestCase): #XXX: #6175 breaks this test! #def testDistWithPortRestrictions(self): # param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)]) - # d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Baz", - # answerParameters=param) + # d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Baz", + # answerParameters=param) # for _ in xrange(32): # d.insert(fakeBridge(443)) # for _ in range(256): @@ -239,7 +239,7 @@ class IPBridgeDistTests(unittest.TestCase): # self.assertTrue(count >= 1) def testDistWithFilterIP6(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) @@ -255,7 +255,7 @@ class IPBridgeDistTests(unittest.TestCase): assert filterBridgesByIP6(random.choice(bridges)) def testDistWithFilterIP4(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) @@ -271,7 +271,7 @@ class IPBridgeDistTests(unittest.TestCase): assert filterBridgesByIP4(random.choice(bridges)) def testDistWithFilterBoth(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) @@ -295,7 +295,7 @@ class IPBridgeDistTests(unittest.TestCase): def testDistWithFilterAll(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) @@ -306,7 +306,7 @@ class IPBridgeDistTests(unittest.TestCase): assert len(b) == 0 def testDistWithFilterBlockedCountries(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True)) d.insert(fakeBridge(or_addresses=True)) @@ -332,7 +332,7 @@ class IPBridgeDistTests(unittest.TestCase): assert len(b) > 0 def testDistWithFilterBlockedCountriesAdvanced(self): - d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + d = bridgedb.Dist.HTTPSDistributor(self.dumbAreaMapper, 3, "Foo") for _ in xrange(250): d.insert(fakeBridge6(or_addresses=True, transports=True)) d.insert(fakeBridge(or_addresses=True, transports=True)) diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index ec3fdd2..26012f7 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -261,7 +261,7 @@ class MainTests(unittest.TestCase): (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) - # Should have an IPBasedDistributor ring, an EmailDistributor ring, + # Should have an HTTPSDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 3) @@ -275,7 +275,7 @@ class MainTests(unittest.TestCase): (hashring, emailDist, httpsDist) = Main.createBridgeRings(self.config, proxyList, self.key) - # Should have an IPBasedDistributor ring, an EmailDistributor ring, + # Should have an HTTPSDistributor ring, an EmailDistributor ring, # and an UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 3) self.assertGreater(len(httpsDist.proxies), 0) @@ -306,7 +306,7 @@ class MainTests(unittest.TestCase): (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) - # Should have an IPBasedDistributor ring, and an UnallocatedHolder ring: + # Should have an HTTPSDistributor ring, and an UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 2) self.assertNotIn('email', hashring.rings) self.assertNotIn(emailDist, hashring.ringsByName.values()) @@ -321,7 +321,7 @@ class MainTests(unittest.TestCase): (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) - # Should have an IPBasedDistributor ring, and an EmailDistributor ring: + # Should have an HTTPSDistributor ring, and an EmailDistributor ring: self.assertEqual(len(hashring.ringsByName.keys()), 2) self.assertNotIn('unallocated', hashring.rings) @@ -339,7 +339,7 @@ class MainTests(unittest.TestCase): (hashring, emailDist, httpsDist) = Main.createBridgeRings(config, proxyList, self.key) - # Should have an IPBasedDistributor ring, an EmailDistributor, and an + # Should have an HTTPSDistributor ring, an EmailDistributor, and an # UnallocatedHolder ring: self.assertEqual(len(hashring.ringsByName.keys()), 3) diff --git a/lib/bridgedb/test/test_https_server.py b/lib/bridgedb/test/test_https_server.py index 68a975e..e4f56f0 100644 --- a/lib/bridgedb/test/test_https_server.py +++ b/lib/bridgedb/test/test_https_server.py @@ -31,7 +31,7 @@ from bridgedb.https import server from bridgedb.schedule import ScheduledInterval from bridgedb.test.https_helpers import _createConfig from bridgedb.test.https_helpers import DummyRequest -from bridgedb.test.https_helpers import DummyIPBasedDistributor +from bridgedb.test.https_helpers import DummyHTTPSDistributor from bridgedb.test.util import DummyBridge from bridgedb.test.util import DummyMaliciousBridge @@ -521,7 +521,7 @@ class BridgesResourceTests(unittest.TestCase): self.pagename = b'bridges.html' self.root = Resource() - self.dist = DummyIPBasedDistributor() + self.dist = DummyHTTPSDistributor() self.sched = ScheduledInterval(1, 'hour') self.nBridgesPerRequest = 2 @@ -796,7 +796,7 @@ class HTTPSServerServiceTests(unittest.TestCase): def setUp(self): """Create a server.MailServerContext and EmailBasedDistributor.""" self.config = _createConfig() - self.distributor = DummyIPBasedDistributor() + self.distributor = DummyHTTPSDistributor() def tearDown(self): """Cleanup method after each ``test_*`` method runs; removes timed out [View Less]

1 0

[bridgedb/develop] Fix Filters.filterBridgesByNotBlockedIn to work with new Bridge class.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 98cf68d778480aa005e5914cac47164f169b87b9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 20:37:24 2015 +0000 Fix Filters.filterBridgesByNotBlockedIn to work with new Bridge class. --- lib/bridgedb/Filters.py | 37 +++++++++++++++++-------------------- lib/bridgedb/bridgerequest.py | 4 +--- lib/bridgedb/test/legacy_Tests.py | 2 +- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/… [View More]Filters.py index 94d4325..41df297 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -92,28 +92,25 @@ def filterBridgesByTransport(methodname, addressClass=None): funcs[ruleset] = _filterByTransport return _filterByTransport -def filterBridgesByNotBlockedIn(countryCode, addressClass=None, methodname=None): - """ if at least one address:port of the selected addressClass and - (optional) transport type is not blocked in countryCode, return True +def filterBridgesByNotBlockedIn(countryCode): + """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't + known to be blocked in **countryCode**. + + :param str countryCode: A two-letter country code. + :rtype: bool + :returns: ``True`` if at least one address of the bridge isn't blocked. + ``False`` otherwise. """ - # default to IPv4 if not specified - if addressClass is None: addressClass = IPv4Address - assert (addressClass) in (IPv4Address, IPv6Address) - ruleset = frozenset([countryCode, addressClass, methodname]) + countryCode = countryCode.lower() + ruleset = frozenset([countryCode]) try: return funcs[ruleset] except KeyError: - def f(bridge): - if bridge.isBlocked(countryCode, addressClass, methodname): - if addressClass is IPv4Address: ac = "IPv4" - else: ac = "IPv6" - logmsg = "Removing %s from set of results for country" - logmsg += " '%s' with address class %s and transport %s" - logging.debug(logmsg % ( bridge.fingerprint, countryCode, ac, - methodname)) + def _filterByNotBlockedIn(bridge): + if bridge.isBlockedIn(countryCode): return False - return True # not blocked - f.__name__ = "filterBridgesNotBlockedIn(%s,%s,%s)" % \ - (countryCode,methodname,addressClass) - funcs[ruleset] = f - return f + return True + _filterByNotBlockedIn.__name__ = "filterBridgesByNotBlockedIn(%s)" % countryCode + setattr(_filterByNotBlockedIn, "description", "unblocked=%s" % countryCode) + funcs[ruleset] = _filterByNotBlockedIn + return _filterByNotBlockedIn diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index f4eaaa4..532558d 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -182,6 +182,4 @@ class BridgeRequestBase(object): self.addFilter(Filters.filterBridgesByTransport(transport, self.addressClass)) for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower(), - self.addressClass, - transport)) + self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index cc1ba6d..5621515 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -357,7 +357,7 @@ class IPBridgeDistTests(unittest.TestCase): for i in xrange(5): b = d.getBridges(randomIPString(), "x", 1, bridgeFilterRules=[ - filterBridgesByNotBlockedIn("cn", methodname='obfs2'), + filterBridgesByNotBlockedIn("cn"), filterBridgesByTransport('obfs2'), ]) try: assert len(b) > 0 [View Less]

1 0

[bridgedb/develop] BridgeRequest.notBlockedIn country codes should be lowercased.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit c712736cc04b0423a18fb9582e8d11c0bf663057 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 07:02:44 2015 +0000 BridgeRequest.notBlockedIn country codes should be lowercased. --- lib/bridgedb/bridgerequest.py | 4 ++-- lib/bridgedb/https/request.py | 4 ++-- lib/bridgedb/test/test_bridgerequest.py | 2 +- lib/bridgedb/test/test_https_request.py | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/… [View More]bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index f7aa031..f4eaaa4 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -150,7 +150,7 @@ class BridgeRequestBase(object): self.addressClass = ipaddr.IPv6Address def withoutBlockInCountry(self, country): - self.notBlockedIn.append(country) + self.notBlockedIn.append(country.lower()) def withPluggableTransportType(self, pt): self.transports.append(pt) @@ -182,6 +182,6 @@ class BridgeRequestBase(object): self.addFilter(Filters.filterBridgesByTransport(transport, self.addressClass)) for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country, + self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower(), self.addressClass, transport)) diff --git a/lib/bridgedb/https/request.py b/lib/bridgedb/https/request.py index 37b3ed6..0ae52c7 100644 --- a/lib/bridgedb/https/request.py +++ b/lib/bridgedb/https/request.py @@ -106,7 +106,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): pass else: if country: - self.notBlockedIn.append(country) + self.notBlockedIn.append(country.lower()) logging.info("HTTPS request for bridges not blocked in: %r" % country) @@ -116,7 +116,7 @@ class HTTPSBridgeRequest(bridgerequest.BridgeRequestBase): if addr.isIPAddress(self.client): country = geo.getCountryCode(ipaddr.IPAddress(self.client)) if country: - self.notBlockedIn.append(country) + self.notBlockedIn.append(country.lower()) logging.info( ("HTTPS client's bridges also shouldn't be blocked " "in their GeoIP country code: %s") % country) diff --git a/lib/bridgedb/test/test_bridgerequest.py b/lib/bridgedb/test/test_bridgerequest.py index 3392a4e..ccbf406 100644 --- a/lib/bridgedb/test/test_bridgerequest.py +++ b/lib/bridgedb/test/test_bridgerequest.py @@ -34,7 +34,7 @@ class BridgeRequestBaseTests(unittest.TestCase): to the ``notBlockedIn`` attribute. """ self.request.withoutBlockInCountry('US') - self.assertIn('US', self.request.notBlockedIn) + self.assertIn('us', self.request.notBlockedIn) def test_BridgeRequestBase_withPluggableTransportType(self): """BridgeRequestBase.withPluggableTransportType() should add the diff --git a/lib/bridgedb/test/test_https_request.py b/lib/bridgedb/test/test_https_request.py index 286e6eb..c39662f 100644 --- a/lib/bridgedb/test/test_https_request.py +++ b/lib/bridgedb/test/test_https_request.py @@ -43,28 +43,28 @@ class HTTPSBridgeRequestTests(unittest.TestCase): """HTTPSBridgeRequest.withoutBlockInCountry() should add the country CC to the ``notBlockedIn`` attribute. """ - httprequest = MockRequest({'unblocked': ['IR']}) + httprequest = MockRequest({'unblocked': ['ir']}) self.request.withoutBlockInCountry(httprequest) - self.assertIn('IR', self.request.notBlockedIn) + self.assertIn('ir', self.request.notBlockedIn) def test_HTTPSBridgeRequest_withoutBlockInCountry_US(self): """HTTPSBridgeRequest.withoutBlockInCountry() should add the country CC to the ``notBlockedIn`` attribute (and not any other countries). """ - httprequest = MockRequest({'unblocked': ['US']}) + httprequest = MockRequest({'unblocked': ['us']}) self.request.withoutBlockInCountry(httprequest) - self.assertNotIn('IR', self.request.notBlockedIn) + self.assertNotIn('ir', self.request.notBlockedIn) def test_HTTPSBridgeRequest_withoutBlockInCountry_no_addClientCountryCode(self): """HTTPSBridgeRequest.withoutBlockInCountry(), when addClientCountryCode=False, shouldn't add the client's country code to the ``notBlockedIn`` attribute. """ - httprequest = MockRequest({'unblocked': ['NL']}) + httprequest = MockRequest({'unblocked': ['nl']}) self.request = request.HTTPSBridgeRequest(addClientCountryCode=False) self.request.client = '5.5.5.5' self.request.withoutBlockInCountry(httprequest) - self.assertItemsEqual(['NL'], self.request.notBlockedIn) + self.assertItemsEqual(['nl'], self.request.notBlockedIn) def test_HTTPSBridgeRequest_withoutBlockInCountry_bad_params(self): """HTTPSBridgeRequest.withoutBlockInCountry() should stop processing if [View Less]

1 0