June 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Rewrite testDistWithFilterIP6() in legacy_Tests.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit c1637a017d7e34a79b71829f0918184b63055c6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 23:35:33 2015 +0000 Rewrite testDistWithFilterIP6() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 16 ---------------- lib/bridgedb/test/test_Dist.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 9a0c74f..c495412 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -236,22 +236,6 @@ class IPBridgeDistTests(unittest.TestCase): # self.assertEquals(len(fps), 5) # self.assertTrue(count >= 1) - def testDistWithFilterIP6(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(500): - bridges = d.getBridges(randomIPv4String(), - "faketimestamp", - bridgeFilterRules=[filterBridgesByIP6]) - bridge = random.choice(bridges) - bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv6Address) - address, portlist = networkstatus.parseALine(bridge_line) - assert type(ipaddr.IPAddress(address)) is ipaddr.IPv6Address - assert filterBridgesByIP6(random.choice(bridges)) - def testDistWithFilterBlockedCountriesAdvanced(self): d = bridgedb.Dist.HTTPSDistributor(3, "Foo") for _ in xrange(250): diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 82ef823..bc437ba 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -289,6 +289,28 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + def test_HTTPSDistributor_getBridges_ipv6(self): + """A request for IPv6 bridges should return IPv6 bridges.""" + dist = Dist.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + def test_HTTPSDistributor_getBridges_ipv4(self): """A request for IPv4 bridges should return IPv4 bridges.""" dist = Dist.HTTPSDistributor(1, self.key)

1 0

[bridgedb/develop] Rewrite commented out testDistWithPortRestrictions() in legacy_Tests.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 003165e7875412944b91785431ee1b749fda2a0d Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 20 03:34:40 2015 +0000 Rewrite commented out testDistWithPortRestrictions() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 22 ---------------------- lib/bridgedb/test/test_Dist.py | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 85ba722..2f1da67 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -185,28 +185,6 @@ class EmailBridgeDistTests(unittest.TestCase): class IPBridgeDistTests(unittest.TestCase): - #XXX: #6175 breaks this test! - #def testDistWithPortRestrictions(self): - # param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)]) - # d = bridgedb.Dist.HTTPSDistributor(3, "Baz", - # answerParameters=param) - # for _ in xrange(32): - # d.insert(fakeBridge(443)) - # for _ in range(256): - # d.insert(fakeBridge()) - # for _ in xrange(32): - # i = randomIP() - # n = d.getBridges(i, "x") - # count = 0 - # fps = {} - # for b in n: - # fps[b.getID()] = 1 - # if b.orport == 443: - # count += 1 - # self.assertEquals(len(fps), len(n)) - # self.assertEquals(len(fps), 5) - # self.assertTrue(count >= 1) - def testDistWithFilterBlockedCountriesAdvanced(self): d = bridgedb.Dist.HTTPSDistributor(3, "Foo") for _ in xrange(250): diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 18b3e79..e8a17e4 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -23,6 +23,7 @@ from bridgedb import Dist from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import BridgeRingParameters from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 @@ -282,6 +283,30 @@ class HTTPSDistributorTests(unittest.TestCase): for i in range(4): self.assertItemsEqual(responses[i], responses[i+1]) + def test_HTTPSDistributor_getBridges_with_BridgeRingParameters(self): + param = BridgeRingParameters(needPorts=[(443, 1)]) + dist = Dist.HTTPSDistributor(3, self.key, answerParameters=param) + + bridges = self.bridges[:32] + for b in self.bridges: + b.orPort = 443 + + [dist.insert(bridge) for bridge in bridges] + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for _ in xrange(32): + bridgeRequest = self.randomClientRequest() + answer = dist.getBridges(bridgeRequest, 1) + count = 0 + fingerprints = {} + for bridge in answer: + fingerprints[bridge.identity] = 1 + if bridge.orPort == 443: + count += 1 + self.assertEquals(len(fingerprints), len(answer)) + self.assertGreater(len(fingerprints), 0) + self.assertTrue(count >= 1) + def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): """Asking for bridge addresses which are simultaneously IPv4 and IPv6 (in that order) should return IPv4 bridges.

1 0

[bridgedb/develop] Rewrite testDistWithProxies() in legacy_Tests.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit dcafe58630f9048b7829184675a9a93deda2f0a3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 19 05:43:39 2015 +0000 Rewrite testDistWithProxies() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 21 --------------------- lib/bridgedb/test/test_Dist.py | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index fd6a739..85ba722 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -185,27 +185,6 @@ class EmailBridgeDistTests(unittest.TestCase): class IPBridgeDistTests(unittest.TestCase): - def testDistWithProxies(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo", [RhymesWith255ProxySet()]) - for _ in xrange(256): - d.insert(fakeBridge()) - - for _ in xrange(256): - # Make sure that the ProxySets do not overlap - f = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(4)]) - g = lambda: ".".join([str(random.randrange(1,255)) for _ in xrange(3)] + ['255']) - n = d.getBridges(g(), "x") - n2 = d.getBridges(f(), "x") - - assert(len(n) > 0) - assert(len(n2) > 0) - - for b in n: - assert (b not in n2) - - for b in n2: - assert (b not in n) - #XXX: #6175 breaks this test! #def testDistWithPortRestrictions(self): # param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)]) diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 23a6492..18b3e79 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -243,6 +243,30 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertNotIn(b.fingerprint, blockedIR) self.assertGreater(len(bridges), 0) + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): + """An HTTPSDistributor should give separate bridges to proxy users.""" + proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)]) + dist = Dist.HTTPSDistributor(3, self.key, proxies) + [dist.insert(bridge) for bridge in self.bridges] + + for _ in range(10): + bridgeRequest1 = self.randomClientRequest() + bridgeRequest1.client = '.'.join(['1.1.1', str(random.randrange(1, 255))]) + + bridgeRequest2 = self.randomClientRequest() + bridgeRequest2.client = '.'.join(['9.9.9', str(random.randrange(1, 255))]) + + n1 = dist.getBridges(bridgeRequest1, 1) + n2 = dist.getBridges(bridgeRequest2, 1) + + self.assertGreater(len(n1), 0) + self.assertGreater(len(n2), 0) + + for b in n1: + self.assertNotIn(b, n2) + for b in n2: + self.assertNotIn(b, n1) + def test_HTTPSDistributor_getBridges_same_bridges_to_same_client(self): """The same client asking for bridges from the HTTPSDistributor multiple times in a row should get the same bridges in response each

1 0

[bridgedb/develop] Rewrite testDistWithFilterBlockedCountriesAdvanced() in legacy_Tests.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 2de77ba16992b8a967bfe68cebc2c17c2b3998fc Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 02:33:08 2015 +0000 Rewrite testDistWithFilterBlockedCountriesAdvanced() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 41 +------------------------------------ lib/bridgedb/test/test_Dist.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 2f1da67..95dc34e 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -183,44 +183,6 @@ class EmailBridgeDistTests(unittest.TestCase): {'example.com':'example.com'}, {'example.com':[]}) -class IPBridgeDistTests(unittest.TestCase): - - def testDistWithFilterBlockedCountriesAdvanced(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True, transports=True)) - d.insert(fakeBridge(or_addresses=True, transports=True)) - - for b in d.hashring.bridges: - # china blocks some transports - for pt in b.transports: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (pt.address, pt.port) - b.blockingCountries[key] = set(['cn']) - for address, portlist in b.or_addresses.items(): - # china blocks some transports - for port in portlist: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (address, port) - b.blockingCountries[key] = set(['cn']) - key = "%s:%s" % (b.ip, b.orport) - b.blockingCountries[key] = set(['cn']) - - # we probably will get at least one bridge back! - # it's pretty unlikely to lose a coin flip 250 times in a row - for i in xrange(5): - b = d.getBridges(randomIPString(), "x", - bridgeFilterRules=[ - filterBridgesByNotBlockedIn("cn"), - filterBridgesByTransport('obfs2'), - ]) - try: assert len(b) > 0 - except AssertionError: - print("epic fail") - b = d.getBridges(randomIPString(), "x", bridgeFilterRules=[ - filterBridgesByNotBlockedIn("us")]) - assert len(b) > 0 - class SQLStorageTests(unittest.TestCase): def setUp(self): @@ -450,8 +412,7 @@ def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, - BridgeStabilityTests]: + for klass in [SQLStorageTests, EmailBridgeDistTests, BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) return suite diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index e8a17e4..85c1b87 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -244,6 +244,45 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertNotIn(b.fingerprint, blockedIR) self.assertGreater(len(bridges), 0) + def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): + dist = Dist.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + # Pretend that China blocks all vanilla bridges: + bridge.setBlockedIn('cn', methodname='vanilla') + # Pretend that China blocks all obfs2: + bridge.setBlockedIn('cn', methodname='obfs2') + # Pretend that China blocks some obfs3: + if self.coinFlip(): + bridge.setBlockedIn('cn', methodname='obfs3') + + [dist.insert(bridge) for bridge in bridges] + + for i in xrange(5): + bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest1.transports.append('obfs2') + bridgeRequest1.generateFilters() + # We shouldn't get any obfs2 bridges, since they're all blocked in + # China: + bridges = dist.getBridges(bridgeRequest1, "faketimestamp") + self.assertEqual(len(bridges), 0) + + bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest2.transports.append('obfs3') + bridgeRequest2.generateFilters() + # We probably will get at least one bridge back! It's pretty + # unlikely to lose a coin flip 500 times in a row. + bridges = dist.getBridges(bridgeRequest2, "faketimestamp") + self.assertGreater(len(bridges), 0) + + bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') + bridgeRequest3.transports.append('obfs3') + bridgeRequest3.generateFilters() + # We should get bridges, since obfs3 isn't blocked in netherlands: + bridges = dist.getBridges(bridgeRequest3, "faketimestamp") + self.assertGreater(len(bridges), 0) + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): """An HTTPSDistributor should give separate bridges to proxy users.""" proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)])

1 0

[bridgedb/develop] Refactor bridgedb.Filters and move it to bridgedb.filters.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 9c09ef7bdb335fd58d48d8f8ff6d79d41fbd088c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 05:09:32 2015 +0000 Refactor bridgedb.Filters and move it to bridgedb.filters. --- doc/sphinx/source/bridgedb.Filters.rst | 8 - doc/sphinx/source/bridgedb.filters.rst | 8 + doc/sphinx/source/bridgedb.rst | 2 +- doc/sphinx/source/conf.py | 2 +- lib/bridgedb/Bridges.py | 12 +- lib/bridgedb/Dist.py | 25 ++- lib/bridgedb/Filters.py | 175 ----------------- lib/bridgedb/bridgerequest.py | 39 ++-- lib/bridgedb/distribute.py | 2 +- lib/bridgedb/filters.py | 238 +++++++++++++++++++++++ lib/bridgedb/https/server.py | 4 - lib/bridgedb/parse/addr.py | 6 +- lib/bridgedb/persistent.py | 12 +- lib/bridgedb/test/legacy_Tests.py | 5 - lib/bridgedb/test/test_Dist.py | 17 +- lib/bridgedb/test/test_filters.py | 333 ++++++++++++++++++++++++++++++++ 16 files changed, 638 insertions(+), 250 deletions(-) diff --git a/doc/sphinx/source/bridgedb.Filters.rst b/doc/sphinx/source/bridgedb.Filters.rst deleted file mode 100644 index caf8dfc..0000000 --- a/doc/sphinx/source/bridgedb.Filters.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.Filters ----------------- - -.. automodule:: bridgedb.Filters - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.filters.rst b/doc/sphinx/source/bridgedb.filters.rst new file mode 100644 index 0000000..9c14ce5 --- /dev/null +++ b/doc/sphinx/source/bridgedb.filters.rst @@ -0,0 +1,8 @@ +bridgedb.filters +---------------- + +.. automodule:: bridgedb.filters + :members: + :undoc-members: + :private-members: + :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 31e3a90..7b1fef7 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -15,7 +15,7 @@ BridgeDB Package and Module Documentation bridgedb.crypto bridgedb.Dist bridgedb.email - bridgedb.Filters + bridgedb.filters bridgedb.geo bridgedb.https bridgedb.interfaces diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index b655c27..9b43d0f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -40,7 +40,7 @@ import bridgedb.email.dkim import bridgedb.email.request import bridgedb.email.server import bridgedb.email.templates -import bridgedb.Filters +import bridgedb.filters import bridgedb.geo import bridgedb.https import bridgedb.https.request diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 507013d..2b56884 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -569,7 +569,7 @@ class FilteredBridgeSplitter(object): filterNames = [] for filterName in [x.func_name for x in list(ringname)]: - # Using `filterAssignBridgesToRing.func_name` gives us a messy + # Using `assignBridgesToSubring.func_name` gives us a messy # string which includes all parameters and memory addresses. Get # rid of this by partitioning at the first `(`: realFilterName = filterName.partition('(')[0] @@ -599,8 +599,8 @@ class FilteredBridgeSplitter(object): # hashring '%s'!" % (inserted, ringname))`, this log message appears: # # Jan 04 23:18:37 [INFO] Inserted 12 bridges into hashring - # frozenset([<function filterBridgesByIP4 at 0x2d67cf8>, <function - # filterAssignBridgesToRing(<function hmac_fn at 0x3778398>, 4, 0) at + # frozenset([<function byIPv4 at 0x2d67cf8>, <function + # assignBridgesToSubring(<function hmac_fn at 0x3778398>, 4, 0) at # 0x37de578>])! # # I suppose since it contains memory addresses, it *is* technically @@ -615,10 +615,10 @@ class FilteredBridgeSplitter(object): subringName = [self.distributorName] subringNumber = None for filterName in filterNames: - if filterName.startswith('filterAssignBridgesToRing'): - subringNumber = filterName.lstrip('filterAssignBridgesToRing') + if filterName.startswith('assignBridgesToSubring'): + subringNumber = filterName.lstrip('assignBridgesToSubring') else: - subringName.append(filterName.lstrip('filterBridgesBy')) + subringName.append(filterName.lstrip('by')) if subring.name and 'Proxy' in subring.name: subringName.append('Proxy') elif subringNumber: diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index fe6b8da..9b9e35c 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -25,10 +25,10 @@ from bridgedb.Bridges import FilteredBridgeSplitter from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.distribute import Distributor -from bridgedb.Filters import filterAssignBridgesToRing -from bridgedb.Filters import filterBridgesByRules -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byFilters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import bySubring from bridgedb.parse import addr @@ -259,7 +259,7 @@ class HTTPSDistributor(Distributor): """ logging.info("Prepopulating %s distributor hashrings..." % self.name) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) @@ -269,8 +269,7 @@ class HTTPSDistributor(Distributor): # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) - self.hashring.addRing(ring, filters, - filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) def insert(self, bridge): @@ -278,7 +277,7 @@ class HTTPSDistributor(Distributor): self.hashring.insert(bridge) def _buildHashringFilters(self, previousFilters, subring): - f = filterAssignBridgesToRing(self.hashring.hmac, self.totalSubrings, subring) + f = bySubring(self.hashring.hmac, subring, self.totalSubrings) previousFilters.append(f) return frozenset(previousFilters) @@ -337,7 +336,7 @@ class HTTPSDistributor(Distributor): logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, filters, filterBridgesByRules(filters), + self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: @@ -457,8 +456,7 @@ class EmailBasedDistributor(Distributor): # add new ring key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules(ruleset), + self.hashring.addRing(ring, ruleset, byFilters(ruleset), populate_from=self.hashring.bridges) returnNum = self.bridgesPerResponse(ring) @@ -482,10 +480,9 @@ class EmailBasedDistributor(Distributor): def prepopulateRings(self): # populate all rings (for dumping assignments and testing) - for filterFn in [filterBridgesByIP4, filterBridgesByIP6]: + for filterFn in [byIPv4, byIPv6]: ruleset = frozenset([filterFn]) key1 = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, - filterBridgesByRules([filterFn]), + self.hashring.addRing(ring, ruleset, byFilters([filterFn]), populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py deleted file mode 100644 index d0d65e8..0000000 --- a/lib/bridgedb/Filters.py +++ /dev/null @@ -1,175 +0,0 @@ -# BridgeDB by Nick Mathewson. -# Copyright (c) 2007-2012, The Tor Project, Inc. -# See LICENSE for licensing information - -from ipaddr import IPv6Address, IPv4Address -import logging - -funcs = {} - -def filterAssignBridgesToRing(hmac, numRings, assignedRing): - logging.debug(("Creating a filter for assigning bridges to subhashring " - "%s-of-%s...") % (assignedRing, numRings)) - ruleset = frozenset([hmac, numRings, assignedRing]) - try: - return funcs[ruleset] - except KeyError: - def _assignBridgesToRing(bridge): - digest = hmac(bridge.identity) - pos = long( digest[:8], 16 ) - which = pos % numRings + 1 - - if which == assignedRing: - return True - return False - _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing%sof%s" - % (assignedRing, numRings)) - # XXX The `description` attribute must contain an `=`, or else - # dumpAssignments() will not work correctly. - setattr(_assignBridgesToRing, "description", "ring=%d" % assignedRing) - funcs[ruleset] = _assignBridgesToRing - return _assignBridgesToRing - -def filterBridgesByRules(rules): - ruleset = frozenset(rules) - try: - return funcs[ruleset] - except KeyError: - def g(x): - r = [f(x) for f in rules] - if False in r: return False - return True - setattr(g, "description", " ".join([getattr(f,'description','') for f in rules])) - funcs[ruleset] = g - return g - -def filterBridgesByIP4(bridge): - try: - if IPv4Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 4: - return True - return False -setattr(filterBridgesByIP4, "description", "ip=4") - -def filterBridgesByIP6(bridge): - try: - if IPv6Address(bridge.address): return True - except ValueError: - pass - - for address, port, version in bridge.allVanillaAddresses: - if version == 6: - return True - return False -setattr(filterBridgesByIP6, "description", "ip=6") - -def filterBridgesByTransport(methodname, addressClass=None): - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - - ruleset = frozenset([methodname, addressClass]) - try: - return funcs[ruleset] - except KeyError: - def _filterByTransport(bridge): - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" - % (methodname, addressClass)) - setattr(_filterByTransport, "description", "transport=%s" % methodname) - funcs[ruleset] = _filterByTransport - return _filterByTransport - -def filterBridgesByUnblockedTransport(methodname, countryCode=None, addressClass=None): - """Return a filter function for :class:`~bridgedb.bridges.Bridge`s. - - The returned filter function should be called on a - :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` - has a :class:`~bridgedb.bridges.PluggableTransport` such that: - - 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches - **methodname**, - - 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an - instance of **addressClass**, and isn't known to be blocked in - **countryCode**. - - :param str methodname: A Pluggable Transport - :data:`~bridge.bridges.PluggableTransport.methodname`. - :type countryCode: str or ``None`` - :param countryCode: A two-letter country code which the filtered - :class:`PluggableTransport`s should not be blocked in. - :type addressClass: ``ipaddr.IPAddress`` - :param addressClass: The IP version that the ``Bridge``'s - ``PluggableTransport`` - :data:`~bridgedb.bridges.PluggableTransport.address`` should have. - :rtype: callable - :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. - """ - if not countryCode: - return filterBridgesByTransport(methodname, addressClass) - - if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): - addressClass = IPv4Address - - # Ignore case - methodname = methodname.lower() - countryCode = countryCode.lower() - - ruleset = frozenset([methodname, countryCode, addressClass.__name__]) - try: - return funcs[ruleset] - except KeyError: - def _filterByUnblockedTransport(bridge): - # Since bridge.transportIsBlockedIn() will return True if the - # bridge has that type of transport AND that transport is blocked, - # we can "fail fast" here by doing this faster check before - # iterating over all the transports testing for the other - # conditions. - if bridge.transportIsBlockedIn(countryCode, methodname): - return False - else: - for transport in bridge.transports: - if (transport.methodname == methodname and - isinstance(transport.address, addressClass)): - return True - return False - _filterByUnblockedTransport.__name__ = ("filterBridgesByUnblockedTransport(%s,%s,%s)" - % (methodname, countryCode, addressClass)) - setattr(_filterByUnblockedTransport, "description", - "transport=%s unblocked=%s" % (methodname, countryCode)) - funcs[ruleset] = _filterByUnblockedTransport - return _filterByUnblockedTransport - -def filterBridgesByNotBlockedIn(countryCode): - """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't - known to be blocked in **countryCode**. - - :param str countryCode: A two-letter country code. - :rtype: bool - :returns: ``True`` if at least one address of the bridge isn't blocked. - ``False`` otherwise. - """ - countryCode = countryCode.lower() - ruleset = frozenset([countryCode]) - try: - return funcs[ruleset] - except KeyError: - def _filterByNotBlockedIn(bridge): - if bridge.isBlockedIn(countryCode): - return False - return True - _filterByNotBlockedIn.__name__ = "filterBridgesByNotBlockedIn(%s)" % countryCode - setattr(_filterByNotBlockedIn, "description", "unblocked=%s" % countryCode) - funcs[ruleset] = _filterByNotBlockedIn - return _filterByNotBlockedIn diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index b699896..2ded7b3 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -11,16 +11,17 @@ #_____________________________________________________________________________ -import logging - import ipaddr +import logging from zope.interface import implements from zope.interface import Attribute from zope.interface import Interface -from bridgedb import Filters from bridgedb.crypto import getHMACFunc +from bridgedb.filters import byIPv +from bridgedb.filters import byNotBlockedIn +from bridgedb.filters import byTransport class IRequestBridges(Interface): @@ -173,20 +174,22 @@ class BridgeRequestBase(object): def generateFilters(self): self.clearFilters() - transport = self.justOnePTType() + pt = self.justOnePTType() + msg = ("Adding a filter to %s for %s for IPv%s" + % (self.__class__.__name__, self.client, self.addressClass)) - if transport: - if self.notBlockedIn: - for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByUnblockedTransport( - transport, country, self.addressClass)) - else: - self.addFilter(Filters.filterBridgesByTransport( - transport, self.addressClass)) - else: - if self.addressClass is ipaddr.IPv6Address: - self.addFilter(Filters.filterBridgesByIP6) - else: - self.addFilter(Filters.filterBridgesByIP4) + self.ipVersion = 4 + if self.addressClass is ipaddr.IPv6Address: + self.ipVersion = 6 + + if self.notBlockedIn: for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) + logging.info("%s %s bridges not blocked in %s..." % + (msg, pt or "vanilla", country)) + self.addFilter(byNotBlockedIn(country, pt, self.addressClass)) + elif pt: + logging.info("%s %s bridges..." % (msg, pt)) + self.addFilter(byTransport(pt, self.addressClass)) + else: + logging.info("%s bridges..." % msg) + self.addFilter(byIPv(self.addressClass)) diff --git a/lib/bridgedb/distribute.py b/lib/bridgedb/distribute.py index c7c9044..f48cbb6 100644 --- a/lib/bridgedb/distribute.py +++ b/lib/bridgedb/distribute.py @@ -47,7 +47,7 @@ DistributorContext { # should go in bridgedb.py } Hashring { - assignBridgesToRings() FORMERLY filterAssignBridgesToRing() + assignBridgesToSubrings() FORMERLY bridgedb.filters.assignBridgesToSubring() + filters bridges uniformly into subrings clear() / __del__() isEmpty property diff --git a/lib/bridgedb/filters.py b/lib/bridgedb/filters.py new file mode 100644 index 0000000..8843937 --- /dev/null +++ b/lib/bridgedb/filters.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_filters ; -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson <nickm(a)torproject.org> +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# (c) 2013-2015, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +import logging + +from ipaddr import IPv4Address +from ipaddr import IPv6Address + +from bridgedb.parse.addr import isIPv + + +_cache = {} + + +def bySubring(hmac, assigned, total): + """Create a filter function which filters for only the bridges which fall + into the same **assigned** subhashring (based on the results of an **hmac** + function). + + :type hmac: callable + :param hmac: An HMAC function, i.e. as returned from + :func:`bridgedb.crypto.getHMACFunc`. + :param int assigned: The subring number that we wish to draw bridges from. + For example, if a user is assigned to subring 2of3 based on their IP + address, then this function should only return bridges which would + also be assigned to subring 2of3. + :param int total: The total number of subrings. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + logging.debug(("Creating a filter for assigning bridges to subhashring " + "%s-of-%s...") % (assigned, total)) + + name = "-".join([str(hmac("")[:8]).encode('hex'), + str(assigned), "of", str(total)]) + try: + return _cache[name] + except KeyError: + def _bySubring(bridge): + position = int(hmac(bridge.identity)[:8], 16) + which = (position % total) + 1 + return True if which == assigned else False + # The `description` attribute must contain an `=`, or else + # dumpAssignments() will not work correctly. + setattr(_bySubring, "description", "ring=%d" % assigned) + _bySubring.__name__ = ("bySubring%sof%s" % (assigned, total)) + _bySubring.name = name + _cache[name] = _bySubring + return _bySubring + +def byFilters(filtres): + """Returns a filter which filters by multiple **filtres**. + + :type filtres: list + :param filtres: A list (or other iterable) of callables which some + :class:`~bridgedb.bridges.Bridge`s should be filtered according to. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + name = [] + for filtre in filtres: + name.extend(filtre.name.split(" ")) + name = " ".join(set(name)) + + try: + return _cache[name] + except KeyError: + def _byFilters(bridge): + results = [f(bridge) for f in filtres] + if False in results: + return False + return True + setattr(_byFilters, "description", + " ".join([getattr(f, "description", "") for f in filtres])) + _byFilters.name = name + _cache[name] = _byFilters + return _byFilters + +def byIPv(ipVersion=None): + """Return ``True`` if at least one of the **bridge**'s addresses has the + specified **ipVersion**. + + :param int ipVersion: Either ``4`` or ``6``. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + + name = "ipv%d" % ipVersion + try: + return _cache[name] + except KeyError: + def _byIPv(bridge): + if isIPv(ipVersion, bridge.address): + return True + else: + for address, port, version in bridge.allVanillaAddresses: + if version == ipVersion or isIPv(ipVersion, address): + return True + return False + setattr(_byIPv, "description", "ip=%d" % ipVersion) + _byIPv.__name__ = "byIPv%d()" % ipVersion + _byIPv.name = name + _cache[name] = _byIPv + return _byIPv + +byIPv4 = byIPv(4) +byIPv6 = byIPv(6) + +def byTransport(methodname=None, ipVersion=None): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + The returned filter function should be called on a + :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` + has a :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, and + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an + instance of **addressClass**. + + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s ``PluggableTransport`` + :data:`~bridgedb.bridges.PluggableTransport.address`` should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not methodname: + return byIPv(ipVersion) + + methodname = methodname.lower() + name = "transport-%s ipv%d" % (methodname, ipVersion) + + try: + return _cache[name] + except KeyError: + def _byTransport(bridge): + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byTransport, "description", "transport=%s" % methodname) + _byTransport.__name__ = "byTransport(%s,%s)" % (methodname, ipVersion) + _byTransport.name = name + _cache[name] = _byTransport + return _byTransport + +def byNotBlockedIn(countryCode=None, methodname=None, ipVersion=4): + """Returns a filter function for :class:`~bridgedb.bridges.Bridge`s. + + If a Pluggable Transport **methodname** was not specified, the returned + filter function returns ``True`` if any of the ``Bridge``'s addresses or + :class:`~bridgedb.bridges.PluggableTransport` addresses aren't blocked in + **countryCode**. See :meth:`~bridgedb.bridges.Bridge.isBlockedIn`. + + Otherwise, if a Pluggable Transport **methodname** was specified, it + returns ``True`` if the ``Bridge`` has a + :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address.version`` + equals the **ipVersion**, and isn't known to be blocked in + **countryCode**. + + :type countryCode: str or ``None`` + :param countryCode: A two-letter country code which the filtered + :class:`PluggableTransport`s should not be blocked in. + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :param int ipVersion: Either ``4`` or ``6``. The IP version that the + ``Bridge``'s addresses should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not ipVersion in (4, 6): + ipVersion = 4 + if not countryCode: + return byTransport(methodname, ipVersion) + + methodname = methodname.lower() if methodname else methodname + countryCode = countryCode.lower() + + name = [] + if methodname: + name.append("transport-%s" % methodname) + name.append("ipv%d" % ipVersion) + name.append("not-blocked-in-%s" % countryCode) + name = " ".join(name) + + try: + return _cache[name] + except KeyError: + def _byNotBlockedIn(bridge): + if not methodname: + return not bridge.isBlockedIn(countryCode) + elif methodname == "vanilla": + if bridge.address.version == ipVersion: + if not bridge.addressIsBlockedIn(countryCode, + bridge.address, + bridge.orPort): + return True + else: + # Since bridge.transportIsBlockedIn() will return True if the + # bridge has that type of transport AND that transport is + # blocked, we can "fail fast" here by doing this faster check + # before iterating over all the transports testing for the + # other conditions. + if bridge.transportIsBlockedIn(countryCode, methodname): + return False + else: + for transport in bridge.transports: + if transport.methodname == methodname: + if transport.address.version == ipVersion: + return True + return False + setattr(_byNotBlockedIn, "description", "unblocked=%s" % countryCode) + _byNotBlockedIn.__name__ = ("byTransportNotBlockedIn(%s,%s,%s)" + % (methodname, countryCode, ipVersion)) + _byNotBlockedIn.name = name + _cache[name] = _byNotBlockedIn + return _byNotBlockedIn diff --git a/lib/bridgedb/https/server.py b/lib/bridgedb/https/server.py index e8cdd62..2a1d510 100644 --- a/lib/bridgedb/https/server.py +++ b/lib/bridgedb/https/server.py @@ -47,10 +47,6 @@ from bridgedb import crypto from bridgedb import strings from bridgedb import translations from bridgedb import txrecaptcha -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.parse import headers from bridgedb.parse.addr import isIPAddress diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index 96dc21e..b3ad680 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -318,7 +318,7 @@ def isIPAddress(ip, compressed=True): return ip return False -def _isIPv(version, ip): +def isIPv(version, ip): """Check if **ip** is a certain **version** (IPv4 or IPv6). .. warning: Do *not* put any calls to the logging module in this function, @@ -352,7 +352,7 @@ def isIPv4(ip): :rtype: boolean :returns: True if the address is an IPv4 address. """ - return _isIPv(4, ip) + return isIPv(4, ip) def isIPv6(ip): """Check if an address is IPv6. @@ -364,7 +364,7 @@ def isIPv6(ip): :rtype: boolean :returns: True if the address is an IPv6 address. """ - return _isIPv(6, ip) + return isIPv(6, ip) def isValidIP(ip): """Check that an IP (v4 or v6) is valid. diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index c996daa..00726f6 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -4,9 +4,9 @@ # # :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> # please also see AUTHORS file -# :copyright: (c) 2013 Isis Lovecruft -# (c) 2007-2013, The Tor Project, Inc. -# (c) 2007-2013, all entities within the AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, all entities within the AUTHORS file # :license: 3-clause BSD, see included LICENSE for information """Module for functionality to persistently store state.""" @@ -23,7 +23,9 @@ except (ImportError, NameError): # pragma: no cover from twisted.python.reflect import safe_repr from twisted.spread import jelly -from bridgedb import Filters, Bridges, Dist +from bridgedb import Bridges +from bridgedb import Dist +from bridgedb import filters from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -32,7 +34,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(Filters, Bridges, Dist) +_security.allowModules(filters, Bridges, Dist) class MissingState(Exception): diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 95dc34e..3ec634c 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -25,11 +25,6 @@ import bridgedb.Storage import re import ipaddr -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 -from bridgedb.Filters import filterBridgesByTransport -from bridgedb.Filters import filterBridgesByNotBlockedIn - from bridgedb.Stability import BridgeHistory from bridgedb.parse import addr diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 85c1b87..46a11b0 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -24,9 +24,8 @@ from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport from bridgedb.Bridges import BridgeRing from bridgedb.Bridges import BridgeRingParameters -from bridgedb.Filters import filterBridgesByNotBlockedIn -from bridgedb.Filters import filterBridgesByIP4 -from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet from bridgedb.test.util import randomHighPort @@ -355,7 +354,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv4() - bridgeRequest.filters.append(filterBridgesByIP6) + bridgeRequest.filters.append(byIPv6) bridgeRequest.generateFilters() bridges = dist.getBridges(bridgeRequest, 1) @@ -367,7 +366,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): """Asking for bridge addresses which are simultaneously IPv6 and IPv4 @@ -379,7 +378,7 @@ class HTTPSDistributorTests(unittest.TestCase): bridgeRequest = self.randomClientRequest() bridgeRequest.withIPv6() bridgeRequest.generateFilters() - bridgeRequest.filters.append(filterBridgesByIP4) + bridgeRequest.filters.append(byIPv4) bridges = dist.getBridges(bridgeRequest, 1) self.assertEqual(len(bridges), 3) @@ -390,7 +389,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv6(self): """A request for IPv6 bridges should return IPv6 bridges.""" @@ -412,7 +411,7 @@ class HTTPSDistributorTests(unittest.TestCase): address, port = addrport.rsplit(':', 1) address = address.strip('[]') self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) - self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + self.assertIsNotNone(byIPv6(random.choice(bridges))) def test_HTTPSDistributor_getBridges_ipv4(self): """A request for IPv4 bridges should return IPv4 bridges.""" @@ -432,4 +431,4 @@ class HTTPSDistributorTests(unittest.TestCase): addrport, fingerprint = bridgeLine.split() address, port = addrport.rsplit(':', 1) self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) - self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + self.assertIsNotNone(byIPv4(random.choice(bridges))) diff --git a/lib/bridgedb/test/test_filters.py b/lib/bridgedb/test/test_filters.py new file mode 100644 index 0000000..73e5685 --- /dev/null +++ b/lib/bridgedb/test/test_filters.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# :license: see included LICENSE for information + +"""Tests for :mod:`bridgedb.filters`.""" + +from __future__ import print_function + +import ipaddr + +from twisted.trial import unittest + +from bridgedb import filters +from bridgedb.bridges import Bridge +from bridgedb.bridges import PluggableTransport +from bridgedb.crypto import getHMACFunc + + +class FiltersTests(unittest.TestCase): + """Tests for :mod:`bridgedb.filters`.""" + + def setUp(self): + """Create a Bridge whose address is 1.1.1.1, orPort is 1111, and + fingerprint is 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'. Also, + create an HMAC function whose key is 'plasma'. + """ + self.bridge = Bridge() + self.bridge.address = '1.1.1.1' + self.bridge.orPort = 1111 + self.bridge.fingerprint = 'a' * 40 + + self.hmac = getHMACFunc('plasma') + + def addIPv4VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '1.1.1.1', 1111, {}) + self.bridge.transports.append(pt) + + def addIPv6VoltronPT(self): + pt = PluggableTransport('a' * 40, 'voltron', '2006:2222::2222', 1111, {}) + self.bridge.transports.append(pt) + + def test_bySubring_1_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 1, 2) + should return that Bridge (because it is in the sub-hashring we asked + for). + """ + filtre = filters.bySubring(self.hmac, 1, 2) + self.assertTrue(filtre(self.bridge)) + + def test_bySubring_2_of_2(self): + """A Bridge with fingerprint 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + should be assigned to sub-hashring 1-of-2 (in this case, using a + particular HMAC key), and therefore filters.bySubring(HMAC, 2, 2) + should *not* return that Bridge (because it is in sub-hashring 1-of-2 + and we asked for Bridges which are in sub-hashring 2-of-2). + """ + filtre = filters.bySubring(self.hmac, 2, 2) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertTrue(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_with_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 + should return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_correct_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_bySubring_byTransport_wrong_subhashring_no_transport(self): + """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the + Bridge has no transports and is assigned to sub-hashring 1-of-2 + should return False. + """ + filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), + filters.byTransport('voltron')]) + self.assertFalse(filtre(self.bridge)) + + def test_byFilters_no_filters(self): + self.addIPv4VoltronPT() + filtre = filters.byFilters([]) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv_ipv5(self): + """Calling byIPv(ipVersion=5) should default to filterint by IPv4.""" + filtre = filters.byIPv(5) + self.assertTrue(filtre(self.bridge)) + + def test_byIPv4_address(self): + """A bridge with an IPv4 address for its main orPort address should + cause filters.byIPv4() to return True. + """ + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_orAddress(self): + """A bridge with an IPv4 address in its orAddresses address should + cause filters.byIPv4() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.bridge.orAddresses = [(ipaddr.IPv4Address('2.2.2.2'), 2222, 4)] + self.assertTrue(filters.byIPv4(self.bridge)) + + def test_byIPv4_none(self): + """A bridge with no IPv4 addresses should cause filters.byIPv4() to + return False. + """ + self.bridge.address = ipaddr.IPv6Address('2006:2222::2222') + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertFalse(filters.byIPv4(self.bridge)) + + def test_byIPv6_address(self): + """A bridge with an IPv6 address for its main orPort address should + cause filters.byIPv6() to return True. + """ + self.bridge.address = '2006:2222::2222' + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_orAddress(self): + """A bridge with an IPv6 address in its orAddresses address should + cause filters.byIPv6() to return True. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + self.assertTrue(filters.byIPv6(self.bridge)) + + def test_byIPv6_none(self): + """A bridge with no IPv6 addresses should cause filters.byIPv6() to + return False. + """ + self.assertFalse(filters.byIPv6(self.bridge)) + + def test_byTransport_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron', ipVersion=6) to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_with_transport_ipv6_filtering_by_ipv4(self): + """A bridge with an IPv6 voltron transport should cause + byTransport('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_no_transports(self): + """A bridge without any transports should cause + byTransport('voltron') to return False. + """ + filtre = filters.byTransport('voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv4(self): + """byTransport() without namimg a transport to filter by should just + return the bridge's IPv4 address. + """ + filtre = filters.byTransport() + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_vanilla_ipv6(self): + """byTranspfort(ipVersion=6) without namimg a transport to filter by + should just return the bridge's IPv4 address. + """ + self.bridge.orAddresses = [(ipaddr.IPv6Address('2006:3333::3333'), 3333, 6)] + filtre = filters.byTransport(ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byTransport_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byTransport('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byTransport('obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True (because it calls + filters.byTransport). + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn(None, methodname='voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4(self): + """A bridge with an IPv4 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_blocked(self): + """A bridge with an IPv4 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv4VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('CN', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6(self): + """A bridge with an IPv6 voltron transport should cause + byNotBlockedIn('voltron') to return True. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv4_not_blocked_ipv4(self): + """A bridge with an IPv6 voltron transport which is not blocked in China + should cause byNotBlockedIn('cn', 'voltron') to return False, because + the IP version is wrong. + """ + self.addIPv6VoltronPT() + filtre = filters.byNotBlockedIn('cn', 'voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_with_transport_ipv6_blocked(self): + """A bridge with an IPv6 voltron transport which is blocked should + cause byNotBlockedIn('voltron') to return False. + """ + self.addIPv6VoltronPT() + self.bridge.setBlockedIn('CN') + filtre = filters.byNotBlockedIn('cn', 'voltron', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_countryCode_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('voltron') to return False (because it calls + filters.byTransport('voltron')). + """ + filtre = filters.byNotBlockedIn(None, methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports(self): + """A bridge without any transports should cause + byNotBlockedIn('cn', 'voltron') to return False. + """ + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_no_transports_blocked(self): + """A bridge without any transports which is also blocked should cause + byNotBlockedIn('voltron') to return False. + """ + self.bridge.setBlockedIn('cn') + filtre = filters.byNotBlockedIn('cn', methodname='voltron') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_wrong_transport(self): + """A bridge with only a Voltron transport should cause + byNotBlockedIn('obfs3') to return False. + """ + self.addIPv4VoltronPT() + filtre = filters.byNotBlockedIn('cn', methodname='obfs3') + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_ipv5(self): + """Calling byNotBlockedIn([…], ipVersion=5) should default to IPv4.""" + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', ipVersion=5) + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked(self): + """Calling byNotBlockedIn('vanilla') should return the IPv4 vanilla + address, if it is not blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla') + self.assertTrue(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_not_blocked_ipv6(self): + """Calling byNotBlockedIn('vanilla', ipVersion=6) should not return the + IPv4 vanilla address, even if it is not blocked, because it has the + wrong IP version. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('cn', methodname='vanilla', ipVersion=6) + self.assertFalse(filtre(self.bridge)) + + def test_byNotBlockedIn_vanilla_blocked(self): + """Calling byNotBlockedIn('vanilla') should not return the IPv4 vanilla + address, if it is blocked. + """ + self.bridge.setBlockedIn('ru') + filtre = filters.byNotBlockedIn('ru', methodname='vanilla') + self.assertFalse(filtre(self.bridge))

1 0

[bridgedb/develop] Add subring numbers to their names to improve log clarity.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit aecbe451d65f55e92de3554ef8a17bb858dda3e1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 02:47:09 2015 +0000 Add subring numbers to their names to improve log clarity. * CHANGE bridgedb.Filters.filterAssignBridgesToRing() to rename generated filter functions to include the subring number and the total number of subrings, e.g. filter.__name__ = "filterAssignBridgesToRing3of4" * CHANGE bridgedb.Bridges.FilteredBridgeSplitter.addRing() to name subrings based on their number. For example, if an HTTPSDistributor has 3 total subrings and subring 3/3 is for known-proxy users, then the subrings would be named like: - HTTPS-IP4-1of3 - HTTPS-IP4-2of3 - HTTPS-IP4-Proxy - HTTPS-IP6-1of3 - HTTPS-IP6-2of3 - HTTPS-IP6-Proxy --- lib/bridgedb/Bridges.py | 18 +++++++++++++----- lib/bridgedb/Filters.py | 10 ++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 4b77cdc..507013d 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -613,13 +613,21 @@ class FilteredBridgeSplitter(object): filterNames = self.extractFilterNames(ringname) subringName = [self.distributorName] + subringNumber = None for filterName in filterNames: - if filterName != 'filterAssignBridgesToRing': - subringName.append(filterName.strip('filterBridgesBy')) + if filterName.startswith('filterAssignBridgesToRing'): + subringNumber = filterName.lstrip('filterAssignBridgesToRing') + else: + subringName.append(filterName.lstrip('filterBridgesBy')) + if subring.name and 'Proxy' in subring.name: + subringName.append('Proxy') + elif subringNumber: + subringName.append(subringNumber) subringName = '-'.join([x for x in subringName]) subring.setName(subringName) - logging.info("Adding subring to %s hashring..." % subring.name) + logging.info("Adding %s subring %s to the %s Distributor's hashring..." % + (subring.name, subringNumber, self.distributorName)) logging.info(" Subring filters: %s" % filterNames) #TODO: drop LRU ring if len(self.filterRings) > self.max_cached_rings @@ -631,8 +639,8 @@ class FilteredBridgeSplitter(object): if isinstance(bridge, Bridge) and filterFn(bridge): subring.insert(bridge) inserted += 1 - logging.info("Bridges inserted into %s subring: %d" - % (subring.name, inserted)) + logging.info("Bridges inserted into %s subring %s: %d" + % (subring.name, subringNumber, inserted)) return True diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index fb0197b..cb5e782 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -8,11 +8,9 @@ import logging funcs = {} def filterAssignBridgesToRing(hmac, numRings, assignedRing): - #XXX: ruleset should have a key unique to this function - # ruleset ensures that the same - logging.debug("Creating a filter for assigning bridges to hashrings...") + logging.debug(("Creating a filter for assigning bridges to subhashring " + "%s-of-%s...") % (assignedRing, numRings)) ruleset = frozenset([hmac, numRings, assignedRing]) - try: return funcs[ruleset] except KeyError: @@ -24,8 +22,8 @@ def filterAssignBridgesToRing(hmac, numRings, assignedRing): if which == assignedRing: return True return False - _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing(%s, %s, %s)" - % (hmac, numRings, assignedRing)) + _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing%sof%s" + % (assignedRing, numRings)) # XXX The `description` attribute must contain an `=`, or else # dumpAssignments() will not work correctly. setattr(_assignBridgesToRing, "description", "ring=%d" % assignedRing)

1 0

[bridgedb/develop] Reuse mocked Bridges across various unittests.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 7632085c0725e32001e41774564bffe7cb221584 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun May 10 22:18:26 2015 +0000 Reuse mocked Bridges across various unittests. --- lib/bridgedb/test/util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index f164aeb..2cddc11 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -168,6 +168,8 @@ randomValidIPv4String = valid(randomIPv4String) randomValidIPv6String = valid(randomIPv6String) randomValidIPString = valid(randomIPString) +_FAKE_BRIDGES = [] + def generateFakeBridges(n=500): """Generate a set of **n** :class:`~bridgedb.bridges.Bridges` with random data. @@ -175,6 +177,11 @@ def generateFakeBridges(n=500): from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport + global _FAKE_BRIDGES + + if _FAKE_BRIDGES: + return _FAKE_BRIDGES + bridges = [] for i in range(n): @@ -199,6 +206,7 @@ def generateFakeBridges(n=500): bridge.orAddresses = addrs bridges.append(bridge) + _FAKE_BRIDGES = bridges return bridges

1 0

[bridgedb/develop] PEP8 remove extra blank line in bridgedb.bridges module.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 3c990f7cd86b7f2ca8799ee4b7054b5e0167620e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue May 12 07:28:26 2015 +0000 PEP8 remove extra blank line in bridgedb.bridges module. --- lib/bridgedb/bridges.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index c98d54c..f6e1d6b 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -1104,7 +1104,6 @@ class Bridge(BridgeBackwardsCompatibility): ("Client requested transport %s, but bridge %s doesn't " "have any of that transport!") % (desired, self)) - unblocked = [] for pt in transports: if not sum([self.transportIsBlockedIn(cc, pt.methodname)

1 0

[bridgedb/develop] Change PluggableTransport.methodname to be a @property.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit 70019ff8944cd29de272d34f432247b055045006 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 03:02:57 2015 +0000 Change PluggableTransport.methodname to be a @property. This ensures that it's always lowercased when set, removing the need for so many lower() calls. --- lib/bridgedb/Filters.py | 12 ++++++------ lib/bridgedb/bridges.py | 29 ++++++++++++++++++++++++++--- lib/bridgedb/test/test_bridges.py | 10 +++++----- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index cb5e782..eb7db4e 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -71,19 +71,19 @@ def filterBridgesByTransport(methodname, addressClass=None): if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): addressClass = IPv4Address + # Ignore case + methodname = methodname.lower() + ruleset = frozenset([methodname, addressClass]) try: return funcs[ruleset] except KeyError: - def _filterByTransport(bridge): for transport in bridge.transports: - if isinstance(transport.address, addressClass): - # ignore method name case - if transport.methodname.lower() == methodname.lower(): - return True + if (transport.methodname == methodname and + isinstance(transport.address, addressClass)): + return True return False - _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" % (methodname, addressClass)) setattr(_filterByTransport, "description", "transport=%s" % methodname) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index dc5b78d..c98d54c 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -316,14 +316,13 @@ class PluggableTransport(BridgeAddressBase): super(PluggableTransport, self).__init__() self._port = None self._methodname = None - self._arguments = None + self._blockedIn = {} self.fingerprint = fingerprint self.address = address self.port = port self.methodname = methodname self.arguments = arguments - self._blockedIn = {} # Because we can intitialise this class with the __init__() # parameters, or use the ``updateFromStemTransport()`` method, we'll @@ -464,6 +463,30 @@ class PluggableTransport(BridgeAddressBase): """Reset this ``PluggableTransport``'s port to ``None``.""" self._port = None + @property + def methodname(self): + """Get this :class:`PluggableTransport`'s methodname. + + :rtype: str + :returns: The (lowercased) methodname of this ``PluggableTransport``, + i.e. ``"obfs3"``, ``"scramblesuit"``, etc. + """ + return self._methodname + + @methodname.setter + def methodname(self, value): + """Set this ``PluggableTransport``'s methodname. + + .. hint:: The **value** will be automatically lowercased. + + :param str value: The new methodname. + """ + if value: + try: + self._methodname = value.lower() + except (AttributeError, TypeError): + raise TypeError("methodname must be a str or unicode") + def getTransportLine(self, includeFingerprint=True, bridgePrefix=False): """Get a Bridge Line for this :class:`PluggableTransport`. @@ -1317,7 +1340,7 @@ class Bridge(BridgeBackwardsCompatibility): **countryCode**, ``False`` otherwise. """ for pt in self.transports: - if pt.methodname.lower() == methodname.lower(): + if pt.methodname == methodname.lower(): if self.addressIsBlockedIn(countryCode, pt.address, pt.port): logging.info("Transport %s of bridge %s is blocked in %s." % (pt.methodname, self, countryCode)) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 499a8c2..db662ae 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -659,11 +659,11 @@ class PluggableTransportTests(unittest.TestCase): bridgeLine = pt.getTransportLine() # We have to check for substrings because we don't know which order - # the PT arguments will end up in the bridge line. Fortunately, the - # following three are the only ones which are important to have in - # order: - self.assertTrue(bridgeLine.startswith("voltronPT")) - self.assertSubstring("voltronPT 1.2.3.4:443 " + self.fingerprint, + # the PT arguments will end up in the bridge line. We also have to + # check for the lowercased transport name. Fortunately, the following + # three are the only ones which are important to have in order: + self.assertTrue(bridgeLine.startswith("voltronpt")) + self.assertSubstring("voltronpt 1.2.3.4:443 " + self.fingerprint, bridgeLine) # These ones can be in any order, but they should be at the end of the # bridge line:

1 0

[bridgedb/develop] Add several additional tests for bridgedb.email.distributor.
by isis＠torproject.org 25 Jun '15

25 Jun '15

commit bb5b29dd19f32fb62d01a4d7ffb65b34493645e4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 22 02:58:31 2015 +0000 Add several additional tests for bridgedb.email.distributor. --- lib/bridgedb/email/distributor.py | 12 +- lib/bridgedb/test/test_email_distributor.py | 177 ++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/email/distributor.py b/lib/bridgedb/email/distributor.py index b73c082..d8ea9bf 100644 --- a/lib/bridgedb/email/distributor.py +++ b/lib/bridgedb/email/distributor.py @@ -101,7 +101,7 @@ class EmailDistributor(Distributor): def bridgesPerResponse(self, hashring=None): return super(EmailDistributor, self).bridgesPerResponse(hashring) - def getBridges(self, bridgeRequest, interval): + def getBridges(self, bridgeRequest, interval, clock=None): """Return a list of bridges to give to a user. .. hint:: All checks on the email address (which should be stored in @@ -119,6 +119,13 @@ class EmailDistributor(Distributor): address. :param interval: The time period when we got this request. This can be any string, so long as it changes with every period. + :type clock: :api:`twisted.internet.task.Clock` + :param clock: If given, use the clock to ask what time it is, rather + than :api:`time.time`. This should likely only be used for + testing. + :rtype: list or ``None`` + :returns: A list of :class:`~bridgedb.bridges.Bridges` for the + ``bridgeRequest.client``, if allowed. Otherwise, returns ``None``. """ if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): raise addr.BadEmail( @@ -129,6 +136,9 @@ class EmailDistributor(Distributor): now = time.time() + if clock: + now = clock.seconds() + with bridgedb.Storage.getDB() as db: wasWarned = db.getWarnedEmail(bridgeRequest.client) lastSaw = db.getEmailTime(bridgeRequest.client) diff --git a/lib/bridgedb/test/test_email_distributor.py b/lib/bridgedb/test/test_email_distributor.py index 96eb306..b4d88b2 100644 --- a/lib/bridgedb/test/test_email_distributor.py +++ b/lib/bridgedb/test/test_email_distributor.py @@ -15,6 +15,7 @@ import logging import tempfile import os +from twisted.internet.task import Clock from twisted.trial import unittest import bridgedb.Storage @@ -24,6 +25,7 @@ from bridgedb.email.distributor import EmailDistributor from bridgedb.email.distributor import IgnoreEmail from bridgedb.email.distributor import TooSoonEmail from bridgedb.email.request import EmailBridgeRequest +from bridgedb.parse.addr import BadEmail from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import normalizeEmail from bridgedb.test.util import generateFakeBridges @@ -37,13 +39,14 @@ BRIDGES = generateFakeBridges() class EmailDistributorTests(unittest.TestCase): """Tests for :class:`bridgedb.email.distributor.EmailDistributor`.""" + # Fail any tests which take longer than 15 seconds. + timeout = 15 + def setUp(self): - self.fd, self.fname = tempfile.mkstemp() + self.fd, self.fname = tempfile.mkstemp(suffix=".sqlite", dir=os.getcwd()) bridgedb.Storage.initializeDBLock() self.db = bridgedb.Storage.openDatabase(self.fname) bridgedb.Storage.setDBFilename(self.fname) - self.cur = self.db.cursor() - self.db.close() self.bridges = BRIDGES self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' @@ -68,7 +71,68 @@ class EmailDistributorTests(unittest.TestCase): bridgeRequest.generateFilters() return bridgeRequest - def test_EmailDistributor_rate_limit(self): + def test_EmailDistributor_getBridges_default_client(self): + """If EmailBridgeRequest.client was not set, then getBridges() should + raise a bridgedb.parse.addr.BadEmail exception. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # The "default" client is literally the string "default", see + # bridgedb.bridgerequest.BridgeRequestBase. + bridgeRequest = self.makeClientRequest('default') + + self.assertRaises(BadEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_getBridges_with_whitelist(self): + """If an email address is in the whitelist, it should get a response + every time it asks (i.e. no rate-limiting). + """ + # The whitelist should be in the form {EMAIL: GPG_FINGERPRINT} + whitelist = {'white(a)list.ed': '0123456789ABCDEF0123456789ABCDEF01234567'} + dist = EmailDistributor(self.key, self.domainmap, self.domainrules, + whitelist=whitelist) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # A request from a whitelisted address should always get a response. + bridgeRequest = self.makeClientRequest('white(a)list.ed') + for i in range(5): + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + def test_EmailDistributor_getBridges_rate_limit_multiple_clients(self): + """Each client should be rate-limited separately.""" + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest1 = self.makeClientRequest('abc(a)example.com') + bridgeRequest2 = self.makeClientRequest('def(a)example.com') + bridgeRequest3 = self.makeClientRequest('ghi(a)example.com') + + # The first request from 'abc' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest1, 1)), 3) + # The second from 'abc' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest1, 1) + # The first request from 'def' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest2, 1)), 3) + # The third from 'abc' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest1, 1) + # The second from 'def' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest2, 1) + # The third from 'def' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest2, 1) + # The fourth from 'abc' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest1, 1) + # The first request from 'ghi' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest3, 1)), 3) + # The second from 'ghi' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest3, 1) + # The third from 'ghi' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest3, 1) + # The fourth from 'ghi' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest3, 1) + + def test_EmailDistributor_getBridges_rate_limit(self): """A client's first email should return bridges. The second should return a warning, and the third should receive no response. """ @@ -77,16 +141,117 @@ class EmailDistributorTests(unittest.TestCase): bridgeRequest = self.makeClientRequest('abc(a)example.com') - # The first request should get a response with bridges + # The first request should get a response with bridges. bridges = dist.getBridges(bridgeRequest, 1) self.assertGreater(len(bridges), 0) [self.assertIsInstance(b, Bridge) for b in bridges] self.assertEqual(len(bridges), 3) - # The second gets a warning, and the third is ignored + # The second gets a warning, and the third is ignored. self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + def test_EmailDistributor_getBridges_rate_limit_expiry(self): + """A client's first email should return bridges. The second should + return a warning, and the third should receive no response. After the + EmailDistributor.emailRateMax is up, the client should be able to + receive a response again. + """ + clock = Clock() + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1, clock)), 3) + # The second gets a warning, and the rest are ignored. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + + clock.advance(2 * dist.emailRateMax) + + # The client should again a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1)), 3, clock) + + def test_EmailDistributor_cleanDatabase(self): + """Calling cleanDatabase() should cleanup email times in database, but + not allow clients who have been recently warned and/or ignored to + receive a response again until the remainder of their MAX_EMAIL_RATE + time is up. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1)), 3) + # The second gets a warning, and the third is ignored. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + dist.cleanDatabase() + + # Cleaning the warning email times in the database shouldn't cause + # 'abc(a)example.com' to be able to email again, because only the times + # which aren't older than EMAIL_MAX_RATE should be cleared. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_prepopulateRings(self): + """Calling prepopulateRings() should add two rings to the + EmailDistributor.hashring. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + + # There shouldn't be any subrings yet. + self.assertEqual(len(dist.hashring.filterRings), 0) + + dist.prepopulateRings() + + # There should now be two subrings, but the subrings should be empty. + self.assertEqual(len(dist.hashring.filterRings), 2) + for (filtre, subring) in dist.hashring.filterRings.values(): + self.assertEqual(len(subring), 0) + + # The subrings in this Distributor have gross names, because the + # filter functions (including their addresses in memory!) are used as + # the subring names. In this case, we should have something like: + # + # frozenset([<function byIPv6 at 0x7eff7ad7fc80>]) + # + # and + # + # frozenset([<function byIPv4 at 0x7eff7ad7fc08>]) + # + # So we have to join the strings together and check the whole thing, + # since we have no other way to use these stupid subring names to + # index into the dictionary they are stored in, because the memory + # addresses are unknowable until runtime. + + # There should be an IPv4 subring and an IPv6 ring: + ringnames = dist.hashring.filterRings.keys() + self.failUnlessIn("IPv4", "".join([str(ringname) for ringname in ringnames])) + self.failUnlessIn("IPv6", "".join([str(ringname) for ringname in ringnames])) + + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # There should still be two subrings. + self.assertEqual(len(dist.hashring.filterRings), 2) + for (filtre, subring) in dist.hashring.filterRings.values(): + self.assertGreater(len(subring), 0) + + # Ugh, the hashring code is so gross looking. + subrings = dist.hashring.filterRings + subring1 = subrings.values()[0][1] + subring2 = subrings.values()[1][1] + # Each subring should have roughly the same number of bridges. + # (Having ±10 bridges in either ring, out of 500 bridges total, should + # be so bad.) + self.assertApproximates(len(subring1), len(subring2), 10) + def test_EmailDistributor_unsupported_domain(self): """An unsupported domain should raise an UnsupportedDomain exception.""" self.assertRaises(UnsupportedDomain, normalizeEmail,

1 0