July 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Rewrite testDistWithFilterIP6() in legacy_Tests.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit c1637a017d7e34a79b71829f0918184b63055c6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 23:35:33 2015 +0000 Rewrite testDistWithFilterIP6() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 16 ---------------- lib/bridgedb/test/test_Dist.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 9a0c74f..c495412 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -236,22 +236,6 @@ class IPBridgeDistTests(unittest.TestCase): # self.assertEquals(len(fps), 5) # self.assertTrue(count >= 1) - def testDistWithFilterIP6(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(500): - bridges = d.getBridges(randomIPv4String(), - "faketimestamp", - bridgeFilterRules=[filterBridgesByIP6]) - bridge = random.choice(bridges) - bridge_line = bridge.getConfigLine(addressClass=ipaddr.IPv6Address) - address, portlist = networkstatus.parseALine(bridge_line) - assert type(ipaddr.IPAddress(address)) is ipaddr.IPv6Address - assert filterBridgesByIP6(random.choice(bridges)) - def testDistWithFilterBlockedCountriesAdvanced(self): d = bridgedb.Dist.HTTPSDistributor(3, "Foo") for _ in xrange(250): diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 82ef823..bc437ba 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -289,6 +289,28 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + def test_HTTPSDistributor_getBridges_ipv6(self): + """A request for IPv6 bridges should return IPv6 bridges.""" + dist = Dist.HTTPSDistributor(3, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + for i in xrange(500): + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, "faketimestamp") + self.assertTrue(type(bridges) is list) + self.assertGreater(len(bridges), 0) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges))) + def test_HTTPSDistributor_getBridges_ipv4(self): """A request for IPv4 bridges should return IPv4 bridges.""" dist = Dist.HTTPSDistributor(1, self.key)

1 0

[bridgedb/master] Remove unnecessary use of logSafely() in bridgedb.Bridges.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 68434ca69f5a5b38fc877abf03064eddbe879956 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 20 03:45:24 2015 +0000 Remove unnecessary use of logSafely() in bridgedb.Bridges. Logging str(bridge) will produce a hashed fingerprint if safelogging is enabled, so there's no use in calling logSafely(bridge.fingerprint). --- lib/bridgedb/Bridges.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 1a06dd8..4b77cdc 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -539,14 +539,12 @@ class FilteredBridgeSplitter(object): """ # The bridge must be running to insert it: if not bridge.flags.running: - logging.warn( - "Skipping hashring insertion for non-running bridge: '%s'" - % logSafely(bridge.fingerprint)) + logging.warn(("Skipping hashring insertion for non-running " + "bridge: %s") % bridge) return index = 0 - logging.debug("Inserting %s into hashring" - % (logSafely(bridge.fingerprint))) + logging.debug("Inserting %s into hashring..." % bridge) for old_bridge in self.bridges[:]: if bridge.fingerprint == old_bridge.fingerprint: self.bridges[index] = bridge @@ -557,8 +555,8 @@ class FilteredBridgeSplitter(object): for ringname, (filterFn, subring) in self.filterRings.items(): if filterFn(bridge): subring.insert(bridge) - logging.debug("Inserted bridge '%s' into '%s' sub hashring" - % (logSafely(bridge.fingerprint), ringname)) + logging.debug("Inserted bridge %s into %s subhashring." % + (bridge, ringname)) def extractFilterNames(self, ringname): """Get the names of the filters applied to a particular sub hashring.

1 0

[bridgedb/master] Add a filter for unblocked pluggable transports.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 5088163284bd512082f113fe84f27f7af86973b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 03:08:03 2015 +0000 Add a filter for unblocked pluggable transports. * ADD Filters.filterBridgesByUnblockedTransport(). * CHANGE bridgerequest.BridgeRequestBase.generateFilters() to handle requests for unblocked bridges with pluggable transports. --- lib/bridgedb/Filters.py | 61 +++++++++++++++++++++++++++++++++++++++++ lib/bridgedb/bridgerequest.py | 25 +++++++++++------ 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index eb7db4e..d0d65e8 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -90,6 +90,67 @@ def filterBridgesByTransport(methodname, addressClass=None): funcs[ruleset] = _filterByTransport return _filterByTransport +def filterBridgesByUnblockedTransport(methodname, countryCode=None, addressClass=None): + """Return a filter function for :class:`~bridgedb.bridges.Bridge`s. + + The returned filter function should be called on a + :class:`~bridgedb.bridges.Bridge`. It returns ``True`` if the ``Bridge`` + has a :class:`~bridgedb.bridges.PluggableTransport` such that: + + 1. The :data:`~bridge.bridges.PluggableTransport.methodname` matches + **methodname**, + + 2. The :data:`~bridgedb.bridges.PluggableTransport.address`` is an + instance of **addressClass**, and isn't known to be blocked in + **countryCode**. + + :param str methodname: A Pluggable Transport + :data:`~bridge.bridges.PluggableTransport.methodname`. + :type countryCode: str or ``None`` + :param countryCode: A two-letter country code which the filtered + :class:`PluggableTransport`s should not be blocked in. + :type addressClass: ``ipaddr.IPAddress`` + :param addressClass: The IP version that the ``Bridge``'s + ``PluggableTransport`` + :data:`~bridgedb.bridges.PluggableTransport.address`` should have. + :rtype: callable + :returns: A filter function for :class:`~bridgedb.bridges.Bridge`s. + """ + if not countryCode: + return filterBridgesByTransport(methodname, addressClass) + + if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): + addressClass = IPv4Address + + # Ignore case + methodname = methodname.lower() + countryCode = countryCode.lower() + + ruleset = frozenset([methodname, countryCode, addressClass.__name__]) + try: + return funcs[ruleset] + except KeyError: + def _filterByUnblockedTransport(bridge): + # Since bridge.transportIsBlockedIn() will return True if the + # bridge has that type of transport AND that transport is blocked, + # we can "fail fast" here by doing this faster check before + # iterating over all the transports testing for the other + # conditions. + if bridge.transportIsBlockedIn(countryCode, methodname): + return False + else: + for transport in bridge.transports: + if (transport.methodname == methodname and + isinstance(transport.address, addressClass)): + return True + return False + _filterByUnblockedTransport.__name__ = ("filterBridgesByUnblockedTransport(%s,%s,%s)" + % (methodname, countryCode, addressClass)) + setattr(_filterByUnblockedTransport, "description", + "transport=%s unblocked=%s" % (methodname, countryCode)) + funcs[ruleset] = _filterByUnblockedTransport + return _filterByUnblockedTransport + def filterBridgesByNotBlockedIn(countryCode): """Return ``True`` if at least one of a bridge's (transport) bridgelines isn't known to be blocked in **countryCode**. diff --git a/lib/bridgedb/bridgerequest.py b/lib/bridgedb/bridgerequest.py index 532558d..b699896 100644 --- a/lib/bridgedb/bridgerequest.py +++ b/lib/bridgedb/bridgerequest.py @@ -171,15 +171,22 @@ class BridgeRequestBase(object): return ptType def generateFilters(self): - if self.addressClass is ipaddr.IPv6Address: - self.addFilter(Filters.filterBridgesByIP6) - else: - self.addFilter(Filters.filterBridgesByIP4) + self.clearFilters() transport = self.justOnePTType() + if transport: - self.clearFilters() - self.addFilter(Filters.filterBridgesByTransport(transport, - self.addressClass)) - for country in self.notBlockedIn: - self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower())) + if self.notBlockedIn: + for country in self.notBlockedIn: + self.addFilter(Filters.filterBridgesByUnblockedTransport( + transport, country, self.addressClass)) + else: + self.addFilter(Filters.filterBridgesByTransport( + transport, self.addressClass)) + else: + if self.addressClass is ipaddr.IPv6Address: + self.addFilter(Filters.filterBridgesByIP6) + else: + self.addFilter(Filters.filterBridgesByIP4) + for country in self.notBlockedIn: + self.addFilter(Filters.filterBridgesByNotBlockedIn(country.lower()))

1 0

[bridgedb/master] Rewrite testDistWithFilterBlockedCountriesAdvanced() in legacy_Tests.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 2de77ba16992b8a967bfe68cebc2c17c2b3998fc Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 02:33:08 2015 +0000 Rewrite testDistWithFilterBlockedCountriesAdvanced() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 41 +------------------------------------ lib/bridgedb/test/test_Dist.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 2f1da67..95dc34e 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -183,44 +183,6 @@ class EmailBridgeDistTests(unittest.TestCase): {'example.com':'example.com'}, {'example.com':[]}) -class IPBridgeDistTests(unittest.TestCase): - - def testDistWithFilterBlockedCountriesAdvanced(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True, transports=True)) - d.insert(fakeBridge(or_addresses=True, transports=True)) - - for b in d.hashring.bridges: - # china blocks some transports - for pt in b.transports: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (pt.address, pt.port) - b.blockingCountries[key] = set(['cn']) - for address, portlist in b.or_addresses.items(): - # china blocks some transports - for port in portlist: - if random.choice(xrange(2)) > 0: - key = "%s:%s" % (address, port) - b.blockingCountries[key] = set(['cn']) - key = "%s:%s" % (b.ip, b.orport) - b.blockingCountries[key] = set(['cn']) - - # we probably will get at least one bridge back! - # it's pretty unlikely to lose a coin flip 250 times in a row - for i in xrange(5): - b = d.getBridges(randomIPString(), "x", - bridgeFilterRules=[ - filterBridgesByNotBlockedIn("cn"), - filterBridgesByTransport('obfs2'), - ]) - try: assert len(b) > 0 - except AssertionError: - print("epic fail") - b = d.getBridges(randomIPString(), "x", bridgeFilterRules=[ - filterBridgesByNotBlockedIn("us")]) - assert len(b) > 0 - class SQLStorageTests(unittest.TestCase): def setUp(self): @@ -450,8 +412,7 @@ def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, - BridgeStabilityTests]: + for klass in [SQLStorageTests, EmailBridgeDistTests, BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) return suite diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index e8a17e4..85c1b87 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -244,6 +244,45 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertNotIn(b.fingerprint, blockedIR) self.assertGreater(len(bridges), 0) + def test_HTTPSDistributor_getBridges_with_varied_blocked_bridges(self): + dist = Dist.HTTPSDistributor(1, self.key) + bridges = self.bridges[:] + + for bridge in bridges: + # Pretend that China blocks all vanilla bridges: + bridge.setBlockedIn('cn', methodname='vanilla') + # Pretend that China blocks all obfs2: + bridge.setBlockedIn('cn', methodname='obfs2') + # Pretend that China blocks some obfs3: + if self.coinFlip(): + bridge.setBlockedIn('cn', methodname='obfs3') + + [dist.insert(bridge) for bridge in bridges] + + for i in xrange(5): + bridgeRequest1 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest1.transports.append('obfs2') + bridgeRequest1.generateFilters() + # We shouldn't get any obfs2 bridges, since they're all blocked in + # China: + bridges = dist.getBridges(bridgeRequest1, "faketimestamp") + self.assertEqual(len(bridges), 0) + + bridgeRequest2 = self.randomClientRequestForNotBlockedIn('cn') + bridgeRequest2.transports.append('obfs3') + bridgeRequest2.generateFilters() + # We probably will get at least one bridge back! It's pretty + # unlikely to lose a coin flip 500 times in a row. + bridges = dist.getBridges(bridgeRequest2, "faketimestamp") + self.assertGreater(len(bridges), 0) + + bridgeRequest3 = self.randomClientRequestForNotBlockedIn('nl') + bridgeRequest3.transports.append('obfs3') + bridgeRequest3.generateFilters() + # We should get bridges, since obfs3 isn't blocked in netherlands: + bridges = dist.getBridges(bridgeRequest3, "faketimestamp") + self.assertGreater(len(bridges), 0) + def test_HTTPSDistributor_getBridges_with_proxy_and_nonproxy_users(self): """An HTTPSDistributor should give separate bridges to proxy users.""" proxies = ProxySet(['.'.join(['1.1.1', str(x)]) for x in range(1, 256)])

1 0

[bridgedb/master] Add subring numbers to their names to improve log clarity.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit aecbe451d65f55e92de3554ef8a17bb858dda3e1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 02:47:09 2015 +0000 Add subring numbers to their names to improve log clarity. * CHANGE bridgedb.Filters.filterAssignBridgesToRing() to rename generated filter functions to include the subring number and the total number of subrings, e.g. filter.__name__ = "filterAssignBridgesToRing3of4" * CHANGE bridgedb.Bridges.FilteredBridgeSplitter.addRing() to name subrings based on their number. For example, if an HTTPSDistributor has 3 total subrings and subring 3/3 is for known-proxy users, then the subrings would be named like: - HTTPS-IP4-1of3 - HTTPS-IP4-2of3 - HTTPS-IP4-Proxy - HTTPS-IP6-1of3 - HTTPS-IP6-2of3 - HTTPS-IP6-Proxy --- lib/bridgedb/Bridges.py | 18 +++++++++++++----- lib/bridgedb/Filters.py | 10 ++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 4b77cdc..507013d 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -613,13 +613,21 @@ class FilteredBridgeSplitter(object): filterNames = self.extractFilterNames(ringname) subringName = [self.distributorName] + subringNumber = None for filterName in filterNames: - if filterName != 'filterAssignBridgesToRing': - subringName.append(filterName.strip('filterBridgesBy')) + if filterName.startswith('filterAssignBridgesToRing'): + subringNumber = filterName.lstrip('filterAssignBridgesToRing') + else: + subringName.append(filterName.lstrip('filterBridgesBy')) + if subring.name and 'Proxy' in subring.name: + subringName.append('Proxy') + elif subringNumber: + subringName.append(subringNumber) subringName = '-'.join([x for x in subringName]) subring.setName(subringName) - logging.info("Adding subring to %s hashring..." % subring.name) + logging.info("Adding %s subring %s to the %s Distributor's hashring..." % + (subring.name, subringNumber, self.distributorName)) logging.info(" Subring filters: %s" % filterNames) #TODO: drop LRU ring if len(self.filterRings) > self.max_cached_rings @@ -631,8 +639,8 @@ class FilteredBridgeSplitter(object): if isinstance(bridge, Bridge) and filterFn(bridge): subring.insert(bridge) inserted += 1 - logging.info("Bridges inserted into %s subring: %d" - % (subring.name, inserted)) + logging.info("Bridges inserted into %s subring %s: %d" + % (subring.name, subringNumber, inserted)) return True diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index fb0197b..cb5e782 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -8,11 +8,9 @@ import logging funcs = {} def filterAssignBridgesToRing(hmac, numRings, assignedRing): - #XXX: ruleset should have a key unique to this function - # ruleset ensures that the same - logging.debug("Creating a filter for assigning bridges to hashrings...") + logging.debug(("Creating a filter for assigning bridges to subhashring " + "%s-of-%s...") % (assignedRing, numRings)) ruleset = frozenset([hmac, numRings, assignedRing]) - try: return funcs[ruleset] except KeyError: @@ -24,8 +22,8 @@ def filterAssignBridgesToRing(hmac, numRings, assignedRing): if which == assignedRing: return True return False - _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing(%s, %s, %s)" - % (hmac, numRings, assignedRing)) + _assignBridgesToRing.__name__ = ("filterAssignBridgesToRing%sof%s" + % (assignedRing, numRings)) # XXX The `description` attribute must contain an `=`, or else # dumpAssignments() will not work correctly. setattr(_assignBridgesToRing, "description", "ring=%d" % assignedRing)

1 0

[bridgedb/master] Change PluggableTransport.methodname to be a @property.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 70019ff8944cd29de272d34f432247b055045006 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 03:02:57 2015 +0000 Change PluggableTransport.methodname to be a @property. This ensures that it's always lowercased when set, removing the need for so many lower() calls. --- lib/bridgedb/Filters.py | 12 ++++++------ lib/bridgedb/bridges.py | 29 ++++++++++++++++++++++++++--- lib/bridgedb/test/test_bridges.py | 10 +++++----- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/bridgedb/Filters.py b/lib/bridgedb/Filters.py index cb5e782..eb7db4e 100644 --- a/lib/bridgedb/Filters.py +++ b/lib/bridgedb/Filters.py @@ -71,19 +71,19 @@ def filterBridgesByTransport(methodname, addressClass=None): if not ((addressClass is IPv4Address) or (addressClass is IPv6Address)): addressClass = IPv4Address + # Ignore case + methodname = methodname.lower() + ruleset = frozenset([methodname, addressClass]) try: return funcs[ruleset] except KeyError: - def _filterByTransport(bridge): for transport in bridge.transports: - if isinstance(transport.address, addressClass): - # ignore method name case - if transport.methodname.lower() == methodname.lower(): - return True + if (transport.methodname == methodname and + isinstance(transport.address, addressClass)): + return True return False - _filterByTransport.__name__ = ("filterBridgesByTransport(%s,%s)" % (methodname, addressClass)) setattr(_filterByTransport, "description", "transport=%s" % methodname) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index dc5b78d..c98d54c 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -316,14 +316,13 @@ class PluggableTransport(BridgeAddressBase): super(PluggableTransport, self).__init__() self._port = None self._methodname = None - self._arguments = None + self._blockedIn = {} self.fingerprint = fingerprint self.address = address self.port = port self.methodname = methodname self.arguments = arguments - self._blockedIn = {} # Because we can intitialise this class with the __init__() # parameters, or use the ``updateFromStemTransport()`` method, we'll @@ -464,6 +463,30 @@ class PluggableTransport(BridgeAddressBase): """Reset this ``PluggableTransport``'s port to ``None``.""" self._port = None + @property + def methodname(self): + """Get this :class:`PluggableTransport`'s methodname. + + :rtype: str + :returns: The (lowercased) methodname of this ``PluggableTransport``, + i.e. ``"obfs3"``, ``"scramblesuit"``, etc. + """ + return self._methodname + + @methodname.setter + def methodname(self, value): + """Set this ``PluggableTransport``'s methodname. + + .. hint:: The **value** will be automatically lowercased. + + :param str value: The new methodname. + """ + if value: + try: + self._methodname = value.lower() + except (AttributeError, TypeError): + raise TypeError("methodname must be a str or unicode") + def getTransportLine(self, includeFingerprint=True, bridgePrefix=False): """Get a Bridge Line for this :class:`PluggableTransport`. @@ -1317,7 +1340,7 @@ class Bridge(BridgeBackwardsCompatibility): **countryCode**, ``False`` otherwise. """ for pt in self.transports: - if pt.methodname.lower() == methodname.lower(): + if pt.methodname == methodname.lower(): if self.addressIsBlockedIn(countryCode, pt.address, pt.port): logging.info("Transport %s of bridge %s is blocked in %s." % (pt.methodname, self, countryCode)) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index 499a8c2..db662ae 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -659,11 +659,11 @@ class PluggableTransportTests(unittest.TestCase): bridgeLine = pt.getTransportLine() # We have to check for substrings because we don't know which order - # the PT arguments will end up in the bridge line. Fortunately, the - # following three are the only ones which are important to have in - # order: - self.assertTrue(bridgeLine.startswith("voltronPT")) - self.assertSubstring("voltronPT 1.2.3.4:443 " + self.fingerprint, + # the PT arguments will end up in the bridge line. We also have to + # check for the lowercased transport name. Fortunately, the following + # three are the only ones which are important to have in order: + self.assertTrue(bridgeLine.startswith("voltronpt")) + self.assertSubstring("voltronpt 1.2.3.4:443 " + self.fingerprint, bridgeLine) # These ones can be in any order, but they should be at the end of the # bridge line:

1 0

[bridgedb/master] Rewrite testDistWithFilterAll() and testDistWithFilterBoth() in legacy_Tests.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 6357566edb74341f84cc3f953ed34bbb1ce01247 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 23:33:42 2015 +0000 Rewrite testDistWithFilterAll() and testDistWithFilterBoth() in legacy_Tests. --- lib/bridgedb/test/legacy_Tests.py | 35 --------------------------- lib/bridgedb/test/test_Dist.py | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index fc4ca5d..043f09b 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -268,41 +268,6 @@ class IPBridgeDistTests(unittest.TestCase): assert type(ipaddr.IPAddress(address)) is ipaddr.IPv4Address assert filterBridgesByIP4(random.choice(bridges)) - def testDistWithFilterBoth(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(50): - bridges = d.getBridges(randomIPv4String(), - "faketimestamp", - bridgeFilterRules=[ - filterBridgesByIP4, - filterBridgesByIP6]) - if bridges: - t = bridges.pop() - assert filterBridgesByIP4(t) - assert filterBridgesByIP6(t) - address, portlist = networkstatus.parseALine( - t.getConfigLine(addressClass=ipaddr.IPv4Address)) - assert type(address) is ipaddr.IPv4Address - address, portlist = networkstatus.parseALine( - t.getConfigLine(addressClass=ipaddr.IPv6Address)) - assert type(address) is ipaddr.IPv6Address - - - def testDistWithFilterAll(self): - d = bridgedb.Dist.HTTPSDistributor(3, "Foo") - for _ in xrange(250): - d.insert(fakeBridge6(or_addresses=True)) - d.insert(fakeBridge(or_addresses=True)) - - for i in xrange(5): - b = d.getBridges(randomIPv4String(), "x", bridgeFilterRules=[ - filterBridgesByIP4, filterBridgesByIP6]) - assert len(b) == 0 - def testDistWithFilterBlockedCountriesAdvanced(self): d = bridgedb.Dist.HTTPSDistributor(3, "Foo") for _ in xrange(250): diff --git a/lib/bridgedb/test/test_Dist.py b/lib/bridgedb/test/test_Dist.py index 2d93a19..9e47a92 100644 --- a/lib/bridgedb/test/test_Dist.py +++ b/lib/bridgedb/test/test_Dist.py @@ -24,6 +24,8 @@ from bridgedb.bridges import Bridge from bridgedb.bridges import PluggableTransport from bridgedb.Bridges import BridgeRing from bridgedb.Filters import filterBridgesByNotBlockedIn +from bridgedb.Filters import filterBridgesByIP4 +from bridgedb.Filters import filterBridgesByIP6 from bridgedb.https.request import HTTPSBridgeRequest from bridgedb.proxy import ProxySet from bridgedb.test.util import randomHighPort @@ -240,3 +242,49 @@ class HTTPSDistributorTests(unittest.TestCase): self.assertFalse(b.isBlockedIn('ir')) self.assertNotIn(b.fingerprint, blockedIR) self.assertGreater(len(bridges), 0) + + def test_HTTPSDistributor_getBridges_ipv4_ipv6(self): + """Asking for bridge addresses which are simultaneously IPv4 and IPv6 + (in that order) should return IPv4 bridges. + """ + dist = Dist.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv4() + bridgeRequest.filters.append(filterBridgesByIP6) + bridgeRequest.generateFilters() + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv4Address) + self.assertIsNotNone(filterBridgesByIP4(random.choice(bridges))) + + def test_HTTPSDistributor_getBridges_ipv6_ipv4(self): + """Asking for bridge addresses which are simultaneously IPv6 and IPv4 + (in that order) should return IPv6 bridges. + """ + dist = Dist.HTTPSDistributor(1, self.key) + [dist.insert(bridge) for bridge in self.bridges[:250]] + + bridgeRequest = self.randomClientRequest() + bridgeRequest.withIPv6() + bridgeRequest.generateFilters() + bridgeRequest.filters.append(filterBridgesByIP4) + + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + bridge = random.choice(bridges) + bridgeLine = bridge.getBridgeLine(bridgeRequest) + addrport, fingerprint = bridgeLine.split() + address, port = addrport.rsplit(':', 1) + address = address.strip('[]') + self.assertIsInstance(ipaddr.IPAddress(address), ipaddr.IPv6Address) + self.assertIsNotNone(filterBridgesByIP6(random.choice(bridges)))

1 0

[bridgedb/master] Add several additional tests for bridgedb.email.distributor.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit bb5b29dd19f32fb62d01a4d7ffb65b34493645e4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Apr 22 02:58:31 2015 +0000 Add several additional tests for bridgedb.email.distributor. --- lib/bridgedb/email/distributor.py | 12 +- lib/bridgedb/test/test_email_distributor.py | 177 ++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/email/distributor.py b/lib/bridgedb/email/distributor.py index b73c082..d8ea9bf 100644 --- a/lib/bridgedb/email/distributor.py +++ b/lib/bridgedb/email/distributor.py @@ -101,7 +101,7 @@ class EmailDistributor(Distributor): def bridgesPerResponse(self, hashring=None): return super(EmailDistributor, self).bridgesPerResponse(hashring) - def getBridges(self, bridgeRequest, interval): + def getBridges(self, bridgeRequest, interval, clock=None): """Return a list of bridges to give to a user. .. hint:: All checks on the email address (which should be stored in @@ -119,6 +119,13 @@ class EmailDistributor(Distributor): address. :param interval: The time period when we got this request. This can be any string, so long as it changes with every period. + :type clock: :api:`twisted.internet.task.Clock` + :param clock: If given, use the clock to ask what time it is, rather + than :api:`time.time`. This should likely only be used for + testing. + :rtype: list or ``None`` + :returns: A list of :class:`~bridgedb.bridges.Bridges` for the + ``bridgeRequest.client``, if allowed. Otherwise, returns ``None``. """ if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): raise addr.BadEmail( @@ -129,6 +136,9 @@ class EmailDistributor(Distributor): now = time.time() + if clock: + now = clock.seconds() + with bridgedb.Storage.getDB() as db: wasWarned = db.getWarnedEmail(bridgeRequest.client) lastSaw = db.getEmailTime(bridgeRequest.client) diff --git a/lib/bridgedb/test/test_email_distributor.py b/lib/bridgedb/test/test_email_distributor.py index 96eb306..b4d88b2 100644 --- a/lib/bridgedb/test/test_email_distributor.py +++ b/lib/bridgedb/test/test_email_distributor.py @@ -15,6 +15,7 @@ import logging import tempfile import os +from twisted.internet.task import Clock from twisted.trial import unittest import bridgedb.Storage @@ -24,6 +25,7 @@ from bridgedb.email.distributor import EmailDistributor from bridgedb.email.distributor import IgnoreEmail from bridgedb.email.distributor import TooSoonEmail from bridgedb.email.request import EmailBridgeRequest +from bridgedb.parse.addr import BadEmail from bridgedb.parse.addr import UnsupportedDomain from bridgedb.parse.addr import normalizeEmail from bridgedb.test.util import generateFakeBridges @@ -37,13 +39,14 @@ BRIDGES = generateFakeBridges() class EmailDistributorTests(unittest.TestCase): """Tests for :class:`bridgedb.email.distributor.EmailDistributor`.""" + # Fail any tests which take longer than 15 seconds. + timeout = 15 + def setUp(self): - self.fd, self.fname = tempfile.mkstemp() + self.fd, self.fname = tempfile.mkstemp(suffix=".sqlite", dir=os.getcwd()) bridgedb.Storage.initializeDBLock() self.db = bridgedb.Storage.openDatabase(self.fname) bridgedb.Storage.setDBFilename(self.fname) - self.cur = self.db.cursor() - self.db.close() self.bridges = BRIDGES self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' @@ -68,7 +71,68 @@ class EmailDistributorTests(unittest.TestCase): bridgeRequest.generateFilters() return bridgeRequest - def test_EmailDistributor_rate_limit(self): + def test_EmailDistributor_getBridges_default_client(self): + """If EmailBridgeRequest.client was not set, then getBridges() should + raise a bridgedb.parse.addr.BadEmail exception. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # The "default" client is literally the string "default", see + # bridgedb.bridgerequest.BridgeRequestBase. + bridgeRequest = self.makeClientRequest('default') + + self.assertRaises(BadEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_getBridges_with_whitelist(self): + """If an email address is in the whitelist, it should get a response + every time it asks (i.e. no rate-limiting). + """ + # The whitelist should be in the form {EMAIL: GPG_FINGERPRINT} + whitelist = {'white(a)list.ed': '0123456789ABCDEF0123456789ABCDEF01234567'} + dist = EmailDistributor(self.key, self.domainmap, self.domainrules, + whitelist=whitelist) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # A request from a whitelisted address should always get a response. + bridgeRequest = self.makeClientRequest('white(a)list.ed') + for i in range(5): + bridges = dist.getBridges(bridgeRequest, 1) + self.assertEqual(len(bridges), 3) + + def test_EmailDistributor_getBridges_rate_limit_multiple_clients(self): + """Each client should be rate-limited separately.""" + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest1 = self.makeClientRequest('abc(a)example.com') + bridgeRequest2 = self.makeClientRequest('def(a)example.com') + bridgeRequest3 = self.makeClientRequest('ghi(a)example.com') + + # The first request from 'abc' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest1, 1)), 3) + # The second from 'abc' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest1, 1) + # The first request from 'def' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest2, 1)), 3) + # The third from 'abc' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest1, 1) + # The second from 'def' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest2, 1) + # The third from 'def' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest2, 1) + # The fourth from 'abc' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest1, 1) + # The first request from 'ghi' should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest3, 1)), 3) + # The second from 'ghi' gets a warning. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest3, 1) + # The third from 'ghi' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest3, 1) + # The fourth from 'ghi' is ignored. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest3, 1) + + def test_EmailDistributor_getBridges_rate_limit(self): """A client's first email should return bridges. The second should return a warning, and the third should receive no response. """ @@ -77,16 +141,117 @@ class EmailDistributorTests(unittest.TestCase): bridgeRequest = self.makeClientRequest('abc(a)example.com') - # The first request should get a response with bridges + # The first request should get a response with bridges. bridges = dist.getBridges(bridgeRequest, 1) self.assertGreater(len(bridges), 0) [self.assertIsInstance(b, Bridge) for b in bridges] self.assertEqual(len(bridges), 3) - # The second gets a warning, and the third is ignored + # The second gets a warning, and the third is ignored. self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + def test_EmailDistributor_getBridges_rate_limit_expiry(self): + """A client's first email should return bridges. The second should + return a warning, and the third should receive no response. After the + EmailDistributor.emailRateMax is up, the client should be able to + receive a response again. + """ + clock = Clock() + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1, clock)), 3) + # The second gets a warning, and the rest are ignored. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1, clock) + + clock.advance(2 * dist.emailRateMax) + + # The client should again a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1)), 3, clock) + + def test_EmailDistributor_cleanDatabase(self): + """Calling cleanDatabase() should cleanup email times in database, but + not allow clients who have been recently warned and/or ignored to + receive a response again until the remainder of their MAX_EMAIL_RATE + time is up. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges. + self.assertEqual(len(dist.getBridges(bridgeRequest, 1)), 3) + # The second gets a warning, and the third is ignored. + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + dist.cleanDatabase() + + # Cleaning the warning email times in the database shouldn't cause + # 'abc(a)example.com' to be able to email again, because only the times + # which aren't older than EMAIL_MAX_RATE should be cleared. + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_prepopulateRings(self): + """Calling prepopulateRings() should add two rings to the + EmailDistributor.hashring. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + + # There shouldn't be any subrings yet. + self.assertEqual(len(dist.hashring.filterRings), 0) + + dist.prepopulateRings() + + # There should now be two subrings, but the subrings should be empty. + self.assertEqual(len(dist.hashring.filterRings), 2) + for (filtre, subring) in dist.hashring.filterRings.values(): + self.assertEqual(len(subring), 0) + + # The subrings in this Distributor have gross names, because the + # filter functions (including their addresses in memory!) are used as + # the subring names. In this case, we should have something like: + # + # frozenset([<function byIPv6 at 0x7eff7ad7fc80>]) + # + # and + # + # frozenset([<function byIPv4 at 0x7eff7ad7fc08>]) + # + # So we have to join the strings together and check the whole thing, + # since we have no other way to use these stupid subring names to + # index into the dictionary they are stored in, because the memory + # addresses are unknowable until runtime. + + # There should be an IPv4 subring and an IPv6 ring: + ringnames = dist.hashring.filterRings.keys() + self.failUnlessIn("IPv4", "".join([str(ringname) for ringname in ringnames])) + self.failUnlessIn("IPv6", "".join([str(ringname) for ringname in ringnames])) + + [dist.hashring.insert(bridge) for bridge in self.bridges] + + # There should still be two subrings. + self.assertEqual(len(dist.hashring.filterRings), 2) + for (filtre, subring) in dist.hashring.filterRings.values(): + self.assertGreater(len(subring), 0) + + # Ugh, the hashring code is so gross looking. + subrings = dist.hashring.filterRings + subring1 = subrings.values()[0][1] + subring2 = subrings.values()[1][1] + # Each subring should have roughly the same number of bridges. + # (Having ±10 bridges in either ring, out of 500 bridges total, should + # be so bad.) + self.assertApproximates(len(subring1), len(subring2), 10) + def test_EmailDistributor_unsupported_domain(self): """An unsupported domain should raise an UnsupportedDomain exception.""" self.assertRaises(UnsupportedDomain, normalizeEmail,

1 0

[bridgedb/master] Move bridgedb.Dist.EmailBasedDistributor → bridgedb.email.distributor.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 55f36f523dae951e2c0e57cdfdc84e3a78ba7cbe Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 21 21:41:20 2015 +0000 Move bridgedb.Dist.EmailBasedDistributor → bridgedb.email.distributor. --- doc/sphinx/source/bridgedb.Dist.rst | 8 - doc/sphinx/source/bridgedb.email.rst | 1 + doc/sphinx/source/bridgedb.rst | 1 - doc/sphinx/source/conf.py | 2 +- lib/bridgedb/Dist.py | 186 ---------------------- lib/bridgedb/Main.py | 9 +- lib/bridgedb/email/autoresponder.py | 8 +- lib/bridgedb/email/distributor.py | 209 +++++++++++++++++++++++++ lib/bridgedb/email/request.py | 6 +- lib/bridgedb/email/server.py | 4 +- lib/bridgedb/email/templates.py | 2 +- lib/bridgedb/parse/addr.py | 7 +- lib/bridgedb/persistent.py | 4 +- lib/bridgedb/test/email_helpers.py | 14 +- lib/bridgedb/test/legacy_Tests.py | 43 +---- lib/bridgedb/test/test_email_autoresponder.py | 2 +- lib/bridgedb/test/test_email_distributor.py | 93 +++++++++++ lib/bridgedb/test/test_email_server.py | 6 +- 18 files changed, 340 insertions(+), 265 deletions(-) diff --git a/doc/sphinx/source/bridgedb.Dist.rst b/doc/sphinx/source/bridgedb.Dist.rst deleted file mode 100644 index 995a7ac..0000000 --- a/doc/sphinx/source/bridgedb.Dist.rst +++ /dev/null @@ -1,8 +0,0 @@ -bridgedb.Dist ----------------- - -.. automodule:: bridgedb.Dist - :members: - :undoc-members: - :private-members: - :show-inheritance: diff --git a/doc/sphinx/source/bridgedb.email.rst b/doc/sphinx/source/bridgedb.email.rst index ba9b4a9..020b614 100644 --- a/doc/sphinx/source/bridgedb.email.rst +++ b/doc/sphinx/source/bridgedb.email.rst @@ -8,6 +8,7 @@ bridgedb.email .. automodule:: bridgedb.email.__init__ .. automodule:: bridgedb.email.autoresponder +.. automodule:: bridgedb.email.distributor .. automodule:: bridgedb.email.dkim .. automodule:: bridgedb.email.request .. automodule:: bridgedb.email.server diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 7b1fef7..207313c 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -13,7 +13,6 @@ BridgeDB Package and Module Documentation bridgedb.captcha bridgedb.configure bridgedb.crypto - bridgedb.Dist bridgedb.email bridgedb.filters bridgedb.geo diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 5038abb..cab614f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -33,9 +33,9 @@ import bridgedb.captcha import bridgedb.Bridges import bridgedb.Bucket import bridgedb.crypto -import bridgedb.Dist import bridgedb.email import bridgedb.email.autoresponder +import bridgedb.email.distributor import bridgedb.email.dkim import bridgedb.email.request import bridgedb.email.server diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py deleted file mode 100644 index 1be705f..0000000 --- a/lib/bridgedb/Dist.py +++ /dev/null @@ -1,186 +0,0 @@ -# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_Dist -*- -# -# This file is part of BridgeDB, a Tor bridge distribution system. -# -# :authors: Nick Mathewson -# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> -# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> -# :copyright: (c) 2013-2015, Isis Lovecruft -# (c) 2013-2015, Matthew Finkel -# (c) 2007-2015, The Tor Project, Inc. -# :license: 3-Clause BSD, see LICENSE for licensing information - -"""This module has functions to decide which bridges to hand out to whom.""" - -import logging -import time - -import bridgedb.Storage - -from bridgedb.Bridges import BridgeRing -from bridgedb.Bridges import FilteredBridgeSplitter -from bridgedb.crypto import getHMAC -from bridgedb.crypto import getHMACFunc -from bridgedb.distribute import Distributor -from bridgedb.filters import byFilters -from bridgedb.filters import byIPv4 -from bridgedb.filters import byIPv6 -from bridgedb.filters import bySubring -from bridgedb.parse import addr - - -MAX_EMAIL_RATE = 3*3600 - -class IgnoreEmail(addr.BadEmail): - """Raised when we get requests from this address after rate warning.""" - -class TooSoonEmail(addr.BadEmail): - """Raised when we got a request from this address too recently.""" - -class EmailRequestedHelp(Exception): - """Raised when a client has emailed requesting help.""" - -class EmailRequestedKey(Exception): - """Raised when an incoming email requested a copy of our GnuPG keys.""" - - -class EmailBasedDistributor(Distributor): - """Object that hands out bridges based on the email address of an incoming - request and the current time period. - - :type hashring: :class:`~bridgedb.Bridges.BridgeRing` - :ivar hashring: A hashring to hold all the bridges we hand out. - """ - - def __init__(self, key, domainmap, domainrules, - answerParameters=None, whitelist=None): - """Create a bridge distributor which uses email. - - :type emailHmac: callable - :param emailHmac: An hmac function used to order email addresses - within a ring. See :func:`~bridgedb.crypto.getHMACFunc`. - :param dict domainmap: A map from lowercase domains that we support - mail from to their canonical forms. See `EMAIL_DOMAIN_MAP` option - in `bridgedb.conf`. - :param domainrules: DOCDOC - :param answerParameters: DOCDOC - :type whitelist: dict or ``None`` - :param whitelist: A dictionary that maps whitelisted email addresses - to GnuPG fingerprints. - """ - super(EmailBasedDistributor, self).__init__(key) - - key1 = getHMAC(key, "Map-Addresses-To-Ring") - self.emailHmac = getHMACFunc(key1, hex=False) - - key2 = getHMAC(key, "Order-Bridges-In-Ring") - # XXXX clear the store when the period rolls over! - self.domainmap = domainmap - self.domainrules = domainrules - self.whitelist = whitelist or dict() - self.answerParameters = answerParameters - - #XXX cache options not implemented - self.hashring = FilteredBridgeSplitter(key2, max_cached_rings=5) - self.name = "Email" - - def bridgesPerResponse(self, hashring=None): - return super(EmailBasedDistributor, self).bridgesPerResponse(hashring) - - def insert(self, bridge): - """Assign a bridge to this distributor.""" - self.hashring.insert(bridge) - - def getBridges(self, bridgeRequest, interval): - """Return a list of bridges to give to a user. - - :type bridgeRequest: :class:`~bridgedb.email.request.EmailBridgeRequest` - :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` - with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` - attribute set to a string containing the client's full, canonicalized - email address. - :param interval: The time period when we got this request. This can be - any string, so long as it changes with every period. - """ - # All checks on the email address, such as checks for whitelisting and - # canonicalization of domain name, are done in - # :meth:`bridgedb.email.autoresponder.getMailTo` and - # :meth:`bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. - if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): - raise addr.BadEmail( - ("%s distributor can't get bridges for invalid email email " - " address: %s") % (self.name, bridgeRequest.client)) - - logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) - - now = time.time() - - with bridgedb.Storage.getDB() as db: - wasWarned = db.getWarnedEmail(bridgeRequest.client) - lastSaw = db.getEmailTime(bridgeRequest.client) - if lastSaw is not None: - if bridgeRequest.client in self.whitelist.keys(): - logging.info(("Whitelisted email address %s was last seen " - "%d seconds ago.") - % (bridgeRequest.client, now - lastSaw)) - elif (lastSaw + MAX_EMAIL_RATE) >= now: - wait = (lastSaw + MAX_EMAIL_RATE) - now - logging.info("Client %s must wait another %d seconds." - % (bridgeRequest.client, wait)) - if wasWarned: - raise IgnoreEmail("Client was warned.", - bridgeRequest.client) - else: - logging.info("Sending duplicate request warning.") - db.setWarnedEmail(bridgeRequest.client, True, now) - db.commit() - raise TooSoonEmail("Must wait %d seconds" % wait, - bridgeRequest.client) - # warning period is over - elif wasWarned: - db.setWarnedEmail(bridgeRequest.client, False) - - pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) - - ring = None - ruleset = frozenset(bridgeRequest.filters) - if ruleset in self.hashring.filterRings.keys(): - logging.debug("Cache hit %s" % ruleset) - _, ring = self.hashring.filterRings[ruleset] - else: - # cache miss, add new ring - logging.debug("Cache miss %s" % ruleset) - - # add new ring - key1 = getHMAC(self.key, "Order-Bridges-In-Ring") - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, byFilters(ruleset), - populate_from=self.hashring.bridges) - - returnNum = self.bridgesPerResponse(ring) - result = ring.getBridges(pos, returnNum) - - db.setEmailTime(bridgeRequest.client, now) - db.commit() - - return result - - def cleanDatabase(self): - with bridgedb.Storage.getDB() as db: - try: - db.cleanEmailedBridges(time.time() - MAX_EMAIL_RATE) - db.cleanWarnedEmails(time.time() - MAX_EMAIL_RATE) - except: - db.rollback() - raise - else: - db.commit() - - def prepopulateRings(self): - # populate all rings (for dumping assignments and testing) - for filterFn in [byIPv4, byIPv6]: - ruleset = frozenset([filterFn]) - key1 = getHMAC(self.key, "Order-Bridges-In-Ring") - ring = BridgeRing(key1, self.answerParameters) - self.hashring.addRing(ring, ruleset, byFilters([filterFn]), - populate_from=self.hashring.bridges) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 3a15145..b281d21 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -30,13 +30,13 @@ from bridgedb.bridges import ServerDescriptorDigestMismatch from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus from bridgedb.bridges import Bridge from bridgedb.configure import loadConfig +from bridgedb.email.distributor import EmailDistributor from bridgedb.https.distributor import HTTPSDistributor from bridgedb.parse import descriptors import bridgedb.Storage from bridgedb import Bridges -from bridgedb import Dist from bridgedb.Stability import updateBridgeHistory @@ -197,7 +197,7 @@ def createBridgeRings(cfg, proxyList, key): :rtype: tuple :returns: A BridgeSplitter hashring, an :class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an - EmailBasedDistributor or None. + :class:`~bridgedb.email.distributor.EmailDistributor` or None. """ # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -222,7 +222,7 @@ def createBridgeRings(cfg, proxyList, key): # As appropriate, create an email-based distributor. if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: logging.debug("Setting up Email Distributor...") - emailDistributor = Dist.EmailBasedDistributor( + emailDistributor = EmailDistributor( crypto.getHMAC(key, "Email-Dist-Key"), cfg.EMAIL_DOMAIN_MAP.copy(), cfg.EMAIL_DOMAIN_RULES.copy(), @@ -333,7 +333,8 @@ def run(options, reactor=reactor): known open proxies. :ivar ipDistributor: A :class:`~bridgedb.https.distributor.HTTPSDistributor`. - :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. + :ivar emailDistributor: A + :class:`~bridgedb.email.distributor.EmailDistributor`. :ivar dict tasks: A dictionary of ``{name: task}``, where name is a string to associate with the ``task``, and ``task`` is some scheduled event, repetitive or otherwise, for the :class:`reactor diff --git a/lib/bridgedb/email/autoresponder.py b/lib/bridgedb/email/autoresponder.py index b98717f..ad63bfd 100644 --- a/lib/bridgedb/email/autoresponder.py +++ b/lib/bridgedb/email/autoresponder.py @@ -47,13 +47,13 @@ from twisted.python import failure from bridgedb import safelog from bridgedb.crypto import NEW_BUFFER_INTERFACE -from bridgedb.Dist import EmailRequestedHelp -from bridgedb.Dist import EmailRequestedKey -from bridgedb.Dist import TooSoonEmail -from bridgedb.Dist import IgnoreEmail from bridgedb.email import dkim from bridgedb.email import request from bridgedb.email import templates +from bridgedb.email.distributor import EmailRequestedHelp +from bridgedb.email.distributor import EmailRequestedKey +from bridgedb.email.distributor import TooSoonEmail +from bridgedb.email.distributor import IgnoreEmail from bridgedb.parse import addr from bridgedb.parse.addr import canonicalizeEmailDomain from bridgedb.util import levenshteinDistance diff --git a/lib/bridgedb/email/distributor.py b/lib/bridgedb/email/distributor.py new file mode 100644 index 0000000..b73c082 --- /dev/null +++ b/lib/bridgedb/email/distributor.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_distributor -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Nick Mathewson +# Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# Matthew Finkel 0x017DD169EA793BE2 <sysrqb(a)torproject.org> +# :copyright: (c) 2013-2015, Isis Lovecruft +# (c) 2013-2015, Matthew Finkel +# (c) 2007-2015, The Tor Project, Inc. +# :license: see LICENSE for licensing information + +"""A :class:`~bridgedb.distribute.Distributor` which hands out +:class:`bridges <bridgedb.bridges.Bridge>` to clients via an email interface. +""" + +import logging +import time + +import bridgedb.Storage + +from bridgedb.Bridges import BridgeRing +from bridgedb.Bridges import FilteredBridgeSplitter +from bridgedb.crypto import getHMAC +from bridgedb.crypto import getHMACFunc +from bridgedb.distribute import Distributor +from bridgedb.filters import byFilters +from bridgedb.filters import byIPv4 +from bridgedb.filters import byIPv6 +from bridgedb.filters import bySubring +from bridgedb.parse import addr + + +#: The minimum amount of time (in seconds) which must pass before a client who +#: has previously been given an email response must wait before being eligible +#: to receive another response. +MAX_EMAIL_RATE = 3 * 3600 + + +class IgnoreEmail(addr.BadEmail): + """Raised when we get requests from this address after rate warning.""" + + +class TooSoonEmail(addr.BadEmail): + """Raised when we got a request from this address too recently.""" + + +class EmailRequestedHelp(Exception): + """Raised when a client has emailed requesting help.""" + + +class EmailRequestedKey(Exception): + """Raised when an incoming email requested a copy of our GnuPG keys.""" + + +class EmailDistributor(Distributor): + """Object that hands out bridges based on the email address of an incoming + request and the current time period. + + :type hashring: :class:`~bridgedb.Bridges.BridgeRing` + :ivar hashring: A hashring to hold all the bridges we hand out. + """ + + #: The minimum amount of time (in seconds) which must pass before a client + #: who has previously been given an email response must wait before being + #: eligible to receive another response. + emailRateMax = MAX_EMAIL_RATE + + def __init__(self, key, domainmap, domainrules, + answerParameters=None, whitelist=None): + """Create a bridge distributor which uses email. + + :type emailHmac: callable + :param emailHmac: An hmac function used to order email addresses + within a ring. See :func:`~bridgedb.crypto.getHMACFunc`. + :param dict domainmap: A map from lowercase domains that we support + mail from to their canonical forms. See `EMAIL_DOMAIN_MAP` option + in `bridgedb.conf`. + :param domainrules: DOCDOC + :param answerParameters: DOCDOC + :type whitelist: dict or ``None`` + :param whitelist: A dictionary that maps whitelisted email addresses + to GnuPG fingerprints. + """ + super(EmailDistributor, self).__init__(key) + + self.domainmap = domainmap + self.domainrules = domainrules + self.whitelist = whitelist or dict() + self.answerParameters = answerParameters + + key1 = getHMAC(key, "Map-Addresses-To-Ring") + key2 = getHMAC(key, "Order-Bridges-In-Ring") + + self.emailHmac = getHMACFunc(key1, hex=False) + #XXX cache options not implemented + self.hashring = FilteredBridgeSplitter(key2, max_cached_rings=5) + + self.name = "Email" + + def bridgesPerResponse(self, hashring=None): + return super(EmailDistributor, self).bridgesPerResponse(hashring) + + def getBridges(self, bridgeRequest, interval): + """Return a list of bridges to give to a user. + + .. hint:: All checks on the email address (which should be stored in + the ``bridgeRequest.client`` attribute), such as checks for + whitelisting and canonicalization of domain name, are done in + :meth:`bridgedb.email.autoresponder.getMailTo` and + :meth:`bridgedb.email.autoresponder.SMTPAutoresponder.runChecks`. + + :type bridgeRequest: + :class:`~bridgedb.email.request.EmailBridgeRequest` + :param bridgeRequest: A + :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the + :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute + set to a string containing the client's full, canonicalized email + address. + :param interval: The time period when we got this request. This can be + any string, so long as it changes with every period. + """ + if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): + raise addr.BadEmail( + ("%s distributor can't get bridges for invalid email address: " + "%s") % (self.name, bridgeRequest.client), bridgeRequest.client) + + logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) + + now = time.time() + + with bridgedb.Storage.getDB() as db: + wasWarned = db.getWarnedEmail(bridgeRequest.client) + lastSaw = db.getEmailTime(bridgeRequest.client) + if lastSaw is not None: + if bridgeRequest.client in self.whitelist: + logging.info( + "Whitelisted address %s was last seen %d seconds ago." + % (bridgeRequest.client, now - lastSaw)) + elif (lastSaw + self.emailRateMax) >= now: + wait = (lastSaw + self.emailRateMax) - now + logging.info("Client %s must wait another %d seconds." + % (bridgeRequest.client, wait)) + if wasWarned: + raise IgnoreEmail( + "Client %s was warned." % bridgeRequest.client, + bridgeRequest.client) + else: + logging.info("Sending duplicate request warning.") + db.setWarnedEmail(bridgeRequest.client, True, now) + db.commit() + raise TooSoonEmail("Must wait %d seconds" % wait, + bridgeRequest.client) + # warning period is over + elif wasWarned: + db.setWarnedEmail(bridgeRequest.client, False) + + pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) + + ring = None + filtres = frozenset(bridgeRequest.filters) + if filtres in self.hashring.filterRings: + logging.debug("Cache hit %s" % filtres) + _, ring = self.hashring.filterRings[filtres] + else: + logging.debug("Cache miss %s" % filtres) + key = getHMAC(self.key, "Order-Bridges-In-Ring") + ring = BridgeRing(key, self.answerParameters) + self.hashring.addRing(ring, filtres, byFilters(filtres), + populate_from=self.hashring.bridges) + + returnNum = self.bridgesPerResponse(ring) + result = ring.getBridges(pos, returnNum) + + db.setEmailTime(bridgeRequest.client, now) + db.commit() + + return result + + def cleanDatabase(self): + """Clear all emailed response and warning times from the database.""" + logging.info(("Cleaning all response and warning times for the %s " + "distributor from the database...") % self.name) + with bridgedb.Storage.getDB() as db: + try: + db.cleanEmailedBridges(time.time() - self.emailRateMax) + db.cleanWarnedEmails(time.time() - self.emailRateMax) + except: + db.rollback() + raise + else: + db.commit() + + def prepopulateRings(self): + """Prepopulate this distributor's hashrings and subhashrings with + bridges. + """ + logging.info("Prepopulating %s distributor hashrings..." % self.name) + + for filterFn in [byIPv4, byIPv6]: + ruleset = frozenset([filterFn]) + key = getHMAC(self.key, "Order-Bridges-In-Ring") + ring = BridgeRing(key, self.answerParameters) + self.hashring.addRing(ring, ruleset, byFilters([filterFn]), + populate_from=self.hashring.bridges) + + # Since prepopulateRings is called every half hour when the bridge + # descriptors are re-parsed, we should clean the database then. + self.cleanDatabase() diff --git a/lib/bridgedb/email/request.py b/lib/bridgedb/email/request.py index 32446ac..50fd32c 100644 --- a/lib/bridgedb/email/request.py +++ b/lib/bridgedb/email/request.py @@ -40,8 +40,8 @@ import logging import re from bridgedb import bridgerequest -from bridgedb.Dist import EmailRequestedHelp -from bridgedb.Dist import EmailRequestedKey +from bridgedb.email.distributor import EmailRequestedHelp +from bridgedb.email.distributor import EmailRequestedKey #: A regular expression for matching the Pluggable Transport method TYPE in @@ -105,7 +105,7 @@ class EmailBridgeRequest(bridgerequest.BridgeRequestBase): def __init__(self): """Process a new bridge request received through the - :class:`~bridgedb.Dist.EmailBasedDistributor`. + :class:`~bridgedb.email.distributor.EmailDistributor`. """ super(EmailBridgeRequest, self).__init__() self._wantsKey = False diff --git a/lib/bridgedb/email/server.py b/lib/bridgedb/email/server.py index 8034d35..736c3f6 100644 --- a/lib/bridgedb/email/server.py +++ b/lib/bridgedb/email/server.py @@ -103,7 +103,7 @@ class MailServerContext(object): """Create a context for storing configs for email bridge distribution. :type config: :class:`bridgedb.persistent.Conf` - :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor` + :type distributor: :class:`~bridgedb.email.distributor.EmailDistributor` :param distributor: The distributor will handle getting the correct bridges (or none) for a client for us. :type schedule: :class:`bridgedb.schedule.ScheduledInterval` @@ -466,7 +466,7 @@ def addServer(config, distributor): :type config: :class:`bridgedb.configure.Conf` :param config: A configuration object. - :type distributor: :class:`bridgedb.Dist.EmailBasedDistributor` + :type distributor: :class:`bridgedb.email.distributor.EmailDistributor` :param dist: A distributor which will handle database interactions, and will decide which bridges to give to who and when. """ diff --git a/lib/bridgedb/email/templates.py b/lib/bridgedb/email/templates.py index 4db4e3c..6998716 100644 --- a/lib/bridgedb/email/templates.py +++ b/lib/bridgedb/email/templates.py @@ -30,7 +30,7 @@ import os from datetime import datetime from bridgedb import strings -from bridgedb.Dist import MAX_EMAIL_RATE +from bridgedb.email.distributor import MAX_EMAIL_RATE def addCommands(template): diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py index b3ad680..cd512d1 100644 --- a/lib/bridgedb/parse/addr.py +++ b/lib/bridgedb/parse/addr.py @@ -217,7 +217,7 @@ def canonicalizeEmailDomain(domain, domainmap): :param str domain: The domain portion of an email address to validate. It will be checked that it is one of the domains allowed to email requests for bridges to the - :class:`~bridgedb.Dist.EmailBasedDistributor`. + :class:`~bridgedb.email.distributor.EmailDistributor`. :param dict domainmap: A map of permitted alternate domains (in lowercase) to their canonical domain names (in lowercase). This can be configured with the ``EMAIL_DOMAIN_MAP`` option in ``bridgedb.conf``, for @@ -441,8 +441,9 @@ def normalizeEmail(emailaddr, domainmap, domainrules, ignorePlus=True): The email address, **emailaddr**, will be parsed and validated, and then checked that it originated from one of the domains allowed to email - requests for bridges to the :class:`~bridgedb.Dist.EmailBasedDistributor` - via the :func:`canonicaliseEmailDomain` function. + requests for bridges to the + :class:`~bridgedb.email.distributor.EmailDistributor` via the + :func:`canonicaliseEmailDomain` function. :param str emailaddr: An email address to normalise. :param dict domainmap: A map of permitted alternate domains (in lowercase) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 0853c1e..22673dd 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -24,8 +24,8 @@ from twisted.python.reflect import safe_repr from twisted.spread import jelly from bridgedb import Bridges -from bridgedb import Dist from bridgedb import filters +from bridgedb.email import distributor as emailDistributor from bridgedb.https import distributor as httpsDistributor from bridgedb.configure import Conf #from bridgedb.proxy import ProxySet @@ -35,7 +35,7 @@ _state = None #: Types and classes which are allowed to be jellied: _security = jelly.SecurityOptions() #_security.allowInstancesOf(ProxySet) -_security.allowModules(filters, Bridges, Dist, httpsDistributor) +_security.allowModules(filters, Bridges, emailDistributor, httpsDistributor) class MissingState(Exception): diff --git a/lib/bridgedb/test/email_helpers.py b/lib/bridgedb/test/email_helpers.py index 20aee57..14c86f4 100644 --- a/lib/bridgedb/test/email_helpers.py +++ b/lib/bridgedb/test/email_helpers.py @@ -13,9 +13,9 @@ import io -from bridgedb.Dist import IgnoreEmail -from bridgedb.Dist import TooSoonEmail from bridgedb.persistent import Conf +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.email.server import MailServerContext from bridgedb.schedule import Unscheduled from bridgedb.test import util @@ -120,8 +120,8 @@ def _createMailServerContext(config=None, distributor=None): class DummyEmailDistributor(object): - """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which is used to - test :class:`bridgedb.EmailServer`. + """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which is used + to test :class:`bridgedb.EmailServer`. """ _bridgesPerResponseMin = 3 @@ -144,9 +144,9 @@ class DummyEmailDistributor(object): class DummyEmailDistributorWithState(DummyEmailDistributor): - """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which raises - :exc:`bridgedb.Dist.TooSoonEmail` on the second email and - :exc:`bridgedb.Dist.IgnoreEmail` on the third. + """A mocked :class:`bridgedb.email.distributor.EmailDistributor` which raises + :exc:`bridgedb.email.distributor.TooSoonEmail` on the second email and + :exc:`bridgedb.email.distributor.IgnoreEmail` on the third. Note that the state tracking is done in a really dumb way. For example, we currently don't consider requests for help text or GnuPG keys to be a diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index c0a9ff0..40f7ae4 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -19,7 +19,6 @@ from datetime import datetime import bridgedb.Bridges import bridgedb.Main -import bridgedb.Dist import bridgedb.schedule import bridgedb.Storage import re @@ -27,6 +26,9 @@ import ipaddr from bridgedb.Stability import BridgeHistory +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse import addr from bridgedb.test.util import bracketIPv6 from bridgedb.test.util import randomIP @@ -94,42 +96,6 @@ def fakeBridge6(orport=8080, running=True, stable=True, or_addresses=False, return b -class EmailBridgeDistTests(unittest.TestCase): - def setUp(self): - self.fd, self.fname = tempfile.mkstemp() - self.db = bridgedb.Storage.Database(self.fname) - bridgedb.Storage.setDB(self.db) - self.cur = self.db._conn.cursor() - - def tearDown(self): - self.db.close() - os.close(self.fd) - os.unlink(self.fname) - - def testEmailRateLimit(self): - db = self.db - EMAIL_DOMAIN_MAP = {'example.com':'example.com'} - d = bridgedb.Dist.EmailBasedDistributor( - "Foo", - {'example.com': 'example.com', - 'dkim.example.com': 'dkim.example.com'}, - {'example.com': [], 'dkim.example.com': ['dkim']}) - for _ in xrange(256): - d.insert(fakeBridge()) - d.getBridges('abc(a)example.com', 1) - self.assertRaises(bridgedb.Dist.TooSoonEmail, - d.getBridges, 'abc(a)example.com', 1) - self.assertRaises(bridgedb.Dist.IgnoreEmail, - d.getBridges, 'abc(a)example.com', 1) - - def testUnsupportedDomain(self): - db = self.db - self.assertRaises(bridgedb.parse.addr.UnsupportedDomain, - bridgedb.parse.addr.normalizeEmail, - 'bad(a)email.com', - {'example.com':'example.com'}, - {'example.com':[]}) - class SQLStorageTests(unittest.TestCase): def setUp(self): @@ -358,8 +324,7 @@ class BridgeStabilityTests(unittest.TestCase): def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - - for klass in [SQLStorageTests, EmailBridgeDistTests, BridgeStabilityTests]: + for klass in [SQLStorageTests, BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) return suite diff --git a/lib/bridgedb/test/test_email_autoresponder.py b/lib/bridgedb/test/test_email_autoresponder.py index 2895802..98302ca 100644 --- a/lib/bridgedb/test/test_email_autoresponder.py +++ b/lib/bridgedb/test/test_email_autoresponder.py @@ -25,7 +25,7 @@ from twisted.test import proto_helpers from bridgedb.email import autoresponder from bridgedb.email.server import SMTPMessage -from bridgedb.Dist import TooSoonEmail +from bridgedb.email.distributor import TooSoonEmail from bridgedb.test.email_helpers import _createConfig from bridgedb.test.email_helpers import _createMailServerContext from bridgedb.test.email_helpers import DummyEmailDistributorWithState diff --git a/lib/bridgedb/test/test_email_distributor.py b/lib/bridgedb/test/test_email_distributor.py new file mode 100644 index 0000000..96eb306 --- /dev/null +++ b/lib/bridgedb/test/test_email_distributor.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2015 Isis Lovecruft +# (c) 2007-2015, The Tor Project, Inc. +# :license: see included LICENSE for information + +"""Tests for :mod:`bridgedb.email.distributor`.""" + +from __future__ import print_function + +import logging +import tempfile +import os + +from twisted.trial import unittest + +import bridgedb.Storage + +from bridgedb.bridges import Bridge +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import IgnoreEmail +from bridgedb.email.distributor import TooSoonEmail +from bridgedb.email.request import EmailBridgeRequest +from bridgedb.parse.addr import UnsupportedDomain +from bridgedb.parse.addr import normalizeEmail +from bridgedb.test.util import generateFakeBridges + +logging.disable(50) + + +BRIDGES = generateFakeBridges() + + +class EmailDistributorTests(unittest.TestCase): + """Tests for :class:`bridgedb.email.distributor.EmailDistributor`.""" + + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + bridgedb.Storage.initializeDBLock() + self.db = bridgedb.Storage.openDatabase(self.fname) + bridgedb.Storage.setDBFilename(self.fname) + self.cur = self.db.cursor() + self.db.close() + + self.bridges = BRIDGES + self.key = 'aQpeOFIj8q20s98awfoiq23rpOIjFaqpEWFoij1X' + self.domainmap = { + 'example.com': 'example.com', + 'dkim.example.com': 'dkim.example.com', + } + self.domainrules = { + 'example.com': ['ignore_dots'], + 'dkim.example.com': ['dkim', 'ignore_dots'] + } + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def makeClientRequest(self, clientEmailAddress): + bridgeRequest = EmailBridgeRequest() + bridgeRequest.client = clientEmailAddress + bridgeRequest.isValid(True) + bridgeRequest.generateFilters() + return bridgeRequest + + def test_EmailDistributor_rate_limit(self): + """A client's first email should return bridges. The second should + return a warning, and the third should receive no response. + """ + dist = EmailDistributor(self.key, self.domainmap, self.domainrules) + [dist.hashring.insert(bridge) for bridge in self.bridges] + + bridgeRequest = self.makeClientRequest('abc(a)example.com') + + # The first request should get a response with bridges + bridges = dist.getBridges(bridgeRequest, 1) + self.assertGreater(len(bridges), 0) + [self.assertIsInstance(b, Bridge) for b in bridges] + self.assertEqual(len(bridges), 3) + + # The second gets a warning, and the third is ignored + self.assertRaises(TooSoonEmail, dist.getBridges, bridgeRequest, 1) + self.assertRaises(IgnoreEmail, dist.getBridges, bridgeRequest, 1) + + def test_EmailDistributor_unsupported_domain(self): + """An unsupported domain should raise an UnsupportedDomain exception.""" + self.assertRaises(UnsupportedDomain, normalizeEmail, + 'bad(a)email.com', self.domainmap, self.domainrules) diff --git a/lib/bridgedb/test/test_email_server.py b/lib/bridgedb/test/test_email_server.py index 561c14c..9c4fabb 100644 --- a/lib/bridgedb/test/test_email_server.py +++ b/lib/bridgedb/test/test_email_server.py @@ -31,9 +31,9 @@ from twisted.trial import unittest from zope.interface import implementedBy -from bridgedb.Dist import EmailBasedDistributor -from bridgedb.Dist import TooSoonEmail from bridgedb.email import server +from bridgedb.email.distributor import EmailDistributor +from bridgedb.email.distributor import TooSoonEmail from bridgedb.parse.addr import BadEmail from bridgedb.schedule import Unscheduled from bridgedb.test import util @@ -504,7 +504,7 @@ class EmailServerServiceTests(SMTPTestCaseMixin, unittest.TestCase): """Unittests for :func:`bridgedb.email.server.addServer`.""" def setUp(self): - """Create a server.MailServerContext and EmailBasedDistributor.""" + """Create a MailServerContext and EmailDistributor.""" self.config = _createConfig() self.context = _createMailServerContext(self.config) self.smtpFromAddr = self.context.smtpFromAddr # 'bridges@localhost'

1 0

[bridgedb/master] PEP8 remove extra blank line in bridgedb.bridges module.
by isis＠torproject.org 25 Jul '15

25 Jul '15

commit 3c990f7cd86b7f2ca8799ee4b7054b5e0167620e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue May 12 07:28:26 2015 +0000 PEP8 remove extra blank line in bridgedb.bridges module. --- lib/bridgedb/bridges.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index c98d54c..f6e1d6b 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -1104,7 +1104,6 @@ class Bridge(BridgeBackwardsCompatibility): ("Client requested transport %s, but bridge %s doesn't " "have any of that transport!") % (desired, self)) - unblocked = [] for pt in transports: if not sum([self.transportIsBlockedIn(cc, pt.methodname)

1 0