August 2014 - tor-commits - lists.torproject.org

[bridgedb/develop] Sphinxify a docstring in test_https.py.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit f1d97e94993de6e4170780b4761b0a674fa835c0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 02:34:44 2014 +0000 Sphinxify a docstring in test_https.py. --- lib/bridgedb/test/test_https.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index f06f6d3..657f516 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -106,10 +106,11 @@ class HTTPTests(unittest.TestCase): return soup def getBridgeLinesFromSoup(self, soup, fieldsPerBridge): - # Now we're looking for something like this in the response: - # <div class="bridge-lines"> - # obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd - # </div> + """We're looking for something like this in the response:: + <div class="bridge-lines"> + obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd + </div> + """ bridges = [] soup = soup.findAll(attrs={'class' : 'bridge-lines'}) self.assertTrue(soup, "Could not find <div class='bridge-lines'>!")

1 0

[bridgedb/develop] Add unittests for bridgedb.bridgerequest.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit 1e7e8324ddb113920348990172e194440f322f55 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 02:42:14 2014 +0000 Add unittests for bridgedb.bridgerequest. --- lib/bridgedb/test/test_bridgerequest.py | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/lib/bridgedb/test/test_bridgerequest.py b/lib/bridgedb/test/test_bridgerequest.py new file mode 100644 index 0000000..1db8814 --- /dev/null +++ b/lib/bridgedb/test/test_bridgerequest.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + + +from twisted.trial import unittest + +from bridgedb import bridgerequest +from bridgedb.bridgerequest import IBridgeRequest +from bridgedb.bridgerequest import BridgeRequestBase + + +class BridgeRequestBaseTests(unittest.TestCase): + """Unittests for :class:`bridgedb.bridgerequest.BridgeRequestBase`.""" + + def setUp(self): + """Setup test run.""" + self.request = BridgeRequestBase() + + def test_BridgeRequestBase_implements_IRequestBridges(self): + """BridgeRequestBase should implement IRequestBridges interface.""" + self.assertTrue(IBridgeRequest.implementedBy(BridgeRequestBase)) + + def test_BridgeRequestBase_withoutBlockInCountry(self): + """BridgeRequestBase.withoutBlockInCountry() should add the country CC + to the ``notBlockedIn`` attribute. + """ + self.request.withoutBlockInCountry('US') + self.assertIn('US', self.request.notBlockedIn) + + def test_BridgeRequestBase_withPluggableTransportType(self): + """BridgeRequestBase.withPluggableTransportType() should add the + pluggable transport type to the ``transport`` attribute. + """ + self.request.withPluggableTransportType('huggable-transport') + self.assertIn('huggable-transport', self.request.transports)

1 0

[bridgedb/develop] Fix old-style classes in bridgedb.Bucket.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit eecabb0e5054ad619fa8591bfc067cc38ed04cc5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:43:12 2014 +0000 Fix old-style classes in bridgedb.Bucket. --- lib/bridgedb/Bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index fcc3794..b5054ac 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -37,7 +37,7 @@ toHex = binascii.b2a_hex # distinguish them from real distributors? PSEUDO_DISTRI_PREFIX = "pseudo_" -class BucketData: +class BucketData(object): """A file bucket value class. name - Name of the bucket (From config), prefixed by pseudo distributor prefix @@ -52,7 +52,7 @@ class BucketData: self.needed = int(needed) self.allocated = 0 -class BucketManager: +class BucketManager(object): """BucketManager reads a number of file bucket identifiers from the config. They're expected to be in the following format:

1 0

[bridgedb/develop] Fix another typo in test_Bucket.py.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit 2f0ea18d2379fc8b67071adc37d9412c8eb77ced Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:56:35 2014 +0000 Fix another typo in test_Bucket.py. --- lib/bridgedb/test/test_Bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index cff8cb8..2912ae1 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -33,7 +33,7 @@ class BucketDataTest(unittest.TestCase): """Set the needed number of bridges""" needed = 10 distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) + bucket = Bucket.BucketData(distname, needed) this(bucket.name).should.be.equal(distname) this(bucket.needed).should.be.equal(needed) @@ -41,7 +41,7 @@ class BucketDataTest(unittest.TestCase): """Set the needed number of bridges to the default""" needed = '*' distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) + bucket = Bucket.BucketData(distname, needed) this(bucket.name).should.be.equal(distname) this(bucket.needed).should.be.equal(needed)

1 0

[bridgedb/develop] Sphinxify all docstrings in bridgedb.Bucket.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit aa67615395a4a3d29e855a6e510ddc1a276dbed7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:54:43 2014 +0000 Sphinxify all docstrings in bridgedb.Bucket. --- lib/bridgedb/Bucket.py | 134 +++++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index b5054ac..5b33a36 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -38,11 +38,15 @@ toHex = binascii.b2a_hex PSEUDO_DISTRI_PREFIX = "pseudo_" class BucketData(object): - """A file bucket value class. - name - Name of the bucket (From config), prefixed by pseudo - distributor prefix - needed - Needed number of bridges for that bucket (From config) - allocated - Number of already allocated bridges for that bucket + """Configures a bridge bucket with the number of bridges which should be + allocated, the name of the bucket, and other similar data. + + :param str name: The name of this bucket (from the config file). This will + be prefixed by the :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :type needed: str or int + :param needed: The number of bridges needed for this bucket (also from the + config file). + :param int allocated: Number of bridges already allocated for this bucket. """ def __init__(self, name, needed): self.name = name @@ -54,57 +58,61 @@ class BucketData(object): class BucketManager(object): """BucketManager reads a number of file bucket identifiers from the config. - They're expected to be in the following format: - - FILE_BUCKETS = { "name1": 10, "name2": 15, "foobar": 3 } - - This syntax means that certain buckets ("name1", "name2" and so on) - are given a number of bridges (10, 15 and so on). Names can be anything. - The name will later be the prefix of the file that is written with the - assigned number of bridges in it. Instead of a number, a wildcard item - ("*") is allowed, too. This means that the corresponsing bucket file - will get the maximum number of possible bridges (as many as are left in - the unallocated bucket). - - The files will be written in ip:port format, one bridge per line. - - The way this works internally is as follows: - - First of all, the assignBridgesToBuckets() routine runs through - the database of bridges and looks up the 'distributor' field of each - bridge. Unallocated bridges are sent to a pool for later assignement. - Already allocated bridges for file bucket distribution are sorted and - checked. - They're checked for whether their bucket identifier still exists in the - current config and also whether the number of assigned bridges is still - valid. If either the bucket identifier is not existing anymore or too - many bridges are currently assigned to it, bridges will go to the - unassigned pool. - - In the second step, after bridges are sorted and the unassigned pool is - ready, the assignBridgesToBuckets() routine assigns one bridge - from the unassigned pool to a known bucket identifier at a time until it - either runs out of bridges in the unallocated pool or the number of - needed bridges for that bucket is reached. - - When all bridges are assigned in this way, they can then be dumped into - files by calling the dumpBridges() routine. - - cfg - The central configuration instance - bucketList - A list of BucketData instances, holding all - configured (and thus requested) buckets with - their respective numbers - unallocatedList - Holding all bridges from the 'unallocated' - pool - unallocated_available - Is at least one unallocated bridge - available? - distributor_prefix - The 'distributor' field in the database will - hold the name of our pseudo-distributor, - prefixed by this - db - The bridge database access instance + + They're expected to be in the following format:: + + FILE_BUCKETS = { "name1": 10, "name2": 15, "foobar": 3 } + + This syntax means that certain buckets ("name1", "name2" and so on) are + given a number of bridges (10, 15 and so on). Names can be anything. The + name will later be the prefix of the file that is written with the + assigned number of bridges in it. Instead of a number, a wildcard item + ("*") is allowed, too. This means that the corresponsing bucket file will + get the maximum number of possible bridges (as many as are left in the + unallocated bucket). + + The files will be written in ip:port format, one bridge per line. + + The way this works internally is as follows: + + First of all, the assignBridgesToBuckets() routine runs through the + database of bridges and looks up the 'distributor' field of each + bridge. Unallocated bridges are sent to a pool for later assignement. + Already allocated bridges for file bucket distribution are sorted and + checked. They're checked for whether their bucket identifier still exists + in the current config and also whether the number of assigned bridges is + still valid. If either the bucket identifier is not existing anymore or + too many bridges are currently assigned to it, bridges will go to the + unassigned pool. + + In the second step, after bridges are sorted and the unassigned pool is + ready, the assignBridgesToBuckets() routine assigns one bridge from the + unassigned pool to a known bucket identifier at a time until it either + runs out of bridges in the unallocated pool or the number of needed + bridges for that bucket is reached. + + When all bridges are assigned in this way, they can then be dumped into + files by calling the dumpBridges() routine. + + :type cfg: :class:`bridgedb.persistent.Conf` + :ivar cfg: The central configuration instance. + :ivar list bucketList: A list of BucketData instances, holding all + configured (and thus requested) buckets with their respective numbers. + :ivar list unallocatedList: Holds all bridges from the 'unallocated' pool. + :ivar bool unallocated_available: Is at least one unallocated bridge + available? + :ivar str distributor_prefix: The 'distributor' field in the database will + hold the name of our pseudo-distributor, prefixed by this string. By + default, this uses :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :ivar db: The bridge database instance. """ def __init__(self, cfg): + """Create a ``BucketManager``. + + :type cfg: :class:`bridgedb.persistent.Conf` + :param cfg: The central configuration instance. + """ self.cfg = cfg self.bucketList = [] self.unallocatedList = [] @@ -112,8 +120,7 @@ class BucketManager(object): self.distributor_prefix = PSEUDO_DISTRI_PREFIX def addToUnallocatedList(self, hex_key): - """Add a bridge by hex_key into the unallocated pool - """ + """Add a bridge by **hex_key** into the unallocated pool.""" with bridgedb.Storage.getDB() as db: try: db.updateDistributorForHexKey("unallocated", hex_key) @@ -126,8 +133,8 @@ class BucketManager(object): self.unallocated_available = True def getBucketByIdent(self, bucketIdent): - """Do we know this bucket identifier? If yes, return the corresponding - BucketData object. + """If we know this bucket identifier, then return the corresponding + :class:`BucketData` object. """ for d in self.bucketList: if d.name == bucketIdent: @@ -135,8 +142,7 @@ class BucketManager(object): return None def assignUnallocatedBridge(self, bucket): - """Assign an unallocated bridge to a certain bucket - """ + """Assign an unallocated bridge to a certain **bucket**.""" hex_key = self.unallocatedList.pop() # Mark pseudo-allocators in the database as such allocator_name = bucket.name @@ -163,8 +169,8 @@ class BucketManager(object): return True def assignBridgesToBuckets(self): - """Read file bucket identifiers from the configuration, sort them and - write necessary changes to the database + """Read file bucket identifiers from the configuration, sort them, and + write necessary changes to the database. """ logging.debug("Assigning bridges to buckets for pseudo-distributors") # Build distributor list @@ -221,8 +227,7 @@ class BucketManager(object): self.bucketList.remove(d) def dumpBridgesToFile(self, filename, bridges): - """Dump a list of given bridges into a file - """ + """Dump a list of given **bridges** into **filename**.""" logging.debug("Dumping bridge assignments to file: %r" % filename) # get the bridge histories and sort by Time On Same Address bridgeHistories = [] @@ -263,8 +268,7 @@ class BucketManager(object): print "I/O error: %s" % filename def dumpBridges(self): - """Dump all known file distributors to files, sort by distributor - """ + """Dump all known file distributors to files, sorted by distributor.""" logging.info("Dumping all distributors to file.") with bridgedb.Storage.getDB() as db: allBridges = db.getAllBridges()

1 0

[bridgedb/develop] Constify the unrealistic maximum number of bridges per bucket
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit b778f85304ace66d1b5dbdcacc781420a71d8201 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 05:51:08 2014 +0000 Constify the unrealistic maximum number of bridges per bucket We will never, ever have 1 million bridges, but because of this we will also never be able to satisfy the requirement of giving a bucket one million bridges. So, when we try distributing the bridges evenly between buckets, the pseudo-distributors that are defined as needing '*' bridges will receive as many as are available to be allocated to them. --- lib/bridgedb/Bucket.py | 6 ++++-- lib/bridgedb/test/test_Bucket.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index a7d3706..e603fe0 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -37,6 +37,9 @@ toHex = binascii.b2a_hex # distinguish them from real distributors? PSEUDO_DISTRI_PREFIX = "pseudo_" +# Set to rediculously high number +BUCKET_MAX_BRIDGES = 1000000 + class BucketData: """A file bucket value class. name - Name of the bucket (From config), prefixed by pseudo @@ -47,8 +50,7 @@ class BucketData: def __init__(self, name, needed): self.name = name if needed == "*": - # Set to rediculously high number - needed = 1000000 + needed = BUCKET_MAX_BRIDGES self.needed = int(needed) self.allocated = 0 diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 7cadd7b..8893dcd 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -39,7 +39,7 @@ class BucketDataTest(unittest.TestCase): distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(1000000) + this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) class BucketManagerTest(unittest.TestCase):

1 0

[bridgedb/develop] Add missing unittest import
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit 0ea71dd97c292063247c25943ed0c4bc11bc93ac Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 06:15:55 2014 +0000 Add missing unittest import --- lib/bridgedb/test/test_Bucket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 8893dcd..82424b6 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -19,6 +19,8 @@ from sure import this from sure import the from sure import expect +from twisted.trial import unittest + from bridgedb import Bucket @@ -31,7 +33,7 @@ class BucketDataTest(unittest.TestCase): distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(bucket.needed) + this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default"""

1 0

[bridgedb/develop] Line wasn't indented in f5fe05ce
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit deada7243f5f03289cda044c187e535f505dc9a5 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 06:27:41 2014 +0000 Line wasn't indented in f5fe05ce --- lib/bridgedb/Bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index e603fe0..40a8a32 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -254,7 +254,7 @@ class BucketManager: if str(bh.fingerprint) in blocklist.keys(): line = line + "\t(Might be blocked): (%s)" % \ ",".join(blocklist[bh.fingerprint]) - f.write(line + '\n') + f.write(line + '\n') else: for bridge in bridges: line = "%s:%d %s" \

1 0

[bridgedb/develop] Fix yet another type error in test_Bucket.py.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit 7c6f68326a920ca6ee9f3089acc493690af303a2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 07:23:01 2014 +0000 Fix yet another type error in test_Bucket.py. --- lib/bridgedb/test/test_Bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 5e39260..fa3bb7e 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -35,7 +35,7 @@ class BucketDataTest(unittest.TestCase): distname = "test-distributor" bucket = Bucket.BucketData(distname, needed) this(bucket.name).should.be.equal(distname) - this(bucket.needed).should.be.equal(Bucket.BUCKET_MAX_BRIDGES) + this(bucket.needed).should.be.equal(needed) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default"""

1 0

[bridgedb/develop] Be more careful about how we split bridge lines in test_https.py.
by isis＠torproject.org 21 Aug '14

21 Aug '14

commit bd590d73dea233d82364745ae282f386ed670c6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 07:45:54 2014 +0000 Be more careful about how we split bridge lines in test_https.py. This is done to ensure that if we don't get the bridges.html page in response (i.e. if the CAPTCHA solution was wrong, or something is just plain wrong with the webserver that is supposed to be running in the background), that the BeautifulSoup HTML tag handling in test_https.py will be a bit more robust. --- lib/bridgedb/test/test_https.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index 40ab865..cb4a53d 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -121,12 +121,21 @@ class HTTPTests(unittest.TestCase): # </div> bridges = [] soup = soup.findAll(attrs={'class' : 'bridge-lines'}) - bridge_lines = [line.text.split('\n') for line in soup] - self.assertTrue(len(bridge_lines) > 0, "Found no bridge lines") - for bridge_line in bridge_lines: - items = bridge_line.split(' ') - self.assertEquals(len(items), fieldsPerBridge, "Expected %d fields in bridge line %s" % (fieldsPerBridge, str(items))) - bridges.append(items) + self.assertTrue(soup, "Could not find <div class='bridge-lines'>!") + + for portion in soup: + bridge_lines = portion.text.strip().split('\n') + for bridge_line in bridge_lines: + fields = bridge_line.split() + print("\nFields in bridge line: %s" % fields) + bridges.append(fields) + + self.assertTrue(len(bridges) > 0, "Found no bridge lines in %s" % soup) + + for bridge in bridges: + self.assertEquals(len(bridge), fieldsPerBridge, + "Expected %d fields in bridge line %s" + % (fieldsPerBridge, bridge)) return bridges def test_get_obfs2_ipv4(self): @@ -137,7 +146,8 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=3) - for pt, bridge, fingerprint in bridges: + for bridge in bridges: + pt = bridge[0] self.assertEquals(PT, pt) def test_get_obfs3_ipv4(self): @@ -146,9 +156,9 @@ class HTTPTests(unittest.TestCase): PT = 'obfs3' soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) - bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=3) - for pt, bridge, fingerprint in bridges: + for bridge in bridges: + pt = bridge[0] self.assertEquals(PT, pt) def test_get_vanilla_ipv4(self): @@ -159,7 +169,7 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=2) - for bridge, fingerprint in bridges: + for bridge in bridges: # TODO: do more interesting checks self.assertTrue(bridge != None) @@ -171,8 +181,8 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=4) - for pt, bridge, fingerprint, password in bridges: + for bridge in bridges: + pt = bridge[0] + password = bridge[-1] self.assertEquals(PT, pt) self.assertTrue(password.find("password=") != -1, "Password field missing expected text") - -

1 0