February 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Add an image to the head of the developer documentation.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 9c2b7cea6bdc33baf57cfb93cbcd0ab45676215b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 28 11:15:06 2014 +0000 Add an image to the head of the developer documentation. --- doc/sphinx/source/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx/source/index.rst b/doc/sphinx/source/index.rst index 70edc17..06eeac8 100644 --- a/doc/sphinx/source/index.rst +++ b/doc/sphinx/source/index.rst @@ -6,6 +6,10 @@ Welcome to BridgeDB's documentation! ==================================== +.. image:: _static/bay-bridge.jpg + :align: center + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Tor Bridge Descriptor Formats: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

1 0

[bridgedb/master] Fix test_Bucket.BucketDataTests to test `needed` attribute.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit dadd800384358356542ccc49bbdad1ae54006cfc Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:35:01 2014 +0000 Fix test_Bucket.BucketDataTests to test `needed` attribute. --- lib/bridgedb/test/test_Bucket.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index ab89fb4..cff8cb8 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -31,19 +31,19 @@ class BucketDataTest(unittest.TestCase): def test_alloc_some_of_the_bridges(self): """Set the needed number of bridges""" - alloc = 10 + needed = 10 distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(bucket.needed) + this(bucket.name).should.be.equal(distname) + this(bucket.needed).should.be.equal(needed) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default""" - alloc = '*' + needed = '*' distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(1000000) + this(bucket.name).should.be.equal(distname) + this(bucket.needed).should.be.equal(needed) class BucketManagerTest(unittest.TestCase):

1 0

[bridgedb/master] PEP8 fix bad indentation in test_https.py.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 2e28fc225ed83d9e776a8bea81d10738d532ce65 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 02:39:02 2014 +0000 PEP8 fix bad indentation in test_https.py. --- lib/bridgedb/test/test_https.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index 35952a5..7b1c8c8 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -164,8 +164,8 @@ class HTTPTests(unittest.TestCase): captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=2) for bridge in bridges: - # TODO: do more interesting checks - self.assertTrue(bridge != None) + # TODO: do more interesting checks + self.assertTrue(bridge != None) def test_get_scramblesuit_ipv4(self): self.openBrowser()

1 0

[bridgedb/master] Sphinxify a docstring in test_https.py.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit f1d97e94993de6e4170780b4761b0a674fa835c0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 02:34:44 2014 +0000 Sphinxify a docstring in test_https.py. --- lib/bridgedb/test/test_https.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index f06f6d3..657f516 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -106,10 +106,11 @@ class HTTPTests(unittest.TestCase): return soup def getBridgeLinesFromSoup(self, soup, fieldsPerBridge): - # Now we're looking for something like this in the response: - # <div class="bridge-lines"> - # obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd - # </div> + """We're looking for something like this in the response:: + <div class="bridge-lines"> + obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd + </div> + """ bridges = [] soup = soup.findAll(attrs={'class' : 'bridge-lines'}) self.assertTrue(soup, "Could not find <div class='bridge-lines'>!")

1 0

[bridgedb/master] Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets_constification_2' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit cfd71bee10489d216ed67bdc02cdb4f68e1230e0 Merge: 2f0ea18 deada72 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 06:59:46 2014 +0000 Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets_constification_2' into develop Conflicts: lib/bridgedb/Bucket.py lib/bridgedb/test/test_Bucket.py lib/bridgedb/Bucket.py | 9 ++++++--- lib/bridgedb/test/test_Bucket.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --cc lib/bridgedb/Bucket.py index 5b33a36,40a8a32..5e7f61b --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@@ -37,16 -37,15 +37,20 @@@ toHex = binascii.b2a_he # distinguish them from real distributors? PSEUDO_DISTRI_PREFIX = "pseudo_" + # Set to rediculously high number + BUCKET_MAX_BRIDGES = 1000000 + -class BucketData: - """A file bucket value class. - name - Name of the bucket (From config), prefixed by pseudo - distributor prefix - needed - Needed number of bridges for that bucket (From config) - allocated - Number of already allocated bridges for that bucket ++ +class BucketData(object): + """Configures a bridge bucket with the number of bridges which should be + allocated, the name of the bucket, and other similar data. + + :param str name: The name of this bucket (from the config file). This will + be prefixed by the :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :type needed: str or int + :param needed: The number of bridges needed for this bucket (also from the + config file). + :param int allocated: Number of bridges already allocated for this bucket. """ def __init__(self, name, needed): self.name = name diff --cc lib/bridgedb/test/test_Bucket.py index 2912ae1,82424b6..5e39260 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@@ -31,19 -29,19 +31,19 @@@ class BucketDataTest(unittest.TestCase) def test_alloc_some_of_the_bridges(self): """Set the needed number of bridges""" - alloc = 10 + needed = 10 distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) + bucket = Bucket.BucketData(distname, needed) + this(bucket.name).should.be.equal(distname) - this(bucket.needed).should.be.equal(needed) ++ this(bucket.needed).should.be.equal(Bucket.BUCKET_MAX_BRIDGES) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default""" - alloc = '*' + needed = '*' distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) + bucket = Bucket.BucketData(distname, needed) + this(bucket.name).should.be.equal(distname) - this(bucket.needed).should.be.equal(needed) ++ this(bucket.needed).should.be.equal(Bucket.BUCKET_MAX_BRIDGES) class BucketManagerTest(unittest.TestCase):

1 0

[bridgedb/master] Constify the unrealistic maximum number of bridges per bucket
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit b778f85304ace66d1b5dbdcacc781420a71d8201 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 05:51:08 2014 +0000 Constify the unrealistic maximum number of bridges per bucket We will never, ever have 1 million bridges, but because of this we will also never be able to satisfy the requirement of giving a bucket one million bridges. So, when we try distributing the bridges evenly between buckets, the pseudo-distributors that are defined as needing '*' bridges will receive as many as are available to be allocated to them. --- lib/bridgedb/Bucket.py | 6 ++++-- lib/bridgedb/test/test_Bucket.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index a7d3706..e603fe0 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -37,6 +37,9 @@ toHex = binascii.b2a_hex # distinguish them from real distributors? PSEUDO_DISTRI_PREFIX = "pseudo_" +# Set to rediculously high number +BUCKET_MAX_BRIDGES = 1000000 + class BucketData: """A file bucket value class. name - Name of the bucket (From config), prefixed by pseudo @@ -47,8 +50,7 @@ class BucketData: def __init__(self, name, needed): self.name = name if needed == "*": - # Set to rediculously high number - needed = 1000000 + needed = BUCKET_MAX_BRIDGES self.needed = int(needed) self.allocated = 0 diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 7cadd7b..8893dcd 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -39,7 +39,7 @@ class BucketDataTest(unittest.TestCase): distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(1000000) + this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) class BucketManagerTest(unittest.TestCase):

1 0

[bridgedb/master] Sphinxify all docstrings in bridgedb.Bucket.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit aa67615395a4a3d29e855a6e510ddc1a276dbed7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:54:43 2014 +0000 Sphinxify all docstrings in bridgedb.Bucket. --- lib/bridgedb/Bucket.py | 134 +++++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index b5054ac..5b33a36 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -38,11 +38,15 @@ toHex = binascii.b2a_hex PSEUDO_DISTRI_PREFIX = "pseudo_" class BucketData(object): - """A file bucket value class. - name - Name of the bucket (From config), prefixed by pseudo - distributor prefix - needed - Needed number of bridges for that bucket (From config) - allocated - Number of already allocated bridges for that bucket + """Configures a bridge bucket with the number of bridges which should be + allocated, the name of the bucket, and other similar data. + + :param str name: The name of this bucket (from the config file). This will + be prefixed by the :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :type needed: str or int + :param needed: The number of bridges needed for this bucket (also from the + config file). + :param int allocated: Number of bridges already allocated for this bucket. """ def __init__(self, name, needed): self.name = name @@ -54,57 +58,61 @@ class BucketData(object): class BucketManager(object): """BucketManager reads a number of file bucket identifiers from the config. - They're expected to be in the following format: - - FILE_BUCKETS = { "name1": 10, "name2": 15, "foobar": 3 } - - This syntax means that certain buckets ("name1", "name2" and so on) - are given a number of bridges (10, 15 and so on). Names can be anything. - The name will later be the prefix of the file that is written with the - assigned number of bridges in it. Instead of a number, a wildcard item - ("*") is allowed, too. This means that the corresponsing bucket file - will get the maximum number of possible bridges (as many as are left in - the unallocated bucket). - - The files will be written in ip:port format, one bridge per line. - - The way this works internally is as follows: - - First of all, the assignBridgesToBuckets() routine runs through - the database of bridges and looks up the 'distributor' field of each - bridge. Unallocated bridges are sent to a pool for later assignement. - Already allocated bridges for file bucket distribution are sorted and - checked. - They're checked for whether their bucket identifier still exists in the - current config and also whether the number of assigned bridges is still - valid. If either the bucket identifier is not existing anymore or too - many bridges are currently assigned to it, bridges will go to the - unassigned pool. - - In the second step, after bridges are sorted and the unassigned pool is - ready, the assignBridgesToBuckets() routine assigns one bridge - from the unassigned pool to a known bucket identifier at a time until it - either runs out of bridges in the unallocated pool or the number of - needed bridges for that bucket is reached. - - When all bridges are assigned in this way, they can then be dumped into - files by calling the dumpBridges() routine. - - cfg - The central configuration instance - bucketList - A list of BucketData instances, holding all - configured (and thus requested) buckets with - their respective numbers - unallocatedList - Holding all bridges from the 'unallocated' - pool - unallocated_available - Is at least one unallocated bridge - available? - distributor_prefix - The 'distributor' field in the database will - hold the name of our pseudo-distributor, - prefixed by this - db - The bridge database access instance + + They're expected to be in the following format:: + + FILE_BUCKETS = { "name1": 10, "name2": 15, "foobar": 3 } + + This syntax means that certain buckets ("name1", "name2" and so on) are + given a number of bridges (10, 15 and so on). Names can be anything. The + name will later be the prefix of the file that is written with the + assigned number of bridges in it. Instead of a number, a wildcard item + ("*") is allowed, too. This means that the corresponsing bucket file will + get the maximum number of possible bridges (as many as are left in the + unallocated bucket). + + The files will be written in ip:port format, one bridge per line. + + The way this works internally is as follows: + + First of all, the assignBridgesToBuckets() routine runs through the + database of bridges and looks up the 'distributor' field of each + bridge. Unallocated bridges are sent to a pool for later assignement. + Already allocated bridges for file bucket distribution are sorted and + checked. They're checked for whether their bucket identifier still exists + in the current config and also whether the number of assigned bridges is + still valid. If either the bucket identifier is not existing anymore or + too many bridges are currently assigned to it, bridges will go to the + unassigned pool. + + In the second step, after bridges are sorted and the unassigned pool is + ready, the assignBridgesToBuckets() routine assigns one bridge from the + unassigned pool to a known bucket identifier at a time until it either + runs out of bridges in the unallocated pool or the number of needed + bridges for that bucket is reached. + + When all bridges are assigned in this way, they can then be dumped into + files by calling the dumpBridges() routine. + + :type cfg: :class:`bridgedb.persistent.Conf` + :ivar cfg: The central configuration instance. + :ivar list bucketList: A list of BucketData instances, holding all + configured (and thus requested) buckets with their respective numbers. + :ivar list unallocatedList: Holds all bridges from the 'unallocated' pool. + :ivar bool unallocated_available: Is at least one unallocated bridge + available? + :ivar str distributor_prefix: The 'distributor' field in the database will + hold the name of our pseudo-distributor, prefixed by this string. By + default, this uses :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :ivar db: The bridge database instance. """ def __init__(self, cfg): + """Create a ``BucketManager``. + + :type cfg: :class:`bridgedb.persistent.Conf` + :param cfg: The central configuration instance. + """ self.cfg = cfg self.bucketList = [] self.unallocatedList = [] @@ -112,8 +120,7 @@ class BucketManager(object): self.distributor_prefix = PSEUDO_DISTRI_PREFIX def addToUnallocatedList(self, hex_key): - """Add a bridge by hex_key into the unallocated pool - """ + """Add a bridge by **hex_key** into the unallocated pool.""" with bridgedb.Storage.getDB() as db: try: db.updateDistributorForHexKey("unallocated", hex_key) @@ -126,8 +133,8 @@ class BucketManager(object): self.unallocated_available = True def getBucketByIdent(self, bucketIdent): - """Do we know this bucket identifier? If yes, return the corresponding - BucketData object. + """If we know this bucket identifier, then return the corresponding + :class:`BucketData` object. """ for d in self.bucketList: if d.name == bucketIdent: @@ -135,8 +142,7 @@ class BucketManager(object): return None def assignUnallocatedBridge(self, bucket): - """Assign an unallocated bridge to a certain bucket - """ + """Assign an unallocated bridge to a certain **bucket**.""" hex_key = self.unallocatedList.pop() # Mark pseudo-allocators in the database as such allocator_name = bucket.name @@ -163,8 +169,8 @@ class BucketManager(object): return True def assignBridgesToBuckets(self): - """Read file bucket identifiers from the configuration, sort them and - write necessary changes to the database + """Read file bucket identifiers from the configuration, sort them, and + write necessary changes to the database. """ logging.debug("Assigning bridges to buckets for pseudo-distributors") # Build distributor list @@ -221,8 +227,7 @@ class BucketManager(object): self.bucketList.remove(d) def dumpBridgesToFile(self, filename, bridges): - """Dump a list of given bridges into a file - """ + """Dump a list of given **bridges** into **filename**.""" logging.debug("Dumping bridge assignments to file: %r" % filename) # get the bridge histories and sort by Time On Same Address bridgeHistories = [] @@ -263,8 +268,7 @@ class BucketManager(object): print "I/O error: %s" % filename def dumpBridges(self): - """Dump all known file distributors to files, sort by distributor - """ + """Dump all known file distributors to files, sorted by distributor.""" logging.info("Dumping all distributors to file.") with bridgedb.Storage.getDB() as db: allBridges = db.getAllBridges()

1 0

[bridgedb/master] Add missing unittest import
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 0ea71dd97c292063247c25943ed0c4bc11bc93ac Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 06:15:55 2014 +0000 Add missing unittest import --- lib/bridgedb/test/test_Bucket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 8893dcd..82424b6 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -19,6 +19,8 @@ from sure import this from sure import the from sure import expect +from twisted.trial import unittest + from bridgedb import Bucket @@ -31,7 +33,7 @@ class BucketDataTest(unittest.TestCase): distname = "test-distributor" bucket = Bucket.BucketData(distname, alloc) this(distname).should.be.equal(bucket.name) - this(alloc).should.be.equal(bucket.needed) + this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default"""

1 0

[bridgedb/master] Line wasn't indented in f5fe05ce
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit deada7243f5f03289cda044c187e535f505dc9a5 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Wed Aug 20 06:27:41 2014 +0000 Line wasn't indented in f5fe05ce --- lib/bridgedb/Bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index e603fe0..40a8a32 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -254,7 +254,7 @@ class BucketManager: if str(bh.fingerprint) in blocklist.keys(): line = line + "\t(Might be blocked): (%s)" % \ ",".join(blocklist[bh.fingerprint]) - f.write(line + '\n') + f.write(line + '\n') else: for bridge in bridges: line = "%s:%d %s" \

1 0

[bridgedb/master] Remove print() calls from test_https.py.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 68bfbcdc899b5c5fbfd9e97e4b36a5a7402939f9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 02:34:07 2014 +0000 Remove print() calls from test_https.py. --- lib/bridgedb/test/test_https.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index cb4a53d..f06f6d3 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -48,8 +48,6 @@ class HTTPTests(unittest.TestCase): self.assertTrue(self.br.viewing_html()) self.assertEquals(self.br.response().geturl(), HTTPS_ROOT) self.assertEquals(self.br.title(), "BridgeDB") - #for link in self.br.links(): print(link) - #for form in self.br.forms(): print(form) return self.br def goToOptionsPage(self): @@ -64,8 +62,6 @@ class HTTPTests(unittest.TestCase): # ------------- Options self.assertEquals(self.br.response().geturl(), HTTPS_ROOT + "/options") - #print(self.br.response().read()) - #for form in self.br.forms(): print(form) return self.br def submitOptions(self, transport, ipv6, captchaResponse): @@ -81,7 +77,6 @@ class HTTPTests(unittest.TestCase): self.br.select_form(nr=0) # change the pluggable transport to something else - #print(self.br.form) self.br.form['transport'] = [transport] if ipv6: self.br.form['ipv6'] = ['yes'] @@ -92,8 +87,6 @@ class HTTPTests(unittest.TestCase): if ipv6: EXPECTED_URL += "&ipv6=yes" self.assertEquals(self.br.response().geturl(), EXPECTED_URL) - #print(self.br.response().read()) - #for form in self.br.forms(): print(form) # As on the previous page, the form does not define a 'name' attribute, forcing # us to use the index of the form i.e. 0 @@ -103,7 +96,6 @@ class HTTPTests(unittest.TestCase): # input the required captcha response. There is only one captcha defined # by default, so this should always be accepted. Note this will not be possible # to automate if used with a real captcha systems (e.g. reCAPTCHA) - #self.br.form['captcha_response_field'] = 'Tvx74PMy' self.br.form['captcha_response_field'] = captchaResponse captcha_response = self.br.submit() @@ -111,7 +103,6 @@ class HTTPTests(unittest.TestCase): # URL should be the same as last time self.assertEquals(self.br.response().geturl(), EXPECTED_URL) soup = BeautifulSoup(captcha_response.read()) - #print soup.prettify() return soup def getBridgeLinesFromSoup(self, soup, fieldsPerBridge): @@ -127,7 +118,6 @@ class HTTPTests(unittest.TestCase): bridge_lines = portion.text.strip().split('\n') for bridge_line in bridge_lines: fields = bridge_line.split() - print("\nFields in bridge line: %s" % fields) bridges.append(fields) self.assertTrue(len(bridges) > 0, "Found no bridge lines in %s" % soup)

1 0