August 2014 - tor-commits - lists.torproject.org

[bridgedb/develop] Be more careful about how we split bridge lines in test_https.py.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit bd590d73dea233d82364745ae282f386ed670c6e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 07:45:54 2014 +0000 Be more careful about how we split bridge lines in test_https.py. This is done to ensure that if we don't get the bridges.html page in response (i.e. if the CAPTCHA solution was wrong, or something is just plain wrong with the webserver that is supposed to be running in the background), that the BeautifulSoup HTML tag handling in test_https.py will be a bit more robust. --- lib/bridgedb/test/test_https.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index 40ab865..cb4a53d 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -121,12 +121,21 @@ class HTTPTests(unittest.TestCase): # </div> bridges = [] soup = soup.findAll(attrs={'class' : 'bridge-lines'}) - bridge_lines = [line.text.split('\n') for line in soup] - self.assertTrue(len(bridge_lines) > 0, "Found no bridge lines") - for bridge_line in bridge_lines: - items = bridge_line.split(' ') - self.assertEquals(len(items), fieldsPerBridge, "Expected %d fields in bridge line %s" % (fieldsPerBridge, str(items))) - bridges.append(items) + self.assertTrue(soup, "Could not find <div class='bridge-lines'>!") + + for portion in soup: + bridge_lines = portion.text.strip().split('\n') + for bridge_line in bridge_lines: + fields = bridge_line.split() + print("\nFields in bridge line: %s" % fields) + bridges.append(fields) + + self.assertTrue(len(bridges) > 0, "Found no bridge lines in %s" % soup) + + for bridge in bridges: + self.assertEquals(len(bridge), fieldsPerBridge, + "Expected %d fields in bridge line %s" + % (fieldsPerBridge, bridge)) return bridges def test_get_obfs2_ipv4(self): @@ -137,7 +146,8 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=3) - for pt, bridge, fingerprint in bridges: + for bridge in bridges: + pt = bridge[0] self.assertEquals(PT, pt) def test_get_obfs3_ipv4(self): @@ -146,9 +156,9 @@ class HTTPTests(unittest.TestCase): PT = 'obfs3' soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) - bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=3) - for pt, bridge, fingerprint in bridges: + for bridge in bridges: + pt = bridge[0] self.assertEquals(PT, pt) def test_get_vanilla_ipv4(self): @@ -159,7 +169,7 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=2) - for bridge, fingerprint in bridges: + for bridge in bridges: # TODO: do more interesting checks self.assertTrue(bridge != None) @@ -171,8 +181,8 @@ class HTTPTests(unittest.TestCase): soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) bridges = self.getBridgeLinesFromSoup(soup, fieldsPerBridge=4) - for pt, bridge, fingerprint, password in bridges: + for bridge in bridges: + pt = bridge[0] + password = bridge[-1] self.assertEquals(PT, pt) self.assertTrue(password.find("password=") != -1, "Password field missing expected text") - -

1 0

[bridgedb/develop] Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets_constification_2' into develop
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit cfd71bee10489d216ed67bdc02cdb4f68e1230e0 Merge: 2f0ea18 deada72 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 06:59:46 2014 +0000 Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets_constification_2' into develop Conflicts: lib/bridgedb/Bucket.py lib/bridgedb/test/test_Bucket.py lib/bridgedb/Bucket.py | 9 ++++++--- lib/bridgedb/test/test_Bucket.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --cc lib/bridgedb/Bucket.py index 5b33a36,40a8a32..5e7f61b --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@@ -37,16 -37,15 +37,20 @@@ toHex = binascii.b2a_he # distinguish them from real distributors? PSEUDO_DISTRI_PREFIX = "pseudo_" + # Set to rediculously high number + BUCKET_MAX_BRIDGES = 1000000 + -class BucketData: - """A file bucket value class. - name - Name of the bucket (From config), prefixed by pseudo - distributor prefix - needed - Needed number of bridges for that bucket (From config) - allocated - Number of already allocated bridges for that bucket ++ +class BucketData(object): + """Configures a bridge bucket with the number of bridges which should be + allocated, the name of the bucket, and other similar data. + + :param str name: The name of this bucket (from the config file). This will + be prefixed by the :data:`PSEUDO_DISTRIBUTOR_PREFIX`. + :type needed: str or int + :param needed: The number of bridges needed for this bucket (also from the + config file). + :param int allocated: Number of bridges already allocated for this bucket. """ def __init__(self, name, needed): self.name = name diff --cc lib/bridgedb/test/test_Bucket.py index 2912ae1,82424b6..5e39260 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@@ -31,19 -29,19 +31,19 @@@ class BucketDataTest(unittest.TestCase) def test_alloc_some_of_the_bridges(self): """Set the needed number of bridges""" - alloc = 10 + needed = 10 distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) + bucket = Bucket.BucketData(distname, needed) + this(bucket.name).should.be.equal(distname) - this(bucket.needed).should.be.equal(needed) ++ this(bucket.needed).should.be.equal(Bucket.BUCKET_MAX_BRIDGES) def test_alloc_all_the_bridges(self): """Set the needed number of bridges to the default""" - alloc = '*' + needed = '*' distname = "test-distributor" - bucket = Bucket.BucketData(distname, alloc) - this(distname).should.be.equal(bucket.name) - this(Bucket.BUCKET_MAX_BRIDGES).should.be.equal(bucket.needed) + bucket = Bucket.BucketData(distname, needed) + this(bucket.name).should.be.equal(distname) - this(bucket.needed).should.be.equal(needed) ++ this(bucket.needed).should.be.equal(Bucket.BUCKET_MAX_BRIDGES) class BucketManagerTest(unittest.TestCase):

1 0

[bridgedb/develop] Disable COLLECT_TIMESTAMPS for CI integration test runs.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit 41d6236f3138cb73723ccfc9d1b7aea5856b5255 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 22:06:21 2014 +0000 Disable COLLECT_TIMESTAMPS for CI integration test runs. --- bridgedb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb.conf b/bridgedb.conf index 651d4a1..a1d6c59 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -167,7 +167,7 @@ LOG_TIME_FORMAT = "%H:%M:%S" # (boolean) If True, then collect, sort, and store all timestamps seen for all # bridges. This operation is extremely expensive, and should be disabled when # it is not necessary. -COLLECT_TIMESTAMPS = True +COLLECT_TIMESTAMPS = False #------------------------------- # General Distribution Options \

1 0

[bridgedb/develop] Increase sleep time in test_bridgedb.py tests for non-CI machines.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit d131f15a98573dd37619b6bb7895e09589e223b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 21:20:37 2014 +0000 Increase sleep time in test_bridgedb.py tests for non-CI machines. --- lib/bridgedb/test/test_bridgedb.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_bridgedb.py b/lib/bridgedb/test/test_bridgedb.py index e54d324..0d53fef 100644 --- a/lib/bridgedb/test/test_bridgedb.py +++ b/lib/bridgedb/test/test_bridgedb.py @@ -56,6 +56,19 @@ class BridgeDBCliTest(unittest.TestCase): return pid + def doSleep(self): + """Sleep for some ammount of time. + + We usually have less fake bridge descriptors with CI runs than we do + during other tests, so we can safely decrease the sleep time on CI + machines. + """ + if os.environ.get("TRAVIS"): + time.sleep(5) + else: + time.sleep(20) + return + def test_bridgedb_assignments_log(self): """This test should only be run if a BridgeDB server has already been started in another process. @@ -81,7 +94,7 @@ class BridgeDBCliTest(unittest.TestCase): os.unlink(self.assignmentsFile) os.kill(self.pid, signal.SIGHUP) - time.sleep(5) + self.doSleep() self.assertTrue(os.path.isfile(self.assignmentsFile)) def test_bridgedb_SIGUSR1_buckets(self): @@ -90,7 +103,7 @@ class BridgeDBCliTest(unittest.TestCase): raise SkipTest("Can't run test: no BridgeDB process running.") os.kill(self.pid, signal.SIGUSR1) - time.sleep(5) + self.doSleep() buckets = [['email', False], ['https', False], ['unallocated', False]] for rundirfile in os.listdir(self.rundir): for bucket in buckets:

1 0

[bridgedb/develop] Fix test_https tests to handle multiple bridge lines.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit 9111886f2785da3995b2e1f78737326aedbcb4a7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 04:30:09 2014 +0000 Fix test_https tests to handle multiple bridge lines. See https://travis-ci.org/isislovecruft/bridgedb/jobs/33035309#L1781 for an example of how this test was failing. --- lib/bridgedb/test/test_https.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index cad0715..40ab865 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -120,12 +120,11 @@ class HTTPTests(unittest.TestCase): # obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd # </div> bridges = [] - bridge_lines = soup.findAll(attrs={'class' : 'bridge-lines'}) + soup = soup.findAll(attrs={'class' : 'bridge-lines'}) + bridge_lines = [line.text.split('\n') for line in soup] self.assertTrue(len(bridge_lines) > 0, "Found no bridge lines") for bridge_line in bridge_lines: - text = bridge_line.text - #print("Found bridge-line: %s" % text) - items = text.split(' ') + items = bridge_line.split(' ') self.assertEquals(len(items), fieldsPerBridge, "Expected %d fields in bridge line %s" % (fieldsPerBridge, str(items))) bridges.append(items) return bridges

1 0

[bridgedb/develop] Add a test for Buckets
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit 12311fff64dba06b24c66dd0523e52f3cd8927b9 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Aug 3 18:18:22 2014 +0000 Add a test for Buckets The others will require mocking a DB connection. --- lib/bridgedb/test/test_Bucket.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py new file mode 100644 index 0000000..7cadd7b --- /dev/null +++ b/lib/bridgedb/test/test_Bucket.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :copyright: (c) 2007-2014, The Tor Project, Inc. +# (c) 2007-2014, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.Bucket` module. + +These tests are meant to ensure that the :mod:`bridgedb.Bucket` module is +functioning as expected. +""" + +from __future__ import print_function + +import sure +from sure import this +from sure import the +from sure import expect + +from bridgedb import Bucket + + +class BucketDataTest(unittest.TestCase): + """Tests for :class:`bridgedb.Bucket.BucketData`.""" + + def test_alloc_some_of_the_bridges(self): + """Set the needed number of bridges""" + alloc = 10 + distname = "test-distributor" + bucket = Bucket.BucketData(distname, alloc) + this(distname).should.be.equal(bucket.name) + this(alloc).should.be.equal(bucket.needed) + + def test_alloc_all_the_bridges(self): + """Set the needed number of bridges to the default""" + alloc = '*' + distname = "test-distributor" + bucket = Bucket.BucketData(distname, alloc) + this(distname).should.be.equal(bucket.name) + this(alloc).should.be.equal(1000000) + + +class BucketManagerTest(unittest.TestCase): + """Tests for :class:`bridgedb.Bucket.BucketManager`.""" + TEST_CONFIG_FILE = StringIO(unicode("""\ + FILE_BUCKETS = { 'test1': 7, 'test2': 11 } + COLLECT_TIMESTAMPS = False + COUNTRY_BLOCK_FILE = []""")) + + def setUp(self): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + exec compiled in configuration + self.config = persistent.Conf(**configuration) + self.state = persistent.State(**config.__dict__) + self.bucket = Bucket.BucketManager(self.config)

1 0

[bridgedb/develop] Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets' into develop
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit d67770d8cfe717bfdc0540311178a3fb6e1b7ced Merge: 0156214 12311ff Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 04:15:12 2014 +0000 Merge remote-tracking branch 'tpo-sysrqb/bug12781-dumpbuckets' into develop lib/bridgedb/Bucket.py | 47 +++++++++++++++++++++--------- lib/bridgedb/test/test_Bucket.py | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 13 deletions(-)

1 0

[bridgedb/develop] Dump bridges to file, despite lack of BridgeHistory
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit f5fe05ce61d0e18bf00638e2eb1fdda785fe4c81 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Aug 3 17:40:53 2014 +0000 Dump bridges to file, despite lack of BridgeHistory We only dumped the bridges assigned to buckets if we had bridgehistory for them. This is not so useful if we disable the stability code which tracks this. We now dump all bridges that are assigned to a distributor to file when the stability code is disabled. --- lib/bridgedb/Bucket.py | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 2914bf5..a7d3706 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -28,6 +28,7 @@ import time import bridgedb.Storage import bridgedb.Bridges import binascii +import sqlite3 from gettext import gettext as _ toHex = binascii.b2a_hex @@ -140,11 +141,15 @@ class BucketManager: # Mark pseudo-allocators in the database as such allocator_name = bucket.name #print "KEY: %d NAME: %s" % (hex_key, allocator_name) + logging.debug("Moving %s to %s" % (hex_key, allocator_name)) with bridgedb.Storage.getDB() as db: try: db.updateDistributorForHexKey(allocator_name, hex_key) except: db.rollback() + logging.warn("Failed to move %s to new distributor (%s)" + % (hex_key, allocator_name)) + # Ok, this seems useless, but for consistancy's sake, we'll # re-assign the bridge from this missed db update attempt to the # unallocated list. Remember? We pop()'d it before. @@ -161,6 +166,7 @@ class BucketManager: """Read file bucket identifiers from the configuration, sort them and write necessary changes to the database """ + logging.debug("Assigning bridges to buckets for pseudo-distributors") # Build distributor list for k, v in self.cfg.FILE_BUCKETS.items(): prefixed_key = self.distributor_prefix + k @@ -197,11 +203,18 @@ class BucketManager: # Loop through bucketList while we have and need unallocated # bridges, assign one bridge at a time while self.unallocated_available and len(self.bucketList) > 0: + logging.debug("We have %d unallocated bridges and %d buckets to " \ + "fill. Let's do it." + % (len(self.unallocatedList), len(self.bucketList))) for d in self.bucketList: if d.allocated < d.needed: - if not self.assignUnallocatedBridge(d): + try: + if not self.assignUnallocatedBridge(d): + break + except sqlite3.DatabaseError as e: dist = d.name.replace(self.distributor_prefix, "") - print "Couldn't assign unallocated bridge to %s" % dist + logging.warn("Couldn't assign unallocated bridge to " \ + "%s: %s" % (dist, e)) else: # When we have enough bridges, remove bucket identifier # from list @@ -215,10 +228,11 @@ class BucketManager: bridgeHistories = [] with bridgedb.Storage.getDB() as db: for b in bridges: - bh = db.getBridgeHistory(b.hex_key) - if bh: bridgeHistories.append(bh) - bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, - y.weightedFractionalUptime)) + if self.cfg.COLLECT_TIMESTAMPS: + bh = db.getBridgeHistory(b.hex_key) + if bh: bridgeHistories.append(bh) + bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, + y.weightedFractionalUptime)) # for a bridge, get the list of countries it might not work in blocklist = dict() @@ -230,14 +244,20 @@ class BucketManager: try: f = open(filename, 'w') - for bh in bridgeHistories: - days = bh.tosa / long(60*60*24) - line = "%s:%s\t(%d days at this address)" % \ - (bh.ip, bh.port, days) - if str(bh.fingerprint) in blocklist.keys(): - line = line + "\t(Might be blocked): (%s)" % \ - ",".join(blocklist[bh.fingerprint]) + if self.cfg.COLLECT_TIMESTAMPS: + for bh in bridgeHistories: + days = bh.tosa / long(60*60*24) + line = "%s:%s\t(%d days at this address)" % \ + (bh.ip, bh.port, days) + if str(bh.fingerprint) in blocklist.keys(): + line = line + "\t(Might be blocked): (%s)" % \ + ",".join(blocklist[bh.fingerprint]) f.write(line + '\n') + else: + for bridge in bridges: + line = "%s:%d %s" \ + % (bridge.address, bridge.or_port, bridge.hex_key) + f.write(line + '\n') f.close() except IOError: print "I/O error: %s" % filename @@ -245,6 +265,7 @@ class BucketManager: def dumpBridges(self): """Dump all known file distributors to files, sort by distributor """ + logging.info("Dumping all distributors to file.") with bridgedb.Storage.getDB() as db: allBridges = db.getAllBridges() bridgeDict = {}

1 0

[bridgedb/develop] Fix missing `twisted.trial.unittest` import in test_Bucket.py.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit fc7582b27bba41f81e1b781422de8d36975aa88b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:08:07 2014 +0000 Fix missing `twisted.trial.unittest` import in test_Bucket.py. --- lib/bridgedb/test/test_Bucket.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 7cadd7b..531a063 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -21,6 +21,8 @@ from sure import expect from bridgedb import Bucket +from twisted.trial import unittest + class BucketDataTest(unittest.TestCase): """Tests for :class:`bridgedb.Bucket.BucketData`."""

1 0

[bridgedb/develop] Fix typo in inline code comment in bridgedb.Bucket module.
by isis＠torproject.org 20 Aug '14

20 Aug '14

commit 0181ced5c082b81e1675192da39ff221bcfa757f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:29:27 2014 +0000 Fix typo in inline code comment in bridgedb.Bucket module. --- lib/bridgedb/Bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index a7d3706..fcc3794 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -47,7 +47,7 @@ class BucketData: def __init__(self, name, needed): self.name = name if needed == "*": - # Set to rediculously high number + # Set to ridiculously high number needed = 1000000 self.needed = int(needed) self.allocated = 0

1 0