February 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Whitespace fixes for bridgedb.test.test_smtp.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 209721600ce0f4249b516775eeea709a61142508 Author: trygve <tor-dev(a)lists.torproject.org> Date: Fri Aug 8 03:15:49 2014 +0000 Whitespace fixes for bridgedb.test.test_smtp. --- lib/bridgedb/test/test_smtp.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/bridgedb/test/test_smtp.py b/lib/bridgedb/test/test_smtp.py index 5e8b5c9..4e59002 100644 --- a/lib/bridgedb/test/test_smtp.py +++ b/lib/bridgedb/test/test_smtp.py @@ -38,11 +38,11 @@ class EmailServer(SMTPServer): self.message_queue.put(data) def thread_proc(self): - ''' This function runs in thread, and will continue looping + ''' This function runs in thread, and will continue looping until the _stop Event object is set by the stop() function''' while self._stop.is_set() == False: asyncore.loop(timeout=0.0, count=1) - # must close, or asyncore will hold on to the socket and subsequent tests will fail with 'Address not in use' + # must close, or asyncore will hold on to the socket and subsequent tests will fail with 'Address not in use' self.close() def start(self): @@ -63,7 +63,7 @@ class EmailServer(SMTPServer): # signal thread_proc to stop self._stop.set() # wait for thread_proc to return (shouldn't take long) - self._thread.join() + self._thread.join() assert self._thread.is_alive() == False, "Thread is alive and kicking" def getAndCheckMessageContains(self, text, timeoutInSecs=2.0): @@ -75,7 +75,7 @@ class EmailServer(SMTPServer): try: self.message_queue.get(block=True, timeout=timeoutInSecs) except Queue.Empty: - return True + return True assert False, "Found a message in the queue, but expected none" def sendMail(fromAddress): @@ -101,7 +101,7 @@ class SMTPTests(unittest.TestCase): # send the mail to bridgedb, choosing a random email address sendMail(fromAddress=FROM_ADDRESS_TEMPLATE % random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS)) - # then check that our local SMTP server received a response + # then check that our local SMTP server received a response # and that response contained some bridges self.server.getAndCheckMessageContains("Here are your bridges") @@ -110,11 +110,11 @@ class SMTPTests(unittest.TestCase): FROM_ADDRESS = FROM_ADDRESS_TEMPLATE % random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS) sendMail(FROM_ADDRESS) - # then check that our local SMTP server received a response + # then check that our local SMTP server received a response # and that response contained some bridges self.server.getAndCheckMessageContains("Here are your bridges") - # send another request from the same email address + # send another request from the same email address sendMail(FROM_ADDRESS) # this time, the email response should not contain any bridges @@ -127,7 +127,7 @@ class SMTPTests(unittest.TestCase): self.server.checkNoMessageReceived(timeoutInSecs=1.0) def test_getBridges_stressTest(self): - ''' Sends a large number of emails in a short period of time, and checks that + ''' Sends a large number of emails in a short period of time, and checks that a response is received for each message ''' NUM_MAILS = 100 for i in range(NUM_MAILS): @@ -137,4 +137,3 @@ class SMTPTests(unittest.TestCase): for i in range(NUM_MAILS): self.server.getAndCheckMessageContains("Here are your bridges") -

1 0

[bridgedb/master] Merge branch 'fix/9874-email' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit ef6a5e4fdf4b890a4bdd2c412e20b4e72fa0fa07 Merge: f565959 cc2ea62 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Aug 12 23:45:16 2014 +0000 Merge branch 'fix/9874-email' into develop .travis.yml | 5 ++ lib/bridgedb/test/test_smtp.py | 165 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+)

1 0

[bridgedb/master] Fix typo in inline code comment in bridgedb.Bucket module.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 0181ced5c082b81e1675192da39ff221bcfa757f Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:29:27 2014 +0000 Fix typo in inline code comment in bridgedb.Bucket module. --- lib/bridgedb/Bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index a7d3706..fcc3794 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -47,7 +47,7 @@ class BucketData: def __init__(self, name, needed): self.name = name if needed == "*": - # Set to rediculously high number + # Set to ridiculously high number needed = 1000000 self.needed = int(needed) self.allocated = 0

1 0

[bridgedb/master] Fix missing `twisted.trial.unittest` import in test_Bucket.py.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit fc7582b27bba41f81e1b781422de8d36975aa88b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 20 05:08:07 2014 +0000 Fix missing `twisted.trial.unittest` import in test_Bucket.py. --- lib/bridgedb/test/test_Bucket.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bridgedb/test/test_Bucket.py b/lib/bridgedb/test/test_Bucket.py index 7cadd7b..531a063 100644 --- a/lib/bridgedb/test/test_Bucket.py +++ b/lib/bridgedb/test/test_Bucket.py @@ -21,6 +21,8 @@ from sure import expect from bridgedb import Bucket +from twisted.trial import unittest + class BucketDataTest(unittest.TestCase): """Tests for :class:`bridgedb.Bucket.BucketData`."""

1 0

[bridgedb/develop] Merge branch 'release-0.2.4'
by isis＠torproject.org 03 Feb '15

03 Feb '15

1 0

[bridgedb/master] Merge branch 'master' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 74cdb092998f006a1253aef74e96b3884dd014eb Merge: 1ed4585 09198e1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Jul 30 21:15:09 2014 +0000 Merge branch 'master' into develop lib/bridgedb/email/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[bridgedb/master] Add 'mechanize==0.2.5' to .test.requirements.txt.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit c61af9b2aed80a98359720d46a40420a296cea48 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 7 00:55:41 2014 +0000 Add 'mechanize==0.2.5' to .test.requirements.txt. --- .test.requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.test.requirements.txt b/.test.requirements.txt index dad35c5..df97c1a 100644 --- a/.test.requirements.txt +++ b/.test.requirements.txt @@ -7,6 +7,7 @@ # coverage==3.7.1 leekspin==0.1.4 +mechanize==0.2.5 pep8==1.5.7 # pylint must be pinned until pylint bug #203 is fixed. See # https://bitbucket.org/logilab/pylint/issue/203/importing-namespace-packages…

1 0

[bridgedb/develop] Merge branch 'master' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 1343b4b7f100782338bcf3a00932043f692e5941 Merge: 9955bd5 e08b98e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 3 02:29:15 2015 +0000 Merge branch 'master' into develop

1 0

[bridgedb/master] Add test_https.py from trygve.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit d5556300a9f53c80760e12b13c94a93d041ca9ff Author: trygve <tor-dev(a)lists.torproject.org> Date: Thu Aug 7 00:44:15 2014 +0000 Add test_https.py from trygve. * FIXES part of #9874. See https://trac.torproject.org/projects/tor/attachment/ticket/9874/test_https.… --- lib/bridgedb/test/test_https.py | 146 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py new file mode 100644 index 0000000..46da832 --- /dev/null +++ b/lib/bridgedb/test/test_https.py @@ -0,0 +1,146 @@ +"""integration tests for BridgeDB . +http://wwwsearch.sourceforge.net/mechanize/ +http://www.crummy.com/software/BeautifulSoup/bs3/documentation.html +""" + +from __future__ import print_function +from twisted.trial import unittest +from BeautifulSoup import BeautifulSoup +import mechanize + +HTTPS_ROOT = 'https://127.0.0.1:6789' +CAPTCHA_RESPONSE = 'Tvx74Pmy' + +class HTTPTests(unittest.TestCase): + def setUp(self): + self.br = None + + def tearDown(self): + self.br = None + + def openBrowser(self): + # use mechanize to open the BridgeDB website in its browser + self.br = mechanize.Browser() + self.br.set_handle_robots(False) # prevents 'HTTP Error 403: request disallowed by robots.txt' + self.br.open(HTTPS_ROOT) + + # -------------- Home/Root page + self.assertTrue(self.br.viewing_html()) + self.assertEquals(self.br.response().geturl(), HTTPS_ROOT) + self.assertEquals(self.br.title(), "BridgeDB") + #for link in self.br.links(): print(link) + #for form in self.br.forms(): print(form) + return self.br + + def goToOptionsPage(self): + # check that we are on the root page + self.assertTrue(self.br.viewing_html()) + self.assertEquals(self.br.response().geturl(), HTTPS_ROOT) + + # follow the link with the word 'bridges' in it. + # Could also use: text='bridges' + # Could also use: url='/options' + self.br.follow_link(text_regex='bridges') + + # ------------- Options + self.assertEquals(self.br.response().geturl(), HTTPS_ROOT + "/options") + #print(self.br.response().read()) + #for form in self.br.forms(): print(form) + return self.br + + def submitOptions(self, transport, ipv6, captchaResponse): + # check that we are on the options page + self.assertEquals(self.br.response().geturl(), HTTPS_ROOT + "/options") + + # At this point, we'd like to be able to set some values in + # the 'advancedOptions' form. Unfortunately the HTML form + # does not define a 'name' attribute, so the we have to rely on + # the fact that this is the only form on the page and will therefore + # always exist at index 0. + #br.select_form(name="advancedOptions") + self.br.select_form(nr=0) + + # change the pluggable transport to something else + #print(self.br.form) + self.br.form['transport'] = [transport] + if ipv6: + self.br.form['ipv6'] = ['yes'] + self.br.submit() + + # ------------- Captcha + EXPECTED_URL = HTTPS_ROOT + "/bridges?transport=%s" % transport + if ipv6: + EXPECTED_URL += "&ipv6=yes" + self.assertEquals(self.br.response().geturl(), EXPECTED_URL) + #print(self.br.response().read()) + #for form in self.br.forms(): print(form) + + # As on the previous page, the form does not define a 'name' attribute, forcing + # us to use the index of the form i.e. 0 + #self.br.select_form(name="captchaSubmission") + self.br.select_form(nr=0) + + # input the required captcha response. There is only one captcha defined + # by default, so this should always be accepted. Note this will not be possible + # to automate if used with a real captcha systems (e.g. reCAPTCHA) + #self.br.form['captcha_response_field'] = 'Tvx74PMy' + self.br.form['captcha_response_field'] = captchaResponse + captcha_response = self.br.submit() + + # ------------- Results + # URL should be the same as last time + self.assertEquals(self.br.response().geturl(), EXPECTED_URL) + soup = BeautifulSoup(captcha_response.read()) + #print soup.prettify() + return soup + + def getBridgeLinesFromSoup(self, soup): + # Now we're looking for something like this in the response: + # <div class="bridge-lines"> + # obfs2 175.213.252.207:11125 5c6da7d927460317c6ff5420b75c2d0f431f18dd + # </div> + bridges = [] + bridge_lines = soup.findAll(attrs={'class' : 'bridge-lines'}) + self.assertTrue(len(bridge_lines) > 0, "Found no bridge lines") + for bridge_line in bridge_lines: + text = bridge_line.text + #print("Found bridge-line: %s" % text) + items = text.split(' ') + self.assertEquals(len(items), 3, "Expected Pluggale Transport, Bridge and Fingerprint in %s" % str(items)) + bridges.append(items) + return bridges + + def test_get_obfs2_ipv4(self): + self.openBrowser() + self.goToOptionsPage() + + PT = 'obfs2' + soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) + + bridges = self.getBridgeLinesFromSoup(soup) + for pt, bridge, fingerprint in bridges: + self.assertEquals(PT, pt) + + def test_get_obfs3_ipv4(self): + self.openBrowser() + self.goToOptionsPage() + + PT = 'obfs3' + soup = self.submitOptions(transport=PT, ipv6=False, captchaResponse=CAPTCHA_RESPONSE) + + bridges = self.getBridgeLinesFromSoup(soup) + for pt, bridge, fingerprint in bridges: + self.assertEquals(PT, pt) + + def test_get_obfs3_ipv6(self): + self.openBrowser() + self.goToOptionsPage() + + PT = 'obfs3' + soup = self.submitOptions(transport=PT, ipv6=True, captchaResponse=CAPTCHA_RESPONSE) + + # at the time of writing, this test fails due to lack of ipv6 bridges + bridges = self.getBridgeLinesFromSoup(soup) + for pt, bridge, fingerprint in bridges: + self.assertEquals(PT, pt) +

1 0

[bridgedb/master] Dump bridges to file, despite lack of BridgeHistory
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit f5fe05ce61d0e18bf00638e2eb1fdda785fe4c81 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Aug 3 17:40:53 2014 +0000 Dump bridges to file, despite lack of BridgeHistory We only dumped the bridges assigned to buckets if we had bridgehistory for them. This is not so useful if we disable the stability code which tracks this. We now dump all bridges that are assigned to a distributor to file when the stability code is disabled. --- lib/bridgedb/Bucket.py | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/Bucket.py b/lib/bridgedb/Bucket.py index 2914bf5..a7d3706 100644 --- a/lib/bridgedb/Bucket.py +++ b/lib/bridgedb/Bucket.py @@ -28,6 +28,7 @@ import time import bridgedb.Storage import bridgedb.Bridges import binascii +import sqlite3 from gettext import gettext as _ toHex = binascii.b2a_hex @@ -140,11 +141,15 @@ class BucketManager: # Mark pseudo-allocators in the database as such allocator_name = bucket.name #print "KEY: %d NAME: %s" % (hex_key, allocator_name) + logging.debug("Moving %s to %s" % (hex_key, allocator_name)) with bridgedb.Storage.getDB() as db: try: db.updateDistributorForHexKey(allocator_name, hex_key) except: db.rollback() + logging.warn("Failed to move %s to new distributor (%s)" + % (hex_key, allocator_name)) + # Ok, this seems useless, but for consistancy's sake, we'll # re-assign the bridge from this missed db update attempt to the # unallocated list. Remember? We pop()'d it before. @@ -161,6 +166,7 @@ class BucketManager: """Read file bucket identifiers from the configuration, sort them and write necessary changes to the database """ + logging.debug("Assigning bridges to buckets for pseudo-distributors") # Build distributor list for k, v in self.cfg.FILE_BUCKETS.items(): prefixed_key = self.distributor_prefix + k @@ -197,11 +203,18 @@ class BucketManager: # Loop through bucketList while we have and need unallocated # bridges, assign one bridge at a time while self.unallocated_available and len(self.bucketList) > 0: + logging.debug("We have %d unallocated bridges and %d buckets to " \ + "fill. Let's do it." + % (len(self.unallocatedList), len(self.bucketList))) for d in self.bucketList: if d.allocated < d.needed: - if not self.assignUnallocatedBridge(d): + try: + if not self.assignUnallocatedBridge(d): + break + except sqlite3.DatabaseError as e: dist = d.name.replace(self.distributor_prefix, "") - print "Couldn't assign unallocated bridge to %s" % dist + logging.warn("Couldn't assign unallocated bridge to " \ + "%s: %s" % (dist, e)) else: # When we have enough bridges, remove bucket identifier # from list @@ -215,10 +228,11 @@ class BucketManager: bridgeHistories = [] with bridgedb.Storage.getDB() as db: for b in bridges: - bh = db.getBridgeHistory(b.hex_key) - if bh: bridgeHistories.append(bh) - bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, - y.weightedFractionalUptime)) + if self.cfg.COLLECT_TIMESTAMPS: + bh = db.getBridgeHistory(b.hex_key) + if bh: bridgeHistories.append(bh) + bridgeHistories.sort(lambda x,y: cmp(x.weightedFractionalUptime, + y.weightedFractionalUptime)) # for a bridge, get the list of countries it might not work in blocklist = dict() @@ -230,14 +244,20 @@ class BucketManager: try: f = open(filename, 'w') - for bh in bridgeHistories: - days = bh.tosa / long(60*60*24) - line = "%s:%s\t(%d days at this address)" % \ - (bh.ip, bh.port, days) - if str(bh.fingerprint) in blocklist.keys(): - line = line + "\t(Might be blocked): (%s)" % \ - ",".join(blocklist[bh.fingerprint]) + if self.cfg.COLLECT_TIMESTAMPS: + for bh in bridgeHistories: + days = bh.tosa / long(60*60*24) + line = "%s:%s\t(%d days at this address)" % \ + (bh.ip, bh.port, days) + if str(bh.fingerprint) in blocklist.keys(): + line = line + "\t(Might be blocked): (%s)" % \ + ",".join(blocklist[bh.fingerprint]) f.write(line + '\n') + else: + for bridge in bridges: + line = "%s:%d %s" \ + % (bridge.address, bridge.or_port, bridge.hex_key) + f.write(line + '\n') f.close() except IOError: print "I/O error: %s" % filename @@ -245,6 +265,7 @@ class BucketManager: def dumpBridges(self): """Dump all known file distributors to files, sort by distributor """ + logging.info("Dumping all distributors to file.") with bridgedb.Storage.getDB() as db: allBridges = db.getAllBridges() bridgeDict = {}

1 0