February 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Merge branch 'fix/9874-email' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 9729bd102a81bba3cddfaf51526e99610b2f3cf5 Merge: 0075c56 82f4654 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 05:54:39 2014 +0000 Merge branch 'fix/9874-email' into develop lib/bridgedb/test/test_smtp.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-)

1 0

[bridgedb/master] Merge commit 'a580c4ba0d30f55546d3d181d7ea878e6a13133c' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 63d56a8142b7dba4eb4eec3da8f3d40a67d1cf59 Merge: b4f20eb a580c4b Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 06:40:22 2014 +0000 Merge commit 'a580c4ba0d30f55546d3d181d7ea878e6a13133c' into develop

1 0

[bridgedb/master] PEP8 OCD import rearrangements in test_https.py.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 2e14f042174c0de3d256b8dbfe13b587574a4dac Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 05:43:40 2014 +0000 PEP8 OCD import rearrangements in test_https.py. --- lib/bridgedb/test/test_https.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index 176b769..95c2be8 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -23,11 +23,13 @@ repository. """ from __future__ import print_function -from twisted.trial import unittest -from BeautifulSoup import BeautifulSoup + import mechanize import os +from BeautifulSoup import BeautifulSoup +from twisted.trial import unittest + from bridgedb.test.util import processExists from bridgedb.test.util import getBridgeDBPID

1 0

[bridgedb/master] Add unittests for bridgedb.parse.versions module.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit ab99f855f708dec213c9eea1489643c01526e0b0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 21 04:47:54 2014 +0000 Add unittests for bridgedb.parse.versions module. --- lib/bridgedb/test/test_parse_versions.py | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/lib/bridgedb/test/test_parse_versions.py b/lib/bridgedb/test/test_parse_versions.py new file mode 100644 index 0000000..3b2e392 --- /dev/null +++ b/lib/bridgedb/test/test_parse_versions.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +#_____________________________________________________________________________ +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2014, The Tor Project, Inc. +# (c) 2014, Isis Lovecruft +# :license: see LICENSE for licensing information +#_____________________________________________________________________________ + +"""Unittests for :mod:`bridgedb.parse.versions`.""" + + +from __future__ import print_function + +from twisted.trial import unittest + +from bridgedb.parse import versions + + +class ParseVersionTests(unittest.TestCase): + """Unitests for :class:`bridgedb.parse.versions.Version`.""" + + def test_Version_with_bad_delimiter(self): + """Test parsing a version number which uses '-' as a delimiter.""" + self.assertRaises(versions.InvalidVersionStringFormat, + versions.Version, '2-6-0', package='tor') + + def test_Version_really_long_version_string(self): + """Parsing a version number which is way too long should raise + an IndexError which is ignored. + """ + v = versions.Version('2.6.0.0.beta', package='tor') + self.assertEqual(v.prerelease, 'beta') + self.assertEqual(v.major, 6) + + def test_Version_string(self): + """Test converting a valid Version object into string form.""" + v = versions.Version('0.2.5.4', package='tor') + self.assertEqual(v.base(), '0.2.5.4')

1 0

[bridgedb/master] Only run test_smtp.py tests if a BridgeDB process is already running.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit e4db0215f3282a6d53ba001ef6f75af648cdebbb Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 05:45:41 2014 +0000 Only run test_smtp.py tests if a BridgeDB process is already running. --- lib/bridgedb/test/test_smtp.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/bridgedb/test/test_smtp.py b/lib/bridgedb/test/test_smtp.py index fb25c46..51d08f5 100644 --- a/lib/bridgedb/test/test_smtp.py +++ b/lib/bridgedb/test/test_smtp.py @@ -9,6 +9,12 @@ import asyncore import threading import Queue import random +import os + +from twisted.trial.unittest import SkipTest + +from bridgedb.test.util import processExists +from bridgedb.test.util import getBridgeDBPID # ------------- SMTP Client Config SMTP_DEBUG_LEVEL = 0 # set to 1 to see SMTP message exchange @@ -109,6 +115,11 @@ class SMTPTests(unittest.TestCase): '''Called at the start of each test, ensures that the SMTP server is running. ''' + here = os.getcwd() + topdir = here.rstrip('_trial_temp') + self.rundir = os.path.join(topdir, 'run') + self.pidfile = os.path.join(self.rundir, 'bridgedb.pid') + self.pid = getBridgeDBPID(self.pidfile) self.server = EmailServer.startServer() def tearDown(self): @@ -117,6 +128,9 @@ class SMTPTests(unittest.TestCase): self.server.stop() def test_getBridges(self): + if not self.pid or not processExists(self.pid): + raise SkipTest("Can't run test: no BridgeDB process running.") + # send the mail to bridgedb, choosing a random email address sendMail(fromAddress=FROM_ADDRESS_TEMPLATE % random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS)) @@ -126,6 +140,9 @@ class SMTPTests(unittest.TestCase): self.server.getAndCheckMessageContains("Here are your bridges") def test_getBridges_rateLimitExceeded(self): + if not self.pid or not processExists(self.pid): + raise SkipTest("Can't run test: no BridgeDB process running.") + # send the mail to bridgedb, choosing a random email address FROM_ADDRESS = FROM_ADDRESS_TEMPLATE % random.randint( MIN_FROM_ADDRESS, MAX_FROM_ADDRESS) @@ -152,6 +169,9 @@ class SMTPTests(unittest.TestCase): '''Sends a large number of emails in a short period of time, and checks that a response is received for each message. ''' + if not self.pid or not processExists(self.pid): + raise SkipTest("Can't run test: no BridgeDB process running.") + NUM_MAILS = 100 for i in range(NUM_MAILS): # Note: if by chance two emails with the same FROM_ADDRESS are

1 0

[bridgedb/master] Add missing errno module import to bridgedb.test.util.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 0075c560e9b2360cba56a244b0a5684540c919f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 05:52:47 2014 +0000 Add missing errno module import to bridgedb.test.util. --- lib/bridgedb/test/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 4c6f795..bec590d 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -14,6 +14,7 @@ from __future__ import print_function from __future__ import unicode_literals +import errno import os from functools import wraps

1 0

[bridgedb/master] Merge branch 'fix/9874-https' into develop
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 25e092708ca0a50399cbd06d67c63cf46f145d3d Merge: f5360f5 2e14f04 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 27 05:46:45 2014 +0000 Merge branch 'fix/9874-https' into develop lib/bridgedb/test/test_https.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-)

1 0

[bridgedb/master] Copy captchas/ into run/ during 'before_script' .travis.yml section.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit c6ec7e51792029a28f1c31809e35a1c6f7e47bde Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 7 06:27:56 2014 +0000 Copy captchas/ into run/ during 'before_script' .travis.yml section. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e664a26..d8edafb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ install: # Start a BridgeDB instance before running the tests: before_script: - mkdir run - - cp -t run bridgedb.conf + - cp -R -t run bridgedb.conf captchas - leekspin -n 100 - cp -t run networkstatus-bridges cached-extrainfo* bridge-descriptors - ./scripts/make-ssl-cert

1 0

[bridgedb/master] Split unittests in test_bridgedb.py; refactor to use already running server.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 9873fad412081d35e508de5c8dd095a17fd921cb Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 7 07:47:05 2014 +0000 Split unittests in test_bridgedb.py; refactor to use already running server. The unittests in `lib/bridgedb/test/test_bridgedb.py` now expect a BridgeDB instance to have already been started *before* the tests run. This change is necessary because it is non-trivial to have two instances of BridgeDB running simultaneously (at least not without changing every port in the config file) and the tests in `lib/bridgedb/test/test_https.py` also require a running server. These refactored tests should raise SkipTest if they believe that there is no running BridgeDB instance to test against. For Travis CI, the BridgeDB instance is started automatically in the 'before_script' section of `travis.yml` (in the top directory of this repo). * FIXES part of #9874. --- lib/bridgedb/test/test_bridgedb.py | 167 +++++++++++++++--------------------- lib/bridgedb/test/util.py | 22 +++++ 2 files changed, 89 insertions(+), 100 deletions(-) diff --git a/lib/bridgedb/test/test_bridgedb.py b/lib/bridgedb/test/test_bridgedb.py index c7aefff..e54d324 100644 --- a/lib/bridgedb/test/test_bridgedb.py +++ b/lib/bridgedb/test/test_bridgedb.py @@ -14,121 +14,88 @@ from __future__ import print_function import os -import shutil import signal import time -from os.path import join as pjoin -from subprocess import Popen, PIPE - -from twisted.python import log -from twisted.python.procutils import which from twisted.trial import unittest +from twisted.trial.unittest import SkipTest -from bridgedb.test.util import fileCheckDecorator +from bridgedb.test.util import pidExists class BridgeDBCliTest(unittest.TestCase): """Test the `bridgedb` command.""" - @fileCheckDecorator - def doCopyFile(self, src, dst, description=None): - shutil.copy(src, dst) - - @fileCheckDecorator - def doMoveFile(self, src, dst, description=None): - shutil.move(src, dst) - - def test_bridgedb_commands(self): - print('') - here = os.getcwd() - runDir = pjoin(here, 'rundir') - topDir = here.rstrip('_trial_temp') - scriptsDir = pjoin(topDir, 'scripts') - - # Create the lowest directory we need, and all its parents: - os.makedirs(os.path.join(runDir, 'gnupghome')) - - conf = pjoin(topDir, 'bridgedb.conf') - confMoved = pjoin(runDir, 'bridgedb.conf') - gpgFile = pjoin(topDir, 'gnupghome', 'TESTING.subkeys.sec') - gpgMoved = pjoin(runDir, 'gnupghome', 'TESTING.subkeys.sec') - certFile = pjoin(topDir, 'cert') - certMoved = pjoin(runDir, 'cert') - keyFile = pjoin(topDir, 'privkey.pem') - keyMoved = pjoin(runDir, 'privkey.pem') - - makeSSLCertScript = os.path.join(scriptsDir, 'make-ssl-cert') - bridgedbScript = which('bridgedb') # this returns a list - - self.doCopyFile(conf, confMoved, 'config') - self.doCopyFile(gpgFile, gpgMoved, 'GPG test key') - print("Running subcommands from directory:\n %r" % runDir) - print("Running %r..." % makeSSLCertScript) - makeSSLCertProcess = Popen(makeSSLCertScript) - makeSSLCertProcess.wait() - self.doMoveFile(certFile, certMoved, 'certificate') - self.doMoveFile(keyFile, keyMoved, 'SSL private key') - - self.assertTrue(os.path.isfile(bridgedbScript[0]), - "Couldn't find bridgedb script %r" % bridgedbScript[0]) - bridgedbScript = bridgedbScript[0] - print("Running bridgedb script %r..." % bridgedbScript) - - os.chdir(runDir) # we have to do this to get files to end up there - print("Running `bridgedb mock' to generate mock bridge descriptors...") - mockProc = Popen([bridgedbScript, 'mock', '-n', '50']) - mockProcCode = mockProc.wait() - print("`bridgedb mock' exited with status code %d" % int(mockProcCode)) - os.chdir(here) - - # See ticket #11216, cached-extrainfo* files should not be parsed - # cumulatively. - eidesc = pjoin(runDir, 'cached-extrainfo') - eindesc = pjoin(runDir, 'cached-extrainfo.new') - self.doCopyFile(eindesc, eidesc, 'duplicated cached-extrainfo(.new)') - self.assertTrue(os.path.isfile(eidesc)) - self.assertTrue(os.path.isfile(eindesc)) - - - print("Running `bridgedb' to test server startups...") - # Sorry Windows users - devnull = open('/dev/null', 'w') - bridgedbProc = Popen([bridgedbScript, '-r', runDir], stdout=devnull) - print("Waiting 10 seconds while bridgedb loads...") - time.sleep(10) - assignments = pjoin(runDir, 'assignments.log') - self.assertTrue(os.path.isfile(assignments)) - os.unlink(assignments) - print("Sending SIGHUP, checking for assignments.log ...") - bridgedbProc.send_signal(signal.SIGHUP) - time.sleep(5) + def setUp(self): + here = os.getcwd() + topdir = here.rstrip('_trial_temp') + self.rundir = os.path.join(topdir, 'run') + self.pidfile = os.path.join(self.rundir, 'bridgedb.pid') + self.pid = self.getBridgeDBPID(self.pidfile) + self.assignmentsFile = os.path.join(self.rundir, 'assignments.log') + + def getBridgeDBPID(self, pidfile="bridgedb.pid"): + """Read the ``bridgedb.pid`` file in **rundir**, if it exists, to get + the PID. + + :param str pidfile: The path to the BridgeDB pidfile. + :rtype: int + :returns: The process ID, if available, otherwise ``0``. + """ + fh = None try: - self.assertTrue(os.path.isfile(assignments)) - except self.failureException as e: - bridgedbProc.send_signal(signal.SIGKILL) - bridgedbProcCode = bridgedbProc.wait() - print("`bridgedb' exited with status code %d" % int(bridgedbProcCode)) - raise e - print("Sending SIGUSR1, checking for bucket files...") - bridgedbProc.send_signal(signal.SIGUSR1) + fh = open(pidfile) + except (IOError, OSError) as err: + print(err) + pid = 0 + else: + pid = int(fh.read()) + + if fh: + fh.close() + + return pid + + def test_bridgedb_assignments_log(self): + """This test should only be run if a BridgeDB server has already been + started in another process. + + To see how this is done for the Travis CI tests, see the + 'before_script' section of the ``.travis.yml`` file in the top + directory of this repository. + + This test ensures that an ``assignments.log`` file is created after a + BridgeDB process was started. + """ + if not self.pid: + raise SkipTest("Can't run test: no BridgeDB process running.") + + self.assertTrue(os.path.isfile(self.assignmentsFile)) + + def test_bridgedb_SIGHUP_assignments_log(self): + """Test that BridgeDB creates a new ``assignments.log`` file after + receiving a SIGHUP. + """ + if not self.pid: + raise SkipTest("Can't run test: no BridgeDB process running.") + + os.unlink(self.assignmentsFile) + os.kill(self.pid, signal.SIGHUP) + time.sleep(5) + self.assertTrue(os.path.isfile(self.assignmentsFile)) + + def test_bridgedb_SIGUSR1_buckets(self): + """Test that BridgeDB dumps buckets appropriately after a SIGUSR1.""" + if not self.pid: + raise SkipTest("Can't run test: no BridgeDB process running.") + + os.kill(self.pid, signal.SIGUSR1) time.sleep(5) buckets = [['email', False], ['https', False], ['unallocated', False]] - for rundirfile in os.listdir(runDir): + for rundirfile in os.listdir(self.rundir): for bucket in buckets: if rundirfile.startswith(bucket[0]): bucket[1] = True break for bucket in buckets: - try: - self.assertTrue(bucket[1], "%s bucket was not dumped!" % bucket[0]) - except self.failureException as e: - bridgedbProc.send_signal(signal.SIGKILL) - bridgedbProcCode = bridgedbProc.wait() - print("`bridgedb' exited with status code %d" % int(bridgedbProcCode)) - raise e - print("Done. Killing processes.") - bridgedbProc.send_signal(signal.SIGINT) - bridgedbProcCode = bridgedbProc.wait() - print("`bridgedb' exited with status code %d" % int(bridgedbProcCode)) - self.assertEqual(bridgedbProcCode, 0) + self.assertTrue(bucket[1], "%s bucket was not dumped!" % bucket[0]) diff --git a/lib/bridgedb/test/util.py b/lib/bridgedb/test/util.py index 5538a29..1067526 100644 --- a/lib/bridgedb/test/util.py +++ b/lib/bridgedb/test/util.py @@ -65,6 +65,28 @@ def fileCheckDecorator(func): % (str(description), dst, src)) return wrapper +def pidExists(pid): + """Test if **pid** exists. + + :raises: OSError, if ``OSError.errno`` wasn't an expected errno (according + to the "ERRORS" section from ``man 2 kill``). + :rtype: bool + :returns: ``True`` if a process with **pid** exists, ``False`` otherwise. + """ + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: # ESRCH: No such process + return False + if err.errno == errno.EPERM: # EPERM: Operation not permitted + # If we're not allowed to signal the process, then there exists a + # process that we don't have permissions to access. + return True + else: + raise + else: + return True + #: Mixin class for use with :api:`~twisted.trial.unittest.TestCase`. A #: ``TestCaseMixin`` can be used to add additional methods, which should be

1 0

[bridgedb/master] Move several test dependencies from .travis.yml to .travis.requirements.txt.
by isis＠torproject.org 03 Feb '15

03 Feb '15

commit 3c9b8732a5c9119837212056306deb4353d6e7bd Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Aug 7 06:06:31 2014 +0000 Move several test dependencies from .travis.yml to .travis.requirements.txt. --- .travis.requirements.txt | 24 ++++++++++++++++++++++++ .travis.yml | 7 ------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.travis.requirements.txt b/.travis.requirements.txt index e018631..428438a 100644 --- a/.travis.requirements.txt +++ b/.travis.requirements.txt @@ -1,3 +1,27 @@ +# +# This is a pip requirements.txt file for running BridgeDB's tests on Travis +# See https://travis-ci.org/ and the .travis.yml file for more info. Please +# note that there may be additional dependencies not listed here but specified +# with their version numbers in .travis.yml in order to submit test runs which +# use multiple different dependency versions. +# +#------------------------------------------------------------------------------ +# +# To run install these dependencies and run the tests, do: +# +# $ pip install -r .test.requirements.txt +# $ make coverage +# +#------------------------------------------------------------------------------ +coverage==3.7.1 +coveralls==0.4.2 +leekspin==0.1.4 +mechanize==0.2.5 +pep8==1.5.7 +# pylint must be pinned until pylint bug #203 is fixed. See +# https://bitbucket.org/logilab/pylint/issue/203/importing-namespace-packages… +pylint==1.1.0 +sure==1.2.2 Babel==0.9.6 BeautifulSoup==3.2.1 Mako==0.8.1 diff --git a/.travis.yml b/.travis.yml index 7374fa7..9066fc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,11 +26,6 @@ addons: - bridges.torproject.org env: - global: - - LEEKSPIN_VERSION=0.1.4 - - COVERAGE_VERSION=3.7.1 - - COVERALLS_VERSION=0.4.2 - - SURE_VERSION=1.2.2 matrix: - TWISTED_VERSION=13.2.0 PYOPENSSL_VERSION=0.13.1 @@ -48,8 +43,6 @@ before_install: install: - sudo apt-get install -qq --no-install-suggests --no-install-recommends build-essential openssl sqlite3 libgpgme11 libgpgme11-dev python-dev python-setuptools - pip install -q -r .travis.requirements.txt - - pip install -q --no-use-wheel leekspin==$LEEKSPIN_VERSION sure==$SURE_VERSION - - pip install -q --no-use-wheel coverage==$COVERAGE_VERSION coveralls==$COVERALLS_VERSION - pip install -q --no-use-wheel Twisted==$TWISTED_VERSION pyOpenSSL==$PYOPENSSL_VERSION - make install

1 0