April 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Don't log special/private attributes when logging changed configuration.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 584d5911df5318caeabc5b57a0b480dc10d211e9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 05:05:28 2015 +0000 Don't log special/private attributes when logging changed configuration. --- lib/bridgedb/configure.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/configure.py b/lib/bridgedb/configure.py index 8a24d35..2992e1c 100644 --- a/lib/bridgedb/configure.py +++ b/lib/bridgedb/configure.py @@ -83,7 +83,8 @@ def loadConfig(configFile=None, configCls=None): if itsSafeToUseLogging: logging.debug("New configuration settings:") logging.debug("\n".join(["{0} = {1}".format(key, value) - for key, value in configuration.items()])) + for key, value in configuration.items() + if not key.startswith('_')])) # Create a :class:`Conf` from the settings stored within the local scope # of the ``configuration`` dictionary:

1 0

[bridgedb/develop] Add an extra newline after email footers to avoid looking too busy.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit efc3fdda61544360466a6882f0fb4a86d1e4abdc Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 05:07:12 2015 +0000 Add an extra newline after email footers to avoid looking too busy. --- lib/bridgedb/email/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/email/templates.py b/lib/bridgedb/email/templates.py index ae85e8f..a6ff38c 100644 --- a/lib/bridgedb/email/templates.py +++ b/lib/bridgedb/email/templates.py @@ -150,7 +150,7 @@ def addFooter(template, clientAddress=None): % (clientAddr, now.strftime('%A, %d %B, %Y'), now.strftime('%H:%M:%S')) - footer += u'\n' + footer += u'\n\n' return footer

1 0

[bridgedb/develop] Decrease insane ammounts of logging for adding proxies to ProxySet.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 5d8f47943c78f87821d6102a72fe321a3fce8b39 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 05:07:58 2015 +0000 Decrease insane ammounts of logging for adding proxies to ProxySet. --- lib/bridgedb/proxy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/bridgedb/proxy.py b/lib/bridgedb/proxy.py index 3940ed6..7fab5af 100644 --- a/lib/bridgedb/proxy.py +++ b/lib/bridgedb/proxy.py @@ -92,7 +92,6 @@ def loadProxiesFromFile(filename, proxySet=None, removeStale=False): if isinstance(proxySet, ProxySet): # ProxySet.add() will validate the IP address if proxySet.add(line, tag=filename): - logging.info("Added %s to the proxy list." % line) addresses.append(line) else: ip = isIPAddress(line) @@ -170,7 +169,7 @@ class ProxySet(MutableSet): ip = isIPAddress(ip) if ip: if self._proxies.isdisjoint(set(ip)): - logging.debug("Adding %s to proxy list %r..." % (ip, self)) + logging.debug("Adding %s to proxy list..." % ip) self._proxies.add(ip) self._proxydict[ip] = tag if tag else time.time() return True

1 0

[bridgedb/develop] Fix documentation for `key` parameter in IPBasedDistributor.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit a43a05802c108292f26c9bc1bc959d5cfb3b62dc Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:36:28 2015 +0000 Fix documentation for `key` parameter in IPBasedDistributor. Conflicts: lib/bridgedb/Dist.py --- lib/bridgedb/Dist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 6bb41b9..78741a7 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -156,9 +156,9 @@ class IPBasedDistributor(Distributor): because the set of known open proxies constitutes its own category. DOCDOC What exactly does a cluster *do*? - :param bytearray key: The master HMAC key for this distributor. All - added bridges are HMACed with this key in order to place them into - the hashrings. + :param bytes key: The master HMAC key for this distributor. All added + bridges are HMACed with this key in order to place them into the + hashrings. :type ipCategories: iterable or None :param ipCategories: DOCDOC :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters`

1 0

[bridgedb/develop] Print each log line from get-tor-exits to it's own line in our log.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 63bd41f58ca2e4fe4b008fcba12739590e3c3b2c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 05:08:38 2015 +0000 Print each log line from get-tor-exits to it's own line in our log. --- lib/bridgedb/proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/proxy.py b/lib/bridgedb/proxy.py index 7fab5af..086f3bd 100644 --- a/lib/bridgedb/proxy.py +++ b/lib/bridgedb/proxy.py @@ -406,7 +406,8 @@ class ExitListProtocol(protocol.ProcessProtocol): def errReceived(self, data): """Some data was received from stderr.""" # The get-exit-list script uses twisted.python.log to log to stderr: - logging.debug(data) # pragma: no cover + for line in data.splitlines(): # pragma: no cover + logging.debug(line) def outReceived(self, data): """Some data was received from stdout."""

1 0

[bridgedb/develop] Move N_IP_CLUSTERS into the HTTPS distributor-specific settings.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 70009a726f6076036f29909fc5476ae4cc94fd2d Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 01:32:45 2015 +0000 Move N_IP_CLUSTERS into the HTTPS distributor-specific settings. --- bridgedb.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 2a1c367..cbdead0 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -217,12 +217,6 @@ NO_DISTRIBUTION_COUNTRIES = ['IR', 'SY'] # will treat them specially. PROXY_LIST_FILES = [] -# How many clusters do we group IPs in when distributing bridges based on IP? -# Note that if PROXY_LIST_FILES is set (below), what we actually do here -# is use one higher than the number here, and the extra cluster is used -# for answering requests made by IP addresses in the PROXY_LIST_FILES file. -N_IP_CLUSTERS = 4 - # If possible, always give a certain number of answers with a given ORPort. # This is a list of ``(port, minimum)`` tuples. FORCE_PORTS = [(443, 1)] @@ -300,6 +294,12 @@ HTTPS_INCLUDE_FINGERPRINTS = True # the *last* entry from its X-Forwarded-For header as the client's IP. HTTPS_USE_IP_FROM_FORWARDED_HEADER = False +# How many clusters do we group IPs in when distributing bridges based on IP? +# Note that if PROXY_LIST_FILES is set (below), what we actually do here +# is use one higher than the number here, and the extra cluster is used +# for answering requests made by IP addresses in the PROXY_LIST_FILES file. +N_IP_CLUSTERS = 4 + # (string or None) The period at which the available bridges rotates to a # separate set of bridges. This setting can be used in the form #

1 0

[bridgedb/develop] PEP8 and Sphinx documentation fixes for bridgedb.geo.getCountryCode().
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 6a3ca67a7119f0958c23298aa6b3fceeb4ad4271 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 2 21:42:18 2015 +0000 PEP8 and Sphinx documentation fixes for bridgedb.geo.getCountryCode(). --- lib/bridgedb/geo.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py index 3b48016..c986fa1 100644 --- a/lib/bridgedb/geo.py +++ b/lib/bridgedb/geo.py @@ -38,20 +38,25 @@ except Exception as err: # pragma: no cover geoip = None geoipv6 = None -def getCountryCode(IPAddr): + +def getCountryCode(ip): """Return the two-letter country code of a given IP address. - :param IPAddr: (:class:`ipaddr.IPAddress`) An IPv4 OR IPv6 address. - """ + :type ip: :class:`ipaddr.IPAddress` + :param ip: An IPv4 OR IPv6 address. + :rtype: ``None`` or str - ip = None + :returns: If the GeoIP databases are loaded, and the **ip** lookup is + successful, then this returns a two-letter country code. Otherwise, + this returns ``None``. + """ + addr = None version = None try: - ip = IPAddr.compressed - version = IPAddr.version + addr = ip.compressed + version = ip.version except AttributeError as err: - logging.warn("Wrong type passed to getCountryCode. Offending call:" - " %r" % err) + logging.warn("Wrong type passed to getCountryCode: %s" % str(err)) return None # GeoIP has two databases: one for IPv4 addresses, and one for IPv6 @@ -59,8 +64,7 @@ def getCountryCode(IPAddr): db = None # First, make sure we loaded GeoIP properly. if None in (geoip, geoipv6): - logging.warn("GeoIP databases failed to load; could not look up"\ - " country code.") + logging.warn("GeoIP databases aren't loaded; can't look up country code") return None else: if version == 4: @@ -69,10 +73,10 @@ def getCountryCode(IPAddr): db = geoipv6 # Look up the country code of the address. - countryCode = db.country_code_by_addr(ip) + countryCode = db.country_code_by_addr(addr) if countryCode: logging.debug("Looked up country code: %s" % countryCode) return countryCode else: - logging.debug("Country code was not detected. IP: %s" % ip) + logging.debug("Country code was not detected. IP: %s" % addr) return None

1 0

[bridgedb/develop] Fix docs for BridgeBackwardsCompatibility.getConfigLine() request param.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit c79290e0b009fdcd303961910db7ddcf0627c32f Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 07:37:06 2015 +0000 Fix docs for BridgeBackwardsCompatibility.getConfigLine() request param. --- lib/bridgedb/bridges.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 5c2731c..5c0afd8 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -715,10 +715,10 @@ class BridgeBackwardsCompatibility(BridgeBase): :type addressClass: :class:`ipaddr.IPv4Address` or :class:`ipaddr.IPv6Address`. :param addressClass: Type of address to choose. - :param str request: A string unique to this request e.g. email-address - or ``uniformMap(ip)`` or ``'default'``. In this case, this is not - a :class:`~bridgerequest.BridgeRequestBase` (as might be expected) - but the equivalent of + :param str request: A string (somewhat) unique to this request, + e.g. email-address or ``IPBasedDistributor.getSubnet(ip)``. In + this case, this is not a :class:`~bridgerequest.BridgeRequestBase` + (as might be expected) but the equivalent of :data:`bridgerequest.BridgeRequestBase.client`. :param str transport: A pluggable transport method name. """

1 0

[bridgedb/develop] Remove duplicate doctest runs for some modules.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 016b6e2ea26c0444bda6074087032d0d42bf7a02 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 14 08:54:58 2015 +0000 Remove duplicate doctest runs for some modules. --- lib/bridgedb/test/legacy_Tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/bridgedb/test/legacy_Tests.py b/lib/bridgedb/test/legacy_Tests.py index 4d0c53c..a039f91 100644 --- a/lib/bridgedb/test/legacy_Tests.py +++ b/lib/bridgedb/test/legacy_Tests.py @@ -9,7 +9,6 @@ separate from the production codebase. from __future__ import print_function -import doctest import os import random import sqlite3 @@ -653,13 +652,6 @@ def testSuite(): for klass in [IPBridgeDistTests, SQLStorageTests, EmailBridgeDistTests, BridgeStabilityTests]: suite.addTest(loader.loadTestsFromTestCase(klass)) - - for module in [ bridgedb.Bridges, - bridgedb.Main, - bridgedb.Dist, - bridgedb.schedule ]: - suite.addTest(doctest.DocTestSuite(module)) - return suite def main():

1 0

[bridgedb/develop] Rewrite `python setyp.py test` code to run Trial tests only.
by isis＠torproject.org 16 Apr '15

16 Apr '15

commit 382a4f5e54a26dc7a2179d17cb4129f68e3e650e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 04:23:24 2015 +0000 Rewrite `python setyp.py test` code to run Trial tests only. --- lib/bridgedb/parse/options.py | 27 ---------- lib/bridgedb/runner.py | 47 ---------------- setup.py | 119 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 84 deletions(-) diff --git a/lib/bridgedb/parse/options.py b/lib/bridgedb/parse/options.py index 2ed93d0..3979615 100644 --- a/lib/bridgedb/parse/options.py +++ b/lib/bridgedb/parse/options.py @@ -251,32 +251,6 @@ class BaseOptions(usage.Options): print(" self['rundir']=%s" % self['rundir']) -class TestOptions(BaseOptions): - """Suboptions for running twisted.trial and unittest based tests.""" - - longdesc = textwrap.dedent("""BridgeDB testing commands. - See the `bridgedb mock` command for generating testing environments.""") - - optFlags = [['coverage', 'c', 'Generate coverage statistics']] - optParameters = [ - ['file', 'f', None, 'Run tests in specific file(s) (trial only)'], - ['unittests', 'u', False, 'Run unittests in bridgedb.Tests'], - ['trial', 't', True, 'Run twisted.trial tests in bridgedb.test']] - - completionData = usage.Completions( - mutuallyExclusive=[('unittests', 'coverage'), - ('unittests', 'file')], - optActions={'file': usage.CompleteFiles('lib/bridgedb/test/test_*.py', - repeat=True, - descr="test filename")}, - extraActions=[ - usage.Completer(descr="extra arguments to pass to trial")]) - - def parseArgs(self, *args): - """Parse any additional arguments after the options and flags.""" - self['test_args'] = args - - class MockOptions(BaseOptions): """Suboptions for creating necessary conditions for testing purposes.""" @@ -315,7 +289,6 @@ class MainOptions(BaseOptions): ['dump-bridges', 'd', 'Dump bridges by hashring assignment into files'], ['reload', 'R', 'Reload bridge descriptors into running servers']] subCommands = [ - ['test', None, TestOptions, "Run twisted.trial tests or unittests"], ['mock', None, MockOptions, "Generate a testing environment"], ['SIGHUP', None, SIGHUPOptions, "Reload bridge descriptors into running servers"], diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index 7be7700..6ac069f 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -94,53 +94,6 @@ def generateDescriptors(count=None, rundir=None): del subprocess return statuscode -def runTrial(options): - """Run Twisted trial based unittests, optionally with coverage. - - :type options: :class:`~bridgedb.opt.TestOptions` - :param options: Parsed options for controlling the twisted.trial test - run. All unrecognised arguments after the known options will be passed - along to trial. - """ - from twisted.scripts import trial - - # Insert 'trial' as the first system cmdline argument: - sys.argv = ['trial'] - - if options['coverage']: - try: - from coverage import coverage - except ImportError as ie: - print(ie.message) - else: - cov = coverage() - cov.start() - sys.argv.append('--coverage') - sys.argv.append('--reporter=bwverbose') - - # Pass all arguments along to its options parser: - if 'test_args' in options: - for arg in options['test_args']: - sys.argv.append(arg) - # Tell trial to test the bridgedb package: - sys.argv.append('bridgedb.test') - trial.run() - - if options['coverage']: - cov.stop() - cov.html_report('_trial_temp/coverage/') - -def runTests(options): - """Run unittest based tests. - - :type options: :class:`~bridgedb.opt.TestOptions` - :param options: Parsed options for controlling the twisted.trial test - run. All unrecognised arguments after the known options will be passed - along to trial. - """ - testModule = __import__('bridgedb.Tests', globals(), '', []) - testModule.Tests.main() - def doDumpBridges(config): """Dump bridges by assignment to a file. diff --git a/setup.py b/setup.py index 9402071..032aa10 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ install_i18n = os.path.join('bridgedb', 'i18n') # Directory to install docs, license, and other text resources into: install_docs = os.path.join('share', 'doc', 'bridgedb') + def get_cmdclass(): """Get our cmdclass dictionary for use in setuptool.setup(). @@ -81,7 +82,7 @@ def get_cmdclass(): to add our own classes to the cmdclass dictionary, and then update that dictionary with the one returned from versioneer.get_cmdclass(). """ - cmdclass = {'test': runTests, + cmdclass = {'test': Trial, 'compile_catalog': compile_catalog, 'extract_messages': extract_messages, 'init_catalog': init_catalog, @@ -238,15 +239,69 @@ def get_data_files(filesonly=False): return data_files -class runTests(setuptools.Command): - # Based on setup.py from mixminion, which is based on setup.py - # from Zooko's pyutil package, which is in turn based on - # http://mail.python.org/pipermail/distutils-sig/2002-January/002714.html - description = "Run unit tests" - user_options = [] +class Trial(setuptools.Command): + """Twisted Trial setuptools command. + + Based on the setuptools Trial command in Zooko's Tahoe-LAFS, as well as + https://github.com/simplegeo/setuptools-trial/ (which is also based on the + Tahoe-LAFS code). + + Pieces of the original implementation of this 'test' command (that is, for + the original pyunit-based BridgeDB tests which, a long time ago, in a + galaxy far far away, lived in bridgedb.Tests) were based on setup.py from + Nick Mathewson's mixminion, which was based on the setup.py from Zooko's + pyutil package, which was in turn based on + http://mail.python.org/pipermail/distutils-sig/2002-January/002714.html. + + Crusty, old-ass Python, like hella wut. + """ + description = "Run Twisted Trial-based tests." + user_options = [ + ('debug', 'b', ("Run tests in a debugger. If that debugger is pdb, will " + "load '.pdbrc' from current directory if it exists.")), + ('debug-stacktraces', 'B', "Report Deferred creation and callback stack traces"), + ('debugger=', None, ("The fully qualified name of a debugger to use if " + "--debug is passed (default: pdb)")), + ('disablegc', None, "Disable the garbage collector"), + ('force-gc', None, "Have Trial run gc.collect() before and after each test case"), + ('jobs=', 'j', "Number of local workers to run, a strictly positive integer"), + ('profile', None, "Run tests under the Python profiler"), + ('random=', 'Z', "Run tests in random order using the specified seed"), + ('reactor=', 'r', "Which reactor to use"), + ('reporter=', None, "Customize Trial's output with a reporter plugin"), + ('rterrors', 'e', "Realtime errors: print out tracebacks as soon as they occur"), + ('spew', None, "Print an insanely verbose log of everything that happens"), + ('testmodule=', None, "Filename to grep for test cases (-*- test-case-name)"), + ('tbformat=', None, ("Specify the format to display tracebacks with. Valid " + "formats are 'plain', 'emacs', and 'cgitb' which uses " + "the nicely verbose stdlib cgitb.text function")), + ('unclean-warnings', None, "Turn dirty reactor errors into warnings"), + ('until-failure', 'u', "Repeat a test (specified by -s) until it fails."), + ('without-module=', None, ("Fake the lack of the specified modules, separated " + "with commas")), + ] + boolean_options = ['debug', 'debug-stacktraces', 'disablegc', 'force-gc', + 'profile', 'rterrors', 'spew', 'unclean-warnings', + 'until-failure'] def initialize_options(self): - pass + self.debug = None + self.debug_stacktraces = None + self.debugger = None + self.disablegc = None + self.force_gc = None + self.jobs = None + self.profile = None + self.random = None + self.reactor = None + self.reporter = None + self.rterrors = None + self.spew = None + self.testmodule = None + self.tbformat = None + self.unclean_warnings = None + self.until_failure = None + self.without_module = None def finalize_options(self): build = self.get_finalized_command('build') @@ -257,11 +312,55 @@ class runTests(setuptools.Command): self.run_command('build') old_path = sys.path[:] sys.path[0:0] = [self.build_purelib, self.build_platlib] + + result = 1 try: - testmod = __import__("bridgedb.Tests", globals(), "", []) - testmod.Tests.main() + result = self.run_tests() finally: sys.path = old_path + raise SystemExit(result) + + def run_tests(self): + # We do the import from Twisted inside the function instead of the top + # of the file because since Twisted is a setup_requires, we can't + # assume that Twisted will be installed on the user's system prior, so + # if we don't do the import here, then importing from this plugin will + # fail. + from twisted.scripts import trial + + if not self.testmodule: + self.testmodule = "bridgedb.test" + + # Handle parsing the trial options passed through the setuptools + # trial command. + cmd_options = [] + for opt in self.boolean_options: + if getattr(self, opt.replace('-', '_'), None): + cmd_options.append('--%s' % opt) + + for opt in ('debugger', 'jobs', 'random', 'reactor', 'reporter', + 'testmodule', 'tbformat', 'without-module'): + value = getattr(self, opt.replace('-', '_'), None) + if value is not None: + cmd_options.extend(['--%s' % opt, value]) + + config = trial.Options() + config.parseOptions(cmd_options) + config['tests'] = [self.testmodule,] + + trial._initialDebugSetup(config) + trialRunner = trial._makeRunner(config) + suite = trial._getSuite(config) + + # run the tests + if self.until_failure: + test_result = trialRunner.runUntilFailure(suite) + else: + test_result = trialRunner.run(suite) + + if test_result.wasSuccessful(): + return 0 # success + return 1 # failure # If there is an environment variable BRIDGEDB_INSTALL_DEPENDENCIES=0, it will

1 0