January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Add additional log statements to bridgedb.Main.startup().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 5ac1057be95c44996b1a8b1288eb3bed47376f24 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:14:12 2013 +0000 Add additional log statements to bridgedb.Main.startup(). * ADD additional log statements to notify when each distributor is setup and when config options are changed on the config instance. --- lib/bridgedb/Main.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 502d754..1480c10 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -291,6 +291,7 @@ def startup(cfg, options): # Create a BridgeSplitter to assign the bridges to the different # distributors. splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key")) + logging.debug("Created splitter: %r" % splitter) # Create ring parameters. forcePorts = getattr(cfg, "FORCE_PORTS") @@ -303,9 +304,13 @@ def startup(cfg, options): emailDistributor = ipDistributor = None # As appropriate, create an IP-based distributor. if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: + logging.debug("Setting up HTTPS Distributor...") categories = [] if proxyList.ipset: + logging.debug("Adding proxyList to HTTPS Distributor categories.") categories.append(proxyList) + logging.debug("HTTPS Distributor categories: '%s'" % categories) + ipDistributor = Dist.IPBasedDistributor( Dist.uniformMap, cfg.N_IP_CLUSTERS, @@ -320,6 +325,8 @@ def startup(cfg, options): if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: for d in cfg.EMAIL_DOMAINS: cfg.EMAIL_DOMAIN_MAP[d] = d + logging.debug("New email domain map: '%s'" % cfg.EMAIL_DOMAIN_MAP) + logging.debug("Setting up Email Distributor...") emailDistributor = Dist.EmailBasedDistributor( Bridges.get_hmac(key, "Email-Dist-Key"), cfg.EMAIL_DOMAIN_MAP.copy(), @@ -357,11 +364,16 @@ def startup(cfg, options): proxyList.replaceProxyList(loadProxyList(cfg)) logging.info("%d bridges loaded", len(splitter)) if emailDistributor: + logging.debug("Prepopulating email distributor hashrings...") emailDistributor.prepopulateRings() # create default rings - logging.info("%d for email", len(emailDistributor.splitter)) + logging.info("Bridges allotted for email distribution: %d" + % len(emailDistributor.splitter)) + if ipDistributor: + logging.debug("Prepopulating HTTPS distributor hashrings...") ipDistributor.prepopulateRings() # create default rings - logging.info("%d for web:", len(ipDistributor.splitter)) + logging.info("Bridges allotted for web distribution: %d" + % len(ipDistributor.splitter)) for (n,(f,r)) in ipDistributor.splitter.filterRings.items(): logging.info("\tby filter set %s, %d" % (n, len(r))) #logging.info(" by location set: %s",

1 0

[bridgedb/master] Separate pseudo-distributor dumping into function in bridgedb.runner.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 3def4de56f19aefcb584f59af7a84d1f648b10e2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:29:01 2013 +0000 Separate pseudo-distributor dumping into function in bridgedb.runner. * MOVE code for dumping bridges in 'unassigned' pseudo-distributor rings to new function, bridgedb.runner.doDumpBridges(). --- lib/bridgedb/Main.py | 5 +---- lib/bridgedb/runner.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 924b10c..3c5aa0c 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -25,7 +25,6 @@ import bridgedb.Bridges as Bridges import bridgedb.Dist as Dist import bridgedb.Time as Time import bridgedb.Storage -import bridgedb.Bucket as Bucket import bridgedb.Util as Util @@ -473,9 +472,7 @@ def runSubcommand(options, config): from bridgedb import runner if options('dump-bridges'): - bucketManager = Bucket.BucketManager(config) - bucketManager.assignBridgesToBuckets() - bucketManager.dumpBridges() + runner.doDumpBridges(config) if options.subCommand is not None: logging.debug("Running BridgeDB command: '%s'" % options.subCommand) diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index 3ba61ec..425c0e8 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -95,3 +95,17 @@ def runTests(options): """ testModule = __import__('bridgedb.Tests', globals(), '', []) testModule.Tests.main() + +def doDumpBridges(config): + """Dump bridges by assignment to a file. + + This function handles the commandline '--dump-bridges' option. + + :type config: :class:`bridgedb.Main.Conf` + :param config: The current configuration. + """ + import bridgedb.Bucket as bucket + + bucketManager = bucket.BucketManager(config) + bucketManager.assignBridgesToBuckets() + bucketManager.dumpBridges()

1 0

[bridgedb/master] Add docstring and use loadConfig() in reload().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 4607e64ad63584343eb9fbcbf4048a7a9b3a6c8e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:18:32 2013 +0000 Add docstring and use loadConfig() in reload(). --- lib/bridgedb/Main.py | 52 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 1480c10..750ad1e 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -346,19 +346,47 @@ def startup(cfg, options): for p in cfg.FILE_BUCKETS.keys(): splitter.addPseudoRing(p) - # Make the parse-bridges function get re-called on SIGHUP. def reload(*args): - logging.info("Caught SIGHUP") - - # reparse the config file - configuration = {} - if options['config']: - execfile(options['config'], configuration) - cfg = Conf(**configuration) - # update loglevel on (re)load - level = getattr(cfg, 'LOGLEVEL', 'WARNING') - level = getattr(logging, level) - logging.getLogger().setLevel(level) + """Reload settings, proxy lists, and bridges. + + The contents of the config file should be compiled (it's roughly 20-30 + times faster to use the ``compile`` builtin on a string before + ``exec``ing it) first, and then ``exec``ed -- not ``execfile``! -- in + order to get the contents of the config file to exist within the scope + of the configuration object. Otherwise, Python *will* default to + placing them directly within the ``globals()`` scope. + + For a more detailed explanation, see http://stackoverflow.com/q/17470193 + and http://lucumr.pocoo.org/2011/2/1/exec-in-python/ + + :type cfg: :class:`Conf` + :param cfg: The current configuration, including any in-memory + settings (i.e. settings whose values were not obtained from the + config file, but were set via a function somewhere) + :type options: :class:`twisted.python.usage.Options` + :param options: Any commandline options. + + :param splitter: XXX + + :type proxyList: :class:`ProxyCategory` + :param proxyList: The container for the IP addresses of any currently + known open proxies. + + :param IPDistributor: A :class:`Dist.IPBasedDistributor`. + :param emailDistributor: A :class:`Dist.EmailDistributor`. + :param dict tasks: A dictionary of {'name': Task}, where Task is some + scheduled event, repetitive or otherwise, for the :class:`reactor + <twisted.internet.epollreactor.EPollReactor>` + """ + logging.debug("Caught SIGHUP") + logging.info("Reloading...") + + cfg = loadConfig(options, cfg) + + # update loglevel on (re)load + level = getattr(cfg, 'LOGLEVEL', 'WARNING') + level = getattr(logging, level) + logging.getLogger().setLevel(level) load(cfg, splitter, clear=True) proxyList.replaceProxyList(loadProxyList(cfg))

1 0

[bridgedb/master] Remove commented out logs, add warning log, in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 5cf66a12797897015672eb9b475641832660f00b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:35:22 2013 +0000 Remove commented out logs, add warning log, in bridgedb.Main. * REMOVE cruft code for a bunch of log statements which used str.join() incorrectly and have been commented out forever anyway. * ADD warning log message if no HTTPS distributor was created. --- lib/bridgedb/Main.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 3c5aa0c..2ab1f5a 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -409,14 +409,8 @@ def startup(cfg, options): % len(ipDistributor.splitter)) for (n,(f,r)) in ipDistributor.splitter.filterRings.items(): logging.info("\tby filter set %s, %d" % (n, len(r))) - #logging.info(" by location set: %s", - # " ".join(str(len(r)) for r in ipDistributor.rings)) - #logging.info(" by category set: %s", - # " ".join(str(len(r)) for r in ipDistributor.categoryRings)) - #logging.info("Here are all known bridges in the category section:") - #for r in ipDistributor.categoryRings: - # for name, b in r.bridges.items(): - # logging.info("%s" % b.getConfigLine(True)) + else: + logging.warn("No HTTP(S) distributor created!") # Dump bridge pool assignments to disk. try:

1 0

[bridgedb/master] Move subCommand handling code to separate function in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 95988603be4eb7202235382371071710760587a2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:22:26 2013 +0000 Move subCommand handling code to separate function in bridgedb.Main. * MOVE code for running options.subCommands and dumping bridge assignments to file to a new function, bridgedb.Main.runSubcommand(). --- lib/bridgedb/Main.py | 49 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 750ad1e..924b10c 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -265,7 +265,13 @@ def startup(cfg, options): # Set up logging. configureLogging(cfg) - # Write the pidfile. + if options['dump-bridges'] or (options.subCommand is not None): + runSubcommand(options, cfg) + + # Write the pidfile only after any options.subCommands are run (because + # these exit when they are finished). Otherwise, if there is a subcommand, + # the real PIDFILE would get overwritten with the PID of the temporary + # bridgedb process running the subcommand. if cfg.PIDFILE: f = open(cfg.PIDFILE, 'w') f.write("%s\n" % os.getpid()) @@ -450,6 +456,40 @@ def startup(cfg, options): os.unlink(cfg.PIDFILE) sys.exit() +def runSubcommand(options, config): + """Run a subcommand from the 'Commands' section of the bridgedb help menu. + + :type options: :class:`bridgedb.opt.MainOptions` + :param options: A pre-parsed options class containing any arguments and + options given in the commandline we were called with. + :type config: :class:`bridgedb.Main.Conf` + :param config: The current configuration. + :raises: :exc:`SystemExit` when all subCommands and subOptions have + finished running. + """ + # Make sure that the runner module is only imported after logging is set + # up, otherwise we run into the same logging configuration problem as + # mentioned above with the EmailServer and HTTPServer. + from bridgedb import runner + + if options('dump-bridges'): + bucketManager = Bucket.BucketManager(config) + bucketManager.assignBridgesToBuckets() + bucketManager.dumpBridges() + + if options.subCommand is not None: + logging.debug("Running BridgeDB command: '%s'" % options.subCommand) + + if 'descriptors' in options.subOptions: + runner.generateDescriptors(options) + + if options.subCommand == 'test': + if options.subOptions['trial']: + runner.runTrial(options.subOptions) + if options.subOptions['unittests']: + runner.runTests(options.subOptions) + raise SystemExit("Subcommand '%s' finished." % options.subCommand) + def run(options): """This is the main entry point into BridgeDB. @@ -484,9 +524,4 @@ def run(options): # Store the rundir in case it needs to be used again later: cfg.RUNDIR = rundir - if options['dump-bridges']: - bucketManager = Bucket.BucketManager(config) - bucketManager.assignBridgesToBuckets() - bucketManager.dumpBridges() - else: - startup(config, options) + startup(cfg, options)

1 0

[bridgedb/master] Add a 'clean' directive to the Makefile.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit a47245cc4831615c63e433e4b3ef43dfc2fac8d6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:41:33 2013 +0000 Add a 'clean' directive to the Makefile. --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 29aed8f..4c49a9b 100644 --- a/Makefile +++ b/Makefile @@ -33,3 +33,9 @@ reinstall: uninstall force-install translations: ./maint/get-completed-translations + +clean: + -rm -rf build + -rm -rf dist + -rm -rf lib/bridgedb.egg-info + -rm -rf _trial_temp

1 0

[bridgedb/master] Change runner.generateDescriptors() to not need whole options class.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 384775613778ae6994ccedb365530ca14491b9bb Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:39:46 2013 +0000 Change runner.generateDescriptors() to not need whole options class. * CHANGE the parameters to bridgedb.runner.generateDescriptors() function to only need specific settings, rather than needing the whole instance of the MainOptions class. --- lib/bridgedb/Main.py | 3 ++- lib/bridgedb/runner.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 2ab1f5a..4f5203f 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -472,7 +472,8 @@ def runSubcommand(options, config): logging.debug("Running BridgeDB command: '%s'" % options.subCommand) if 'descriptors' in options.subOptions: - runner.generateDescriptors(options) + runner.generateDescriptors(options.subOptions['descriptors'], + config.RUNDIR) if options.subCommand == 'test': if options.subOptions['trial']: diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index 425c0e8..b477c35 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -19,7 +19,7 @@ from __future__ import print_function import sys -def generateDescriptors(options): +def generateDescriptors(count=None, rundir=None): """Run a script which creates fake bridge descriptors for testing purposes. This will run scripts/gen_bridge_descriptors to create a bridge router @@ -31,11 +31,12 @@ def generateDescriptors(options): :returns: The process returncode from the gen_bridge_descriptors script. """ import subprocess + import os.path proc = None - rundir = options['rundir'] script = 'gen_bridge_descriptors' - count = options.subOptions['descriptors'] + rundir = rundir if os.path.isdir(rundir) else None + count = count if count else 3 try: proc = subprocess.Popen([script, '-n', str(count)], close_fds=True, cwd=rundir)

1 0

[bridgedb/master] Update docstring for bridgedb.runner.generateDescriptors().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 5fff432beae935dd76443e86e064bfcad3f54335 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:41:56 2013 +0000 Update docstring for bridgedb.runner.generateDescriptors(). --- lib/bridgedb/runner.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index b477c35..1f30dfb 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -22,13 +22,23 @@ import sys def generateDescriptors(count=None, rundir=None): """Run a script which creates fake bridge descriptors for testing purposes. - This will run scripts/gen_bridge_descriptors to create a bridge router - descriptors, bridge extrainfo descriptors, a consensus document, and a - networkstatus document. - - :param integer howmany: Number of fake bridges to generate descriptor for. - :rtype: integer - :returns: The process returncode from the gen_bridge_descriptors script. + This will run ``scripts/gen_bridge_descriptors`` to create bridge server + descriptors, bridge extra-info descriptors, and networkstatus document. + + ** NOTE: ** + This function can take a very long time to run, especially in headless + environments where entropy sources are minimal, because it creates the + keys for each mocked OR, which are embedded in the server descriptors, + used to calculate the OR fingerprints, and sign the descriptors, among + other things. + + :param integer count: Number of mocked bridges to generate descriptor + for. (default: 3) + :type rundir: string or None + :param rundir: If given, use this directory as the current working + directory for the bridge descriptor generator script to run in. The + directory MUST already exist, and the descriptor files will be created + in it. If None, use the whatever directory we are currently in. """ import subprocess import os.path

1 0

[bridgedb/master] Remove subCommand handling code from scripts/bridgedb.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit be700c03fe0b09d95cec08640f1cb78ddd7f7f54 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:47:45 2013 +0000 Remove subCommand handling code from scripts/bridgedb. Also, minimize imports in scripts/bridgedb to precisely what is needed, so as not to risk errors caused by something being imported too early. --- scripts/bridgedb | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/scripts/bridgedb b/scripts/bridgedb index e121bd2..7afbed9 100644 --- a/scripts/bridgedb +++ b/scripts/bridgedb @@ -11,33 +11,21 @@ from __future__ import print_function -import bridgedb -import os +import os.path import sys -from bridgedb import runner -from bridgedb.parse import options +from os import getcwd +from bridgedb.Main import run +from bridgedb.parse.options import parseOptions -option = options.parseOptions() + +option = parseOptions() if option.subCommand is not None: # Hack to set the PYTHONPATH: sys.path[:] = map(os.path.abspath, sys.path) - sys.path.insert(0, os.path.abspath(os.getcwd())) - sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '../lib'))) - - if 'descriptors' in option.subOptions: - runner.generateDescriptors(option) - - if option.subCommand == 'test': - if option.subOptions['trial']: - runner.runTrial(option.subOptions) - if option.subOptions['unittests']: - runner.runTests(option.subOptions) - sys.exit() - -# If no suboption was given, just run bridgedb: -else: - from bridgedb import Main - Main.run(option) + sys.path.insert(0, os.path.abspath(getcwd())) + sys.path.insert(0, os.path.abspath(os.path.join(getcwd(), '../lib'))) + +run(option)

1 0

[bridgedb/master] Flush after writing bridge assignments to file.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 8f06618812b769b93eceb9eb78fc2026d2e76b8c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 08:42:27 2013 +0000 Flush after writing bridge assignments to file. --- lib/bridgedb/Main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 4f5203f..0d7200a 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -420,6 +420,7 @@ def startup(cfg, options): f.write("bridge-pool-assignment %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")) splitter.dumpAssignments(f) + f.flush() f.close() except IOError: logging.info("I/O error while writing assignments")

1 0