January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Make log message in FixedBridgeSplitter.insert more understandable.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit acf462a71326d979002b62e008a31b27cec59178 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 13:33:42 2013 +0000 Make log message in FixedBridgeSplitter.insert more understandable. --- lib/bridgedb/Bridges.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index d58d7f0..340ac44 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -1032,7 +1032,9 @@ class FilteredBridgeSplitter(BridgeHolder): :param bridge: The bridge to add. """ if not bridge.running: - logging.debug("insert non-running bridge %s" % bridge.getID()) + logging.warn( + "Skipping hashring insertion for non-running bridge: '%s'" + % bridge.getID()) return self.bridges.append(bridge)

1 0

[bridgedb/master] Remove a log statement from bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit cb9e7090a7e5e6a2ae7504844ad76b0d527b95b9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 05:47:21 2013 +0000 Remove a log statement from bridgedb.Main. Because of the duplicate bridge problem we've been having, this log statement was causing nearly every bridge fingerprint to get recorded to the logfiles every 30 minutes (when BridgeDB is SIGHUPed). * REMOVE a log statement. --- lib/bridgedb/Main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 964c737..27b1339 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -106,7 +106,6 @@ def load(cfg, splitter, clear=False): if bridge.getID() in bridges: logging.warn( "Parsed bridge that we've already added. Skipping.") - logging.debug(" Bridge: %s" % bridge.getID()) continue else: bridges[bridge.getID()] = bridge

1 0

[bridgedb/master] Make bridgedb.Main._reloadFn placeholder take any arguments.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 307e189927bd001c33e61a55ff9295bde46d044e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 06:05:02 2013 +0000 Make bridgedb.Main._reloadFn placeholder take any arguments. * ADD a docstring to it, not that it is no longer a lambda function. --- lib/bridgedb/Main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 29d457b..75e20b3 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -186,7 +186,10 @@ def loadProxyList(cfg): f.close() return ipset -_reloadFn = lambda x: True +def _reloadFn(*args, **kwargs): + """Placeholder callback function for :func:`_handleSIGHUP`.""" + return True + def _handleSIGHUP(*args): """Called when we receive a SIGHUP; invokes _reloadFn.""" reactor.callLater(0, _reloadFn, *args)

1 0

[bridgedb/master] Catch potential KeyErrors on bridge ID lookups in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 23f401d5e015ebfb283df1a86e82540c1507083d Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 06:00:48 2013 +0000 Catch potential KeyErrors on bridge ID lookups in bridgedb.Main. We should catch the KeyError here, as sometimes this loop gets a bridge ID from an extra-info descriptor and tries to lookup the bridge, causing a KeyError on the lookup. * ADD try/except wrapper to catch KeyErrors on looking up bridges by ID in the bridges dict in bridgedb.Main.load(). --- lib/bridgedb/Main.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 5479d6a..29d457b 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -136,12 +136,17 @@ def load(cfg, splitter, clear=False): f = open(filename, 'r') for transport in Bridges.parseExtraInfoFile(f): ID, method_name, address, port, argdict = transport - if bridges[ID].running: - logging.debug(" Appending transport to running bridge") - bridgePT = Bridges.PluggableTransport( - bridges[ID], method_name, address, port, argdict) - bridges[ID].transports.append(bridgePT) - assert bridges[ID].transports, "We added a transport but it disappeared!" + try: + if bridges[ID].running: + logging.debug(" Appending transport to running bridge") + bridgePT = Bridges.PluggableTransport( + bridges[ID], method_name, address, port, argdict) + bridges[ID].transports.append(bridgePT) + assert bridges[ID].transports, \ + "We added a transport but it disappeared!" + except KeyError as error: + logging.error("Could not find bridge with fingerprint '%s'." + % Bridges.toHex(ID)) logging.debug("Closing extra-info document") f.close() if hasattr(cfg, "COUNTRY_BLOCK_FILE"):

1 0

[bridgedb/master] Configure and start logging ASAP in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit c109f90004f889571a1cb725c11f1d332280812e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 06:16:52 2013 +0000 Configure and start logging ASAP in bridgedb.Main. This is the earliest point at which we have everything we need to call configureLogging(); we probably should not delay it, otherwise we risk losing information on anything happening earlier. * MOVE call to bridgedb.Main.configureLogging() to be as early in the code as possible. --- lib/bridgedb/Main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index cc53ecf..f87c1df 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -223,15 +223,15 @@ def startup(cfg, options): else: cfg.PROXY_LIST_FILES = [] + # Set up logging. + configureLogging(cfg) + # Write the pidfile. if cfg.PIDFILE: f = open(cfg.PIDFILE, 'w') f.write("%s\n" % os.getpid()) f.close() - # Set up logging. - configureLogging(cfg) - # Import Servers after logging is set up. Otherwise, python will create a # default handler that logs to the console and ignore further basicConfig # calls.

1 0

[bridgedb/master] Expand ~s and get the abspath for proxy list files.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit ffe6effb616746e76547a3f0b80d4adbf28b52cb Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 06:06:26 2013 +0000 Expand ~s and get the abspath for proxy list files. --- lib/bridgedb/Main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 75e20b3..b949b00 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -217,8 +217,8 @@ def startup(cfg, options): if v: setattr(cfg, key, os.path.expanduser(v)) if hasattr(cfg, "PROXY_LIST_FILES"): - cfg.PROXY_LIST_FILES = [ - os.path.expanduser(v) for v in cfg.PROXY_LIST_FILES ] + cfg.PROXY_LIST_FILES = [os.path.abspath(os.path.expanduser(fn)) + for fn in cfg.PROXY_LIST_FILES] else: cfg.PROXY_LIST_FILES = []

1 0

[bridgedb/master] Change an assert to a check+log in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 21f88bde6fa4f13ebdd16178feb22b6745cb4d9c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 06:11:27 2013 +0000 Change an assert to a check+log in bridgedb.Main. This is done so that if BridgeDB is run with higher-than-default optimization levels, i.e. ``python -O2`` (which strips out assert statements), then the problem will still get logged. * CHANGE an assert statement which checks that a ``Bridges.PluggableTransport`` added to a ``Bridges.Bridge.transports`` list actually got added, into a more explicit, non-assert check with a log statement if it fails. --- lib/bridgedb/Main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index b949b00..cc53ecf 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -142,8 +142,9 @@ def load(cfg, splitter, clear=False): bridgePT = Bridges.PluggableTransport( bridges[ID], method_name, address, port, argdict) bridges[ID].transports.append(bridgePT) - assert bridges[ID].transports, \ - "We added a transport but it disappeared!" + if not bridgePT in bridges[ID].transports: + logging.critical("""Added transport...it disappeared! + Transport: %r""" % bridgePT) except KeyError as error: logging.error("Could not find bridge with fingerprint '%s'." % Bridges.toHex(ID))

1 0

[bridgedb/master] Don't create new loggers in modules loaded into bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit d0cfeacd66cc744f44134dc13f87b323de0f4b41 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 5 07:57:45 2013 +0000 Don't create new loggers in modules loaded into bridgedb.Main. If these create their own loggers, and/or are imported as absolute imports specifying the full package name, then the new loggers get created before the root logger is configured. * REMOVE logger creation in bridgedb.parse.headers and bridgedb.crypto. --- lib/bridgedb/Main.py | 2 +- lib/bridgedb/crypto.py | 5 ++--- lib/bridgedb/parse/headers.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index f87c1df..384547e 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -16,9 +16,9 @@ import gettext from twisted.internet import reactor +from bridgedb import crypto from bridgedb.parse import options -import bridgedb.crypto import bridgedb.Bridges as Bridges import bridgedb.Dist as Dist import bridgedb.Time as Time diff --git a/lib/bridgedb/crypto.py b/lib/bridgedb/crypto.py index 124d337..70e9ee1 100644 --- a/lib/bridgedb/crypto.py +++ b/lib/bridgedb/crypto.py @@ -13,7 +13,6 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -log = logging.getLogger(__name__) import os @@ -45,7 +44,7 @@ def getKey(filename): try: fh = open(filename, 'rb') except IOError: - log.debug("getKey(): Creating new secret key.") + logging.debug("getKey(): Creating new secret key.") key = OpenSSL.rand.bytes(32) flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0) fd = os.open(filename, flags, 0400) @@ -53,7 +52,7 @@ def getKey(filename): os.fsync(fd) fd.close() else: - log.debug("getKey(): Secret key file found. Loading...") + logging.debug("getKey(): Secret key file found. Loading...") key = fh.read() fh.close() return key diff --git a/lib/bridgedb/parse/headers.py b/lib/bridgedb/parse/headers.py index d9f1c4b..6b9d371 100644 --- a/lib/bridgedb/parse/headers.py +++ b/lib/bridgedb/parse/headers.py @@ -20,7 +20,6 @@ """ import logging -log = logging.getLogger() import re import os

1 0

[bridgedb/master] Add a unittest runner to scripts/bridgedb.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit c9dc17e85b5f19b791b1a509fe65a1bb9e319b47 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Oct 3 06:29:23 2013 +0000 Add a unittest runner to scripts/bridgedb. * ADD commands 'test' and 'coverage' to scripts/bridgedb. To run twisted.trial based unittests, after BridgeDB has been installed, do: $ bridgedb test and to run the same tests under python-coverage, do: $ bridgedb coverage * ADD dependencies for unittests and coverage: - coverage - trialcoverage - sure * FIXES #9872 --- requirements.txt | 3 +++ scripts/bridgedb | 31 +++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7dc41a4..5760060 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,14 @@ Mako==0.8.1 MarkupSafe==0.18 Twisted==13.1.0 argparse>=1.2.1 +coverage==3.6 ipaddr>=2.1.10 pyOpenSSL>=0.13 pygeoip==0.2.7 pygpgme==0.3 recaptcha>=1.0rc1 recaptcha-client>=1.0.6 +sure==1.2.2 +trialcoverage==0.3.12 wsgiref>=0.1.2 zope.interface>=4.0.5 diff --git a/scripts/bridgedb b/scripts/bridgedb index 069cccd..8e8800b 100644 --- a/scripts/bridgedb +++ b/scripts/bridgedb @@ -1,4 +1,31 @@ #!/usr/bin/env python -import bridgedb.Main -bridgedb.Main.run() +import bridgedb +import sys + +if len(sys.argv) == 1: + from bridgedb import Main + Main.run() +else: + if sys.argv[1] in ('test', 'coverage'): + import os + from twisted.scripts import trial + + if sys.argv[1] == 'coverage': + run_python_coverage = True + + # Hack to set the PYTHONPATH: + sys.path[:] = map(os.path.abspath, sys.path) + sys.path.insert(0, os.path.abspath(os.getcwd())) + + if run_python_coverage: + from coverage import coverage + cov = coverage() + cov.start() + + sys.argv = ['trial', 'bridgedb'] + trial.run() + + if run_python_coverage: + cov.stop() + cov.html_report('.coverage-results')

1 0

[bridgedb/master] Initial commit
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 38e2382e4f4194050e1909539918f5253e9d7625 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Fri Aug 16 02:16:21 2013 +0000 Initial commit --- generatedescriptors.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/generatedescriptors.py b/generatedescriptors.py new file mode 100644 index 0000000..d507401 --- /dev/null +++ b/generatedescriptors.py @@ -0,0 +1,84 @@ +#!/usr/sbin/env python -tt + +import sys +import random +import time +import ipaddr +from datetime import datetime +import binascii + + +def randomIP(): + return randomIP4() + +def randomIP4(): + return ipaddr.IPAddress(random.getrandbits(32)) + +def randomPort(): + return random.randint(1,65535) + +def gettimestamp(): + ts = time.strftime("%Y-%m-%d %H:%M:%S") + return "opt published %s\n" % ts + +def getHexString(size): + s = "" + for i in xrange(size): + s+= random.choice("ABCDEF0123456789") + return s + + +baseDesc = "router Unnamed %s %s 0 9030\n"\ +"opt fingerprint %s\n"\ +"opt @purpose bridge\n" + +baseStatus = "r %s %s %s %s %s %d %d\n"\ +"s Running Stable\n" + +baseExtraInfo = "extra-info %s %s\n"\ +"transport %s %s:%d\n" + +ei = "" +df = "" +sf = "" +for i in xrange(500): + fp = "DEAD BEEF F00F DEAD BEEF F00F " + \ + getHexString(4) + " " + getHexString(4) + " " + \ + getHexString(4) + " " + getHexString(4) + ID = binascii.a2b_hex(fp.replace(" ", "")) + + sf += "".join(baseStatus % ("namedontmattah", binascii.b2a_base64(ID)[:-2], + "randomstring", time.strftime("%Y-%m-%d %H:%M:%S"), randomIP(), + randomPort(), randomPort())) + + df += "".join(baseDesc % (randomIP(), randomPort(), fp)) + df += gettimestamp() + df += "router-signature\n" + + if i % 4 == 0: + ei += "".join(baseExtraInfo % ("namedontmattah", fp.replace(" ", ""), + random.choice(["obfs2", "obfs3", "obfs2"]), + randomIP(), randomPort())) + +ei += "router-signature\n" + +try: + f = open("networkstatus-bridges", 'w') + f.write(sf) + f.close() +except: + print "Failed to open status file" + +try: + f = open("bridge-descriptors", 'w') + f.write(df) + f.close() +except: + print "Failed to open df file" + +try: + f = open("extra-infos", 'w') + f.write(ei) + f.close() +except: + print "Failed to open ei file"

1 0