April 2014 - tor-commits - lists.torproject.org

[tor/master] Making entire exit policy available to Tor controller.
by nickm＠torproject.org 08 Apr '14

08 Apr '14

commit 51e13cd1ad5f7c130521e2a964ea462f35d1880e Author: rl1987 <nobody(a)nowhere.int> Date: Fri Aug 23 21:06:42 2013 +0300 Making entire exit policy available to Tor controller. --- changes/ticket7952 | 4 +++ src/or/control.c | 3 ++ src/or/policies.c | 23 ++++++++++++ src/or/router.c | 71 ++++++++++++++++++++++++++++-------- src/or/router.h | 3 ++ src/test/include.am | 1 + src/test/test.c | 2 ++ src/test/test_policy.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 186 insertions(+), 14 deletions(-) diff --git a/changes/ticket7952 b/changes/ticket7952 new file mode 100644 index 0000000..8ba20dc --- /dev/null +++ b/changes/ticket7952 @@ -0,0 +1,4 @@ + o Minor features (controller): + - Make the entire exit policy available from the control port via + GETINFO exit-policy/*. Implements enhancement #7952. Patch from + "rl1987". diff --git a/src/or/control.c b/src/or/control.c old mode 100644 new mode 100755 index 7034605..a6caf0d --- a/src/or/control.c +++ b/src/or/control.c @@ -2201,6 +2201,9 @@ static const getinfo_item_t getinfo_items[] = { "v3 Networkstatus consensus as retrieved from a DirPort."), ITEM("exit-policy/default", policies, "The default value appended to the configured exit policy."), + ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), + ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), + ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), { NULL, NULL, NULL, 0 } }; diff --git a/src/or/policies.c b/src/or/policies.c index be4da55..d5bd22d 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -13,6 +13,7 @@ #include "dirserv.h" #include "nodelist.h" #include "policies.h" +#include "router.h" #include "routerparse.h" #include "geoip.h" #include "ht.h" @@ -1680,6 +1681,28 @@ getinfo_helper_policies(control_connection_t *conn, (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); + } else if (!strcmpstart(question, "exit-policy/")) { + const routerinfo_t *me = router_get_my_routerinfo(); + + int include_ipv4 = 0; + int include_ipv6 = 0; + + if (!strcmp(question, "exit-policy/ipv4")) { + include_ipv4 = 1; + } else if (!strcmp(question, "exit-policy/ipv6")) { + include_ipv6 = 1; + } else if (!strcmp(question, "exit-policy/full")) { + include_ipv4 = include_ipv6 = 1; + } else { + return 0; /* No such key. */ + } + + if (!me) { + *errmsg = "router_get_my_routerinfo returned NULL"; + return -1; + } + + *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); } return 0; } diff --git a/src/or/router.c b/src/or/router.c old mode 100644 new mode 100755 index 1063eda..5465128 --- a/src/or/router.c +++ b/src/or/router.c @@ -2403,20 +2403,13 @@ router_dump_router_to_string(routerinfo_t *router, if (!router->exit_policy || !smartlist_len(router->exit_policy)) { smartlist_add(chunks, tor_strdup("reject *:*\n")); } else if (router->exit_policy) { - int i; - for (i = 0; i < smartlist_len(router->exit_policy); ++i) { - char pbuf[POLICY_BUF_LEN]; - addr_policy_t *tmpe = smartlist_get(router->exit_policy, i); - int result; - if (tor_addr_family(&tmpe->addr) == AF_INET6) - continue; /* Don't include IPv6 parts of address policy */ - result = policy_write_item(pbuf, POLICY_BUF_LEN, tmpe, 1); - if (result < 0) { - log_warn(LD_BUG,"descriptor policy_write_item ran out of room!"); - goto err; - } - smartlist_add_asprintf(chunks, "%s\n", pbuf); - } + char *exit_policy = router_dump_exit_policy_to_string(router,1,0); + + if (!exit_policy) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", exit_policy); + tor_free(exit_policy); } if (router->ipv6_exit_policy) { @@ -2483,6 +2476,56 @@ router_dump_router_to_string(routerinfo_t *router, return output; } +/** + * OR only: Given router, produce a string with its exit policy. + * If include_ipv4 is true, include IPv4 entries. + * If include_ipv6 is true, include IPv6 entries. + */ +char * +router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6) +{ + smartlist_t *exit_policy_strings; + char *policy_string = NULL; + + if ((!router->exit_policy) || (router->policy_is_reject_star)) { + return tor_strdup("reject *:*"); + } + + exit_policy_strings = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(router->exit_policy, addr_policy_t *, tmpe) { + char *pbuf; + int bytes_written_to_pbuf; + if ((tor_addr_family(&tmpe->addr) == AF_INET6) && (!include_ipv6)) { + continue; /* Don't include IPv6 parts of address policy */ + } + if ((tor_addr_family(&tmpe->addr) == AF_INET) && (!include_ipv4)) { + continue; /* Don't include IPv4 parts of address policy */ + } + + pbuf = tor_malloc(POLICY_BUF_LEN); + bytes_written_to_pbuf = policy_write_item(pbuf,POLICY_BUF_LEN, tmpe, 1); + + if (bytes_written_to_pbuf < 0) { + log_warn(LD_BUG, "router_dump_exit_policy_to_string ran out of room!"); + tor_free(pbuf); + goto done; + } + + smartlist_add(exit_policy_strings,pbuf); + } SMARTLIST_FOREACH_END(tmpe); + + policy_string = smartlist_join_strings(exit_policy_strings, "\n", 0, NULL); + + done: + SMARTLIST_FOREACH(exit_policy_strings, char *, str, tor_free(str)); + smartlist_free(exit_policy_strings); + + return policy_string; +} + /** Copy the primary (IPv4) OR port (IP address and TCP port) for * router into *ap_out. */ void diff --git a/src/or/router.h b/src/or/router.h index 1079ec7..150d107 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -97,6 +97,9 @@ int router_pick_published_address(const or_options_t *options, uint32_t *addr); int router_rebuild_descriptor(int force); char *router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key); +char *router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6); void router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *addr_port_out); void router_get_pref_orport(const routerinfo_t *router, diff --git a/src/test/include.am b/src/test/include.am index e3f2795..c6ef7ef 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -36,6 +36,7 @@ src_test_test_SOURCES = \ src/test/test_socks.c \ src/test/test_util.c \ src/test/test_config.c \ + src/test/test_policy.c \ src/ext/tinytest.c src_test_test_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) diff --git a/src/test/test.c b/src/test/test.c index f895563..d21beb9 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1572,6 +1572,7 @@ extern struct testcase_t cell_queue_tests[]; extern struct testcase_t options_tests[]; extern struct testcase_t socks_tests[]; extern struct testcase_t extorport_tests[]; +extern struct testcase_t exit_policy_tests[]; static struct testgroup_t testgroups[] = { { "", test_array }, @@ -1593,6 +1594,7 @@ static struct testgroup_t testgroups[] = { { "circuitmux/", circuitmux_tests }, { "options/", options_tests }, { "extorport/", extorport_tests }, + { "policy/" , exit_policy_tests }, END_OF_GROUPS }; diff --git a/src/test/test_policy.c b/src/test/test_policy.c new file mode 100644 index 0000000..8f7fa25 --- /dev/null +++ b/src/test/test_policy.c @@ -0,0 +1,93 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "router.h" +#include "routerparse.h" +#include "policies.h" +#include "test.h" + +static void +test_dump_exit_policy_to_string(void *arg) +{ + char *ep; + addr_policy_t *policy_entry; + + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + + (void)arg; + + ri->policy_is_reject_star = 1; + ri->exit_policy = NULL; // expecting "reject *:*" + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("reject *:*",ep); + + tor_free(ep); + + ri->exit_policy = smartlist_new(); + ri->policy_is_reject_star = 0; + + policy_entry = router_parse_addr_policy_item_from_string("accept *:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*",ep); + + tor_free(ep); + + policy_entry = router_parse_addr_policy_item_from_string("reject *:25",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25",ep); + + tor_free(ep); + + policy_entry = + router_parse_addr_policy_item_from_string("reject 8.8.8.8:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*",ep); + + policy_entry = + router_parse_addr_policy_item_from_string("reject6 [FC00::]/7:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*\n" + "reject6 [fc00::]/7:*",ep); + + policy_entry = + router_parse_addr_policy_item_from_string("accept6 [c000::]/3:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*\n" + "reject6 [fc00::]/7:*\naccept6 [c000::]/3:*",ep); + + done: + + SMARTLIST_FOREACH(ri->exit_policy, addr_policy_t *, + entry, addr_policy_free(entry)); + tor_free(ri); + tor_free(ep); +} + +struct testcase_t exit_policy_tests[] = { + { "router_dump_exit_policy_to_string", test_dump_exit_policy_to_string, 0, + NULL, NULL }, + END_OF_TESTCASES +}; +

1 0

[translation/tor-launcher-properties_completed] Update translations for tor-launcher-properties_completed
by translation＠torproject.org 08 Apr '14

08 Apr '14

commit 686d83655351b03a3228f6c7eaa9468c2e01045a Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 8 17:45:33 2014 +0000 Update translations for tor-launcher-properties_completed --- uk/torlauncher.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/uk/torlauncher.properties b/uk/torlauncher.properties index 8377c9c..4045c47 100644 --- a/uk/torlauncher.properties +++ b/uk/torlauncher.properties @@ -55,3 +55,4 @@ torlauncher.bootstrapWarning.identity=невідповідність ідент torlauncher.bootstrapWarning.timeout=тайм-аут з'єднання torlauncher.bootstrapWarning.noroute=не вказаний шлях до хосту torlauncher.bootstrapWarning.ioerror=помилка читання / запису +torlauncher.bootstrapWarning.pt_missing=пропущений підключуваний транспорт

1 0

[translation/tor-launcher-properties] Update translations for tor-launcher-properties
by translation＠torproject.org 08 Apr '14

08 Apr '14

commit 9652c9ed25f6b8e965251ccd5e2f2cbcbde8764f Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 8 17:45:30 2014 +0000 Update translations for tor-launcher-properties --- uk/torlauncher.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uk/torlauncher.properties b/uk/torlauncher.properties index 1c7b6b7..4045c47 100644 --- a/uk/torlauncher.properties +++ b/uk/torlauncher.properties @@ -55,4 +55,4 @@ torlauncher.bootstrapWarning.identity=невідповідність ідент torlauncher.bootstrapWarning.timeout=тайм-аут з'єднання torlauncher.bootstrapWarning.noroute=не вказаний шлях до хосту torlauncher.bootstrapWarning.ioerror=помилка читання / запису -torlauncher.bootstrapWarning.pt_missing=missing pluggable transport +torlauncher.bootstrapWarning.pt_missing=пропущений підключуваний транспорт

1 0

[tor/master] Drop MAX_REND_FAILURES to 8
by nickm＠torproject.org 08 Apr '14

08 Apr '14

commit 2f59d6e2d81cea3005e488e31f824ff1b9b303c4 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 2 15:36:13 2014 -0400 Drop MAX_REND_FAILURES to 8 --- changes/bug4241 | 3 +++ src/or/rendservice.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes/bug4241 b/changes/bug4241 new file mode 100644 index 0000000..a7d34bf --- /dev/null +++ b/changes/bug4241 @@ -0,0 +1,3 @@ + o Minor bugfixes: + - Only retry attempts to connect to a chosen rendezvous point 8 times, + not 30. Fixes bug #4241; bugfix on 0.1.0.1-rc. diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 500efaf..abd78da 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -82,7 +82,7 @@ typedef struct rend_service_port_config_t { #define MAX_INTRO_CIRCS_PER_PERIOD 10 /** How many times will a hidden service operator attempt to connect to * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 30 +#define MAX_REND_FAILURES 8 /** How many seconds should we spend trying to connect to a requested * rendezvous point before giving up? */ #define MAX_REND_TIMEOUT 30

1 0

[tor/master] Merge remote-tracking branch 'public/bug4241'
by nickm＠torproject.org 08 Apr '14

08 Apr '14

commit 3ac426afe8f169cb2c16de9d37b8630b7c3216f5 Merge: fffc59b 2f59d6e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 8 12:41:03 2014 -0400 Merge remote-tracking branch 'public/bug4241' changes/bug4241 | 3 +++ src/or/rendservice.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[bridgedb/develop] Add config options for safelogging and logging threads.
by isis＠torproject.org 08 Apr '14

08 Apr '14

commit 292ac8e0b1baa5fde56ef7f9869dd19eae29599d Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 11 13:54:53 2014 +0000 Add config options for safelogging and logging threads. * ADD several new config options: - LOG_THREADS: adds the thread name and id, if available, to log messages. - LOG_TRACE: log the line number, module name, and function name where the log call was made. - LOG_TIME_FORMAT: Useful for when you don't need to be told that it's Feb 11th over and over again. --- bridgedb.conf | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 006f715..0b27adc 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -15,11 +15,15 @@ # for details. # :copyright: (c) 2007-2013 The Tor Project, Inc. # (c) 2007-2013, all sentient entities within the AUTHORS file -# :version: 0.0.6 +# :version: 0.0.7 #=============================================================================== # # CHANGELOG: -# ~~~~~~~~~~ +# ~~~~~~~~~~ +# Changes in version 0.0.7 - 2014-03-31 +# * ADD new settings for tracing function calls and thread info within logged +# messages: LOG_THREADS, LOG_TRACE, and LOG_TIME_FORMAT. +# # Changes in version 0.0.6 - 2014-03-28 # * CHANGE gimp-captchas to be the norm. # * ADD bucket for support team. @@ -133,6 +137,16 @@ SAFELOGGING = True LOGFILE_COUNT = 5 LOGFILE_ROTATE_SIZE = 10000000 +# If True, include thread IDs and numbers in log messages, if available: +LOG_THREADS = False + +# If True, include the module name, function name, and line number of the +# calling function where the log message was generated: +LOG_TRACE = True + +# A strftime(3) format string that specifies how to log timestamps: +LOG_TIME_FORMAT = "%H:%M:%S" + #--------------------------- # Database/Parsing Options \ #------------------------------------------------------------------------------

1 0

[bridgedb/develop] Move manual safelog setting and functions to bridgedb.safelog module.
by isis＠torproject.org 08 Apr '14

08 Apr '14

commit b5734531a5d1e8789638a7c1d8cb45e8650fd3af Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 7 11:23:47 2014 +0000 Move manual safelog setting and functions to bridgedb.safelog module. * MOVE util.safe_logging → safelog.safe_logging. * MOVE util.set_safe_logging → safelog.setSafeLogging * MOVE util.logSafely → safelog.logSafely * REMOVE manual calls to `logSafely()` for values which are automatically scrubbed. --- lib/bridgedb/Bridges.py | 10 +++++----- lib/bridgedb/Dist.py | 23 +++++++++++------------ lib/bridgedb/EmailServer.py | 24 ++++++++++++------------ lib/bridgedb/HTTPServer.py | 15 +++++++-------- lib/bridgedb/Main.py | 17 +++++++++-------- lib/bridgedb/safelog.py | 1 + lib/bridgedb/util.py | 9 --------- 7 files changed, 45 insertions(+), 54 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index a60ff85..3ed1eff 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -23,10 +23,10 @@ import random import bridgedb.Storage import bridgedb.Bucket -from bridgedb import util from bridgedb.crypto import getHMACFunc from bridgedb.parse import addr from bridgedb.parse import networkstatus +from bridgedb.safelog import logSafely try: from cStringIO import StringIO @@ -1035,7 +1035,7 @@ class BridgeRing(BridgeHolder): else: logging.debug( "Got duplicate bridge %r in main hashring for position %r." - % (util.logSafely(k.encode('hex')), pos.encode('hex'))) + % (logSafely(k.encode('hex')), pos.encode('hex'))) keys = keys[:N] keys.sort() @@ -1280,12 +1280,12 @@ class FilteredBridgeSplitter(BridgeHolder): if not bridge.running: logging.warn( "Skipping hashring insertion for non-running bridge: '%s'" - % util.logSafely(bridge.fingerprint)) + % logSafely(bridge.fingerprint)) return index = 0 logging.debug("Inserting %s into splitter" - % (util.logSafely(bridge.fingerprint))) + % (logSafely(bridge.fingerprint))) for old_bridge in self.bridges[:]: if bridge.fingerprint == old_bridge.fingerprint: self.bridges[index] = bridge @@ -1297,7 +1297,7 @@ class FilteredBridgeSplitter(BridgeHolder): if filterFn(bridge): subring.insert(bridge) logging.debug("Inserted bridge '%s' into '%s' sub hashring" - % (util.logSafely(bridge.fingerprint), ringname)) + % (logSafely(bridge.fingerprint), ringname)) def extractFilterNames(self, ringname): """Get the names of the filters applied to a particular sub hashring. diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index c6e8756..38ca36d 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -22,13 +22,13 @@ from ipaddr import IPAddress import bridgedb.Bridges import bridgedb.Storage -from bridgedb import util from bridgedb.crypto import getHMAC from bridgedb.crypto import getHMACFunc from bridgedb.Filters import filterAssignBridgesToRing from bridgedb.Filters import filterBridgesByRules from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 +from bridgedb.safelog import logSafely def uniformMap(ip): @@ -237,7 +237,7 @@ class IPBasedDistributor(Distributor): for an example of how this is used. """ logging.info("Attempting to return %d bridges to client %s..." - % (N, util.logSafely(ip))) + % (N, ip)) if not bridgeFilterRules: bridgeFilterRules=[] @@ -253,7 +253,7 @@ class IPBasedDistributor(Distributor): area = self.areaMapper(ip) logging.debug("IP mapped to area:\t%s" - % util.logSafely("{0}.0/24".format(area))) + % logSafely("{0}.0/24".format(area))) key1 = '' pos = 0 @@ -269,7 +269,7 @@ class IPBasedDistributor(Distributor): len(self.categories), n) bridgeFilterRules.append(g) - logging.info("category<%s>%s", epoch, util.logSafely(area)) + logging.info("category<%s>%s", epoch, logSafely(area)) pos = self.areaOrderHmac("category<%s>%s" % (epoch, area)) key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % n) @@ -477,8 +477,9 @@ class EmailBasedDistributor(Distributor): try: emailaddress = normalizeEmail(emailaddress, self.domainmap, self.domainrules) - except BadEmail: - return [] #XXXX log the exception + except BadEmail as err: + logging.warn(err) + return [] if emailaddress is None: return [] #XXXX raise an exception. @@ -487,19 +488,17 @@ class EmailBasedDistributor(Distributor): lastSaw = db.getEmailTime(emailaddress) logging.info("Attempting to return for %d bridges for %s..." - % (N, util.logSafely(emailaddress))) + % (N, emailaddress)) if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now: logging.info("Client %s sent duplicate request within %d seconds." - % (util.logSafely(emailaddress), MAX_EMAIL_RATE)) + % (emailaddress, MAX_EMAIL_RATE)) if wasWarned: logging.info( "Client was already warned about duplicate requests.") - raise IgnoreEmail("Client was warned", - util.logSafely(emailaddress)) + raise IgnoreEmail("Client was warned", emailaddress) else: - logging.info("Sending duplicate request warning to %s..." - % util.logSafely(emailaddress)) + logging.info("Sending duplicate request warning.") db.setWarnedEmail(emailaddress, True, now) db.commit() diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index 599b446..f9f43bb 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -26,10 +26,10 @@ from twisted.mail import smtp from zope.interface import implements -import bridgedb.util as util from bridgedb.Dist import BadEmail, TooSoonEmail, IgnoreEmail from bridgedb import Dist from bridgedb import I18n +from bridgedb import safelog from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByTransport @@ -104,13 +104,13 @@ def getMailResponse(lines, ctx): return None, None if not addrdomain: - logging.info("Couldn't parse domain from %r", util.logSafely(clientAddr)) + logging.info("Couldn't parse domain from %r" % clientAddr) if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP: addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain) if addrdomain not in ctx.cfg.EMAIL_DOMAINS: - logging.info("Unrecognized email domain %r", util.logSafely(addrdomain)) + logging.warn("Unrecognized email domain %r", addrdomain) return None, None rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, []) @@ -188,7 +188,7 @@ def getMailResponse(lines, ctx): # Handle rate limited email except TooSoonEmail as err: logging.info("Got a mail too frequently; warning '%s': %s." - % (util.logSafely(clientAddr), err)) + % (clientAddr, err)) # Compose a warning email # MAX_EMAIL_RATE is in seconds, convert to hours @@ -198,12 +198,12 @@ def getMailResponse(lines, ctx): except IgnoreEmail as err: logging.info("Got a mail too frequently; ignoring '%s': %s." - % (util.logSafely(clientAddr), err)) + % (clientAddr, err)) return None, None except BadEmail as err: logging.info("Got a mail from a bad email address '%s': %s." - % (util.logSafely(clientAddr), err)) + % (clientAddr, err)) return None, None if bridges: @@ -273,8 +273,8 @@ def replyToMail(lines, ctx): sendToUser, response = getMailResponse(lines, ctx) if response is None: - logging.debug("getMailResponse() said not to reply to %s, so I won't." - % util.logSafely(sendToUser)) + logging.debug("We don't really feel like talking to %s right now." + % sendToUser) return response.seek(0) @@ -283,7 +283,7 @@ def replyToMail(lines, ctx): factory = smtp.SMTPSenderFactory(ctx.smtpFromAddr, sendToUser, response, d, retries=0, timeout=30) d.addErrback(_ebReplyToMailFailure) - logging.info("Sending reply to %r", util.logSafely(sendToUser)) + logging.info("Sending reply to %s" % sendToUser) reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) return d @@ -355,7 +355,7 @@ class MailMessage(object): def lineReceived(self, line): """Called when we get another line of an incoming message.""" self.nBytes += len(line) - if not util.safe_logging: + if not safelog.safe_logging: logging.debug("> %s", line.rstrip("\r\n")) if self.nBytes > self.ctx.maximumSize: self.ignoring = True @@ -467,11 +467,11 @@ def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, # Only log the email text (including all headers) if SAFE_LOGGING is # disabled: - if not util.safe_logging: + if not safelog.safe_logging: mail.seek(0) logging.debug("Email contents:\n%s" % mail.read()) else: - logging.debug("Email text for %r created." % util.logSafely(clientAddr)) + logging.debug("Email text for %r created." % clientAddr) mail.seek(0) return clientAddr, mail diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 06ae2ed..f460ac9 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -36,12 +36,12 @@ import bridgedb.I18n as I18n from bridgedb import captcha from bridgedb import crypto from bridgedb import txrecaptcha -from bridgedb import util from bridgedb.Filters import filterBridgesByIP4 from bridgedb.Filters import filterBridgesByIP6 from bridgedb.Filters import filterBridgesByTransport from bridgedb.Filters import filterBridgesByNotBlockedIn from bridgedb.parse import headers +from bridgedb.safelog import logSafely @@ -289,8 +289,8 @@ class GimpCaptchaProtectedResource(CaptchaProtectedResource): clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) valid = captcha.GimpCaptcha.check(challenge, solution, self.secretKey, clientHMACKey) - logging.debug("%sorrect captcha from %r: %r." % ( - "C" if valid else "Inc", util.logSafely(clientIP), solution)) + logging.debug("%sorrect captcha from %r: %r." + % ("C" if valid else "Inc", clientIP, solution)) return valid @@ -466,7 +466,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): remoteIP = self.getRemoteIP() logging.debug("Captcha from %r. Parameters: %r" - % (util.logSafely(clientIP), request.args)) + % (clientIP, request.args)) def checkResponse(solution, request): """Check the :class:`txrecaptcha.RecaptchaResponse`. @@ -481,12 +481,11 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): # require networking (as well as automated CAPTCHA # breaking). Hence, the 'no cover' pragma. if solution.is_valid: # pragma: no cover - logging.info("Valid CAPTCHA solution from %r." - % util.logSafely(clientIP)) + logging.info("Valid CAPTCHA solution from %r." % clientIP) return (True, request) else: logging.info("Invalid CAPTCHA solution from %r: %r" - % (util.logSafely(clientIP), solution.error_code)) + % (clientIP, solution.error_code)) return (False, request) d = txrecaptcha.submit(challenge, response, self.recaptchaPrivKey, @@ -672,7 +671,7 @@ class WebResourceBridges(resource.Resource): unblocked = False logging.info("Replying to web request from %s. Parameters were %r" - % (util.logSafely(ip), request.args)) + % (ip, request.args)) rules = [] bridgeLines = None diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 379c775..e29e991 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -20,6 +20,7 @@ from twisted.internet import reactor from bridgedb import crypto from bridgedb import persistent +from bridgedb import safelog from bridgedb import util from bridgedb.parse import options @@ -40,7 +41,7 @@ def configureLogging(cfg): logfile = getattr(cfg, 'LOGFILE', "") logfile_count = getattr(cfg, 'LOGFILE_COUNT', 5) logfile_rotate_size = getattr(cfg, 'LOGFILE_ROTATE_SIZE', 10000000) - util.set_safe_logging(safelogging) + safelog.setSafeLogging(safelogging) logging.getLogger().setLevel(level) if logfile: @@ -238,9 +239,9 @@ def loadConfig(configFile=None, configCls=None): .. _faster: http://lucumr.pocoo.org/2011/2/1/exec-in-python/ :ivar boolean itsSafeToUseLogging: This is called in :func:`startup` - before :func:`configureLogging`. When called from ``startup``, the - ``configCls`` parameter is not given, because that is the first time - that a :class:`Conf` is created. If a :class:`logging.Logger` is + before :func:`safelog.configureLogging`. When called from ``startup``, + the ``configCls`` parameter is not given, because that is the first + time that a :class:`Conf` is created. If a :class:`logging.Logger` is created in this function, then logging will not be correctly configured, therefore, if the ``configCls`` parameter is not given, then it's the first time this function has been called and it is @@ -456,10 +457,10 @@ def startup(options): # Set up logging as early as possible. We cannot import from the bridgedb # package any of our modules which import :mod:`logging` and start using - # it, at least, not until :func:`configureLogging` is called. Otherwise a - # default handler that logs to the console will be created by the imported - # module, and all further calls to :func:`logging.basicConfig` will be - # ignored. + # it, at least, not until :func:`safelog.configureLogging` is + # called. Otherwise a default handler that logs to the console will be + # created by the imported module, and all further calls to + # :func:`logging.basicConfig` will be ignored. configureLogging(config) if options['dump-bridges'] or (options.subCommand is not None): diff --git a/lib/bridgedb/safelog.py b/lib/bridgedb/safelog.py index dac65f8..eb6810b 100644 --- a/lib/bridgedb/safelog.py +++ b/lib/bridgedb/safelog.py @@ -32,6 +32,7 @@ Module Overview: :: safelog | + |_setSafeLogging - Enable or disable safelogging globally. |_logSafely - Utility for manually sanitising a portion of a log message | |_BaseSafelogFilter - Base class for log message sanitisation filters diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index 723ed8e..ebca7e1 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -11,14 +11,5 @@ """Common utilities for BridgeDB.""" -safe_logging = True -def set_safe_logging(safe): - global safe_logging - safe_logging = safe -def logSafely(val): - if safe_logging: - return "[scrubbed]" - else: - return val

1 0

[bridgedb/develop] Add module bridgedb.safelog with automatic log filters.
by isis＠torproject.org 08 Apr '14

08 Apr '14

commit 1206c811f73451d9e2a0e4c186ebcfe6a62ecaa3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Feb 12 15:24:53 2014 +0000 Add module bridgedb.safelog with automatic log filters. Include automatic filters for email addresses and IPv4/6 addresses. Things such as bridge fingerprints will still need to be manually scrubbed by calling `bridgedb.safelog.logSafely()`. --- lib/bridgedb/safelog.py | 196 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/lib/bridgedb/safelog.py b/lib/bridgedb/safelog.py new file mode 100644 index 0000000..dac65f8 --- /dev/null +++ b/lib/bridgedb/safelog.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_safelog -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# :copyright: (c) 2013-2014, Isis Lovecruft +# (c) 2007-2014, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Filters for log sanitisation. + +The ``Safelog*Filter`` classes within this module can be instantiated and +adding to any :class:`logging.Handler`, in order to transparently filter +substrings within log messages which match the given ``pattern``. Matching +substrings may be optionally additionally validated by implementing the +:meth:`~BaseSafelogFilter.doubleCheck` method before they are finally replaced +with the ``replacement`` string. For example:: + + >>> import io + >>> import logging + >>> from bridgedb import safelog + >>> handler = logging.StreamHandler(io.BytesIO()) + >>> logger = logging.getLogger() + >>> logger.addHandler(handler) + >>> logger.addFilter(safelog.SafelogEmailFilter()) + >>> logger.info("Sent response email to: blackhole(a)torproject.org") + +.. + +Module Overview: +~~~~~~~~~~~~~~~~ +:: + safelog + | + |_logSafely - Utility for manually sanitising a portion of a log message + | + |_BaseSafelogFilter - Base class for log message sanitisation filters + | |_doubleCheck - Optional stricter validation on matching substrings + | |_filter - Determine if some part of a log message should be filtered + | + |_SafelogEmailFilter - Filter for removing email addresses from logs + |_SafelogIPv6Filter - Filter for removing IPv4 addresses from logs + |_SafelogIPv6Filter - Filter for removing IPv6 addresses from logs +:: +""" + +import functools +import logging +import re + +from bridgedb.parse import addr + + +safe_logging = True + + +def setSafeLogging(safe): + """Enable or disable automatic filtering of log messages. + + :param bool safe: If ``True``, filter email and IP addresses from log + messages automagically. + """ + global safe_logging + safe_logging = safe + +def logSafely(string): + """Utility for manually sanitising a portion of a log message. + + :param str string: If ``SAFELOGGING`` is enabled, sanitise this **string** + by replacing it with ``"[scrubbed]"``. Otherwise, return the + **string** unchanged. + :rtype: str + :returns: ``"[scrubbed]"`` or the original string. + """ + if safe_logging: + return "[scrubbed]" + return string + + +class BaseSafelogFilter(logging.Filter): + """Base class for creating log message sanitisation filters. + + A :class:`BaseSafelogFilter` uses a compiled regex :cvar:`pattern` to + match particular items of data in log messages which should be sanitised + (if ``SAFELOGGING`` is enabled in :file:`bridgedb.conf`). + + .. note:: The ``pattern`` is used only for string *matching* purposes, and + *not* for validation. In other words, a ``pattern`` which matches email + addresses should simply match something which appears to be an email + address, even though that matching string might not technically be a + valid email address vis-á-vis :rfc:`5321`. + + In addition, a ``BaseSafelogFilter`` uses a :cvar:`easyFind`, which is + simply a string or character to search for before running checking against + the regular expression, to attempt to avoid regexing *everything* which + passes through the logger. + + :cvar pattern: A compiled regular expression, whose matches will be + scrubbed from log messages and replaced with :cvar:`replacement`. + :cvar easyFind: A simpler string to search for before regex matching. + :cvar replacement: The string to replace ``pattern`` matches + with. (default: ``"[scrubbed]"``) + """ + pattern = re.compile("FILTERME") + easyFind = "FILTERME" + replacement = "[scrubbed]" + + def doubleCheck(self, match): + """Subclasses should override this function to implement any additional + substring filtering to decrease the false positive rate, i.e. any + additional filtering or validation which is *more* costly than + checking against the regular expression, :cvar:`pattern`. + + To use only the :cvar:`pattern` matching in :meth:`filter`, and not + use this method, simply do:: + + return True + + :param str match: Some portion of the :ivar:`logging.LogRecord.msg` + string which has already passed the checks in :meth:`filter`, for + which additional validation/checking is required. + :rtype: bool + :returns: ``True`` if the additional validation passes (in other + words, the **match** *should* be filtered), and ``None`` or + ``False`` otherwise. + """ + return True + + def filter(self, record): + """Filter a log record. + + The log **record** is filtered, and thus sanitised by replacing + matching substrings with the :cvar:`replacement` string, if the + following checks pass: + + 0. ``SAFELOGGING`` is currently enabled. + 1. The ``record.msg`` string contains :cvar:`easyFind`. + 2. The ``record.msg`` matches the regular expression, + :cvar:`pattern`. + + :type record: :class:`logging.LogRecord` + :param record: Basically, anything passed to :func:`logging.log`. + """ + if safe_logging: + msg = str(record.msg) + if msg.find(self.easyFind) > 0: + matches = self.pattern.findall(msg) + for match in matches: + if self.doubleCheck(match): + msg = msg.replace(match, self.replacement) + record.msg = msg + return record + + +class SafelogEmailFilter(BaseSafelogFilter): + """A log filter which removes email addresses from log messages.""" + + pattern = re.compile( + "([a-zA-Z0-9]+[.+a-zA-Z0-9]*[(a)]{1}[a-zA-Z0-9]+[.-a-zA-Z0-9]*[.]{1}[a-zA-Z]+)") + easyFind = "@" + + @functools.wraps(BaseSafelogFilter.filter) + def filter(self, record): + return BaseSafelogFilter.filter(self, record) + + +class SafelogIPv4Filter(BaseSafelogFilter): + """A log filter which removes IPv4 addresses from log messages.""" + + pattern = re.compile("(?:\d{1,3}\.?){4}") + easyFind = "." + + def doubleCheck(self, match): + """Additional check to ensure that **match** is an IPv4 address.""" + if addr.isIPv4(match): + return True + + @functools.wraps(BaseSafelogFilter.filter) + def filter(self, record): + return BaseSafelogFilter.filter(self, record) + + +class SafelogIPv6Filter(BaseSafelogFilter): + """A log filter which removes IPv6 addresses from log messages.""" + + pattern = re.compile("([:]?[a-fA-F0-9:]+[:]+[a-fA-F0-9:]+){1,8}") + easyFind = ":" + + def doubleCheck(self, match): + """Additional check to ensure that **match** is an IPv6 address.""" + if addr.isIPv6(match): + return True + + @functools.wraps(BaseSafelogFilter.filter) + def filter(self, record): + return BaseSafelogFilter.filter(self, record)

1 0

[bridgedb/develop] Add bridgedb.safelog sources to Sphinx build configs.
by isis＠torproject.org 08 Apr '14

08 Apr '14

commit 6a0ed07abf32366642badd9c32c0b0a55fd37688 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 7 12:16:21 2014 +0000 Add bridgedb.safelog sources to Sphinx build configs. --- doc/sphinx/source/bridgedb.rst | 1 + doc/sphinx/source/bridgedb.safelog.rst | 9 +++++++++ doc/sphinx/source/conf.py | 1 + 3 files changed, 11 insertions(+) diff --git a/doc/sphinx/source/bridgedb.rst b/doc/sphinx/source/bridgedb.rst index 86e5abf..ac143cd 100644 --- a/doc/sphinx/source/bridgedb.rst +++ b/doc/sphinx/source/bridgedb.rst @@ -18,6 +18,7 @@ bridgedb bridgedb.parse bridgedb.persistent bridgedb.runner + bridgedb.safelog bridgedb.Stability bridgedb.Storage bridgedb.Tests diff --git a/doc/sphinx/source/bridgedb.safelog.rst b/doc/sphinx/source/bridgedb.safelog.rst new file mode 100644 index 0000000..f6f3d50 --- /dev/null +++ b/doc/sphinx/source/bridgedb.safelog.rst @@ -0,0 +1,9 @@ +.. _safelog: + +bridgedb.safelog +---------------- + +.. automodule:: bridgedb.safelog + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 2f05ca7..9ab7b73 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -39,6 +39,7 @@ import bridgedb.I18n import bridgedb.Main import bridgedb.persistent import bridgedb.runner +import bridgedb.safelog import bridgedb.Stability import bridgedb.Storage import bridgedb.test

1 0

[bridgedb/develop] Move log configuration functions to bridgedb.util module.
by isis＠torproject.org 08 Apr '14

08 Apr '14

commit cc83936455cbe4e39016b3aef9d345652a82130d Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 7 12:17:49 2014 +0000 Move log configuration functions to bridgedb.util module. * MOVE `Main.configureLogging()` → `util.configureLogging()`. * CHANGE `configureLogging()` function to use `logging.config.dictConfig()` with automatic safelogging filters from `bridgedb.safelog` module. * ADD `util._getRotatingFileHandlers()` function for chmod/chown'ing the rotated log files to be readable only by the running process owner. * ADD `util.JustifiedLogFormatter` class for justified printing of log messages when the config options LOG_TRACE and LOG_THREADS are enabled (this extra information is then printed in justified columns on the left, leaving the actual log messages left-justified to the same width on the right, which in my opinion is much easier to read quickly). --- lib/bridgedb/Main.py | 39 +--------- lib/bridgedb/util.py | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 38 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index e29e991..90a03ef 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -13,7 +13,6 @@ import signal import sys import time import logging -import logging.handlers import gettext from twisted.internet import reactor @@ -29,42 +28,6 @@ import bridgedb.Dist as Dist import bridgedb.Time as Time import bridgedb.Storage -def configureLogging(cfg): - """Set up Python's logging subsystem based on the configuratino. - """ - - # Turn on safe logging by default - safelogging = getattr(cfg, 'SAFELOGGING', True) - - level = getattr(cfg, 'LOGLEVEL', 'WARNING') - level = getattr(logging, level) - logfile = getattr(cfg, 'LOGFILE', "") - logfile_count = getattr(cfg, 'LOGFILE_COUNT', 5) - logfile_rotate_size = getattr(cfg, 'LOGFILE_ROTATE_SIZE', 10000000) - safelog.setSafeLogging(safelogging) - - logging.getLogger().setLevel(level) - if logfile: - handler = logging.handlers.RotatingFileHandler(logfile, 'a', - logfile_rotate_size, - logfile_count) - formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s', - "%b %d %H:%M:%S") - handler.setFormatter(formatter) - logging.getLogger().addHandler(handler) - - logging.info("Logger Started.") - logging.info("Level: %s", level) - if logfile: - logging.info("Log File: %s", os.path.abspath(logfile)) - logging.info("Log File Count: %d", logfile_count) - logging.info("Rotate Logs After Size: %d", logfile_rotate_size) - else: - logging.info("Logging to stderr") - if safelogging: - logging.info("Safe Logging: Enabled") - else: - logging.warn("Safe Logging: Disabled") def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. @@ -461,7 +424,7 @@ def startup(options): # called. Otherwise a default handler that logs to the console will be # created by the imported module, and all further calls to # :func:`logging.basicConfig` will be ignored. - configureLogging(config) + util.configureLogging(config) if options['dump-bridges'] or (options.subCommand is not None): runSubcommand(options, config) diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index ebca7e1..691b9bc 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -11,5 +11,197 @@ """Common utilities for BridgeDB.""" +import logging +import logging.config +import logging.handlers +import os +def _getLogHandlers(logToFile=True, logToStderr=True): + """Get the appropriate list of log handlers. + + :param bool logToFile: If ``True``, add a logfile handler. + :param bool logToStderr: If ``True``, add a stream handler to stderr. + :rtype: list + :returns: A list containing the appropriate log handler names from the + :class:`logging.config.dictConfigClass`. + """ + logHandlers = [] + if logToFile: + logHandlers.append('rotating') + if logToStderr: + logHandlers.append('console') + return logHandlers + +def _getRotatingFileHandler(filename, mode='a', filesize=1000000, filecount=0, + encoding='utf-8', uid=None, gid=None): + """Get a :class:`logging.RotatingFileHandler` with a logfile which is + readable+writable only by the given **uid** and **gid**. + + :param str filename: The full path to the log file. + :param str mode: The mode to open **filename** with. (default: ``'a'``) + :param int filesize: Rotate logfiles after they have grown to this size in + bytes. + :param int filecount: The number of logfiles to keep in rotation. + :param str encoding: The encoding for the logfile. + :param int uid: The owner UID to set on the logfile. + :param int gid: The GID to set on the logfile. + :rtype: :class:`logging.handlers.RotatingFileHandler` + :returns: A logfile handler which will rotate files and chown/chmod newly + created files. + """ + # Default to the current process owner's uid and gid: + uid = os.getuid() if not uid else uid + gid = os.getgid() if not gid else gid + + if not os.path.exists(filename): + open(filename, 'a').close() + os.chown(filename, uid, gid) + os.chmod(filename, os.ST_WRITE | os.ST_APPEND) + + fileHandler = logging.handlers.RotatingFileHandler(filename, mode, + maxBytes=filesize, + backupCount=filecount, + encoding=encoding) + return fileHandler + +def configureLogging(cfg): + """Set up Python's logging subsystem based on the configuration. + + :type cfg: :class:`~bridgedb.persistent.Conf` + :param cfg: The current configuration, including any in-memory settings. + """ + from bridgedb import safelog + + # Turn on safe logging by default: + safelogging = getattr(cfg, 'SAFELOGGING', True) + safelog.setSafeLogging(safelogging) + + level = getattr(cfg, 'LOGLEVEL', 'WARNING') + logLevel = getattr(logging, level, 0) + logStderr = getattr(cfg, 'LOG_TO_STDERR', False) + logfileName = getattr(cfg, 'LOGFILE', "") + logfileCount = getattr(cfg, 'LOGFILE_COUNT', 3) - 1 + logfileRotateSize = getattr(cfg, 'LOGFILE_ROTATE_SIZE', 10000000) + logThreads = getattr(cfg, 'LOG_THREADS', False) + logTrace = getattr(cfg, 'LOG_TRACE', False) + logTimeFormat = getattr(cfg, 'LOG_TIME_FORMAT', "%H:%M:%S") + + logFilters = [] + if safelogging: + logFilters = ['safelogEmail', 'safelogIPv4', 'safelogIPv6'] + + logConfig = { + 'version': 1, + 'filters': { + 'safelogEmail': {'()': safelog.SafelogEmailFilter}, + 'safelogIPv4': {'()': safelog.SafelogIPv4Filter}, + 'safelogIPv6': {'()': safelog.SafelogIPv6Filter}, + }, + 'formatters': { + 'default': {'()': JustifiedLogFormatter, + # These values below are kwargs passed to + # :class:`JustifiedFormatter`: + 'logThreads': logThreads, + 'logTrace': logTrace, + 'datefmt': logTimeFormat}, + }, + 'handlers': { + 'console': {'class': 'logging.StreamHandler', + 'level': logLevel, + 'formatter': 'default', + 'filters': logFilters}, + 'rotating': {'()': _getRotatingFileHandler, + 'level': logLevel, + 'formatter': 'default', + 'filters': logFilters, + # The values below are passed to the handler creator, + # :func:`getChownedFileHandler`, as kwargs: + 'filename': logfileName, + 'filesize': logfileRotateSize, + 'filecount': logfileCount}, + }, + 'root': { + 'handlers': _getLogHandlers(logfileName, logStderr), + 'level': logLevel, + }, + } + + logging.config.dictConfig(logConfig) + + logging.info("Logger Started.") + logging.info("Level: %s", logLevel) + logging.info("Safe Logging: %sabled" % ("En" if safelogging else "Dis")) + + +class JustifiedLogFormatter(logging.Formatter): + """A logging formatter which pretty prints thread and calling function + information, in addition to the normal timestamp, log level, and log + message. + + :ivar int width: The width of the column for the calling function + information, if the latter is to be included. + """ + width = 30 + + def __init__(self, logThreads=False, logTrace=False, + datefmt="%H:%M:%s"): + """If **logTrace** is ``True``, the line number, module name, and + function name where the logger was called will be included in the + message, and the width of this information will always equal ``width``. + + :param bool logThreads: If ``True``, include the current thread name + and ID in formatted log messages. + :param bool logTrace: If ``True``, include information on the calling + function in formatted log messages. + """ + super(JustifiedLogFormatter, self).__init__(datefmt=datefmt) + self.logThreads = logThreads + self.logTrace = logTrace + + _fmt = ["%(asctime)s %(levelname)-7.7s"] + if self.logThreads: + _fmt.append("[%(threadName)s id:%(thread)d]") + _fmt.append("%(callingFunc)s") + _fmt.append("%(message)s") + + self._fmt = " ".join(_fmt) + + def _formatCallingFuncName(self, record): + """Format the combined module name and function name of the place where + the log message/record was recorded, so that the formatted string is + left-justified and not longer than the :cvar:`width`. + + :type record: :class:`logging.LogRecord` + :param record: A record of an event created by calling a logger. + :returns: The :class:`logging.LogRecord` with its ``message`` + attribute rewritten to contain the module and function name, + truncated to ``width``, or padded on the right with spaces as is + necessary. + """ + callingFunc = "" + if self.logTrace: + # The '.' character between the module name and function name + # would otherwise be interpreted as a format string specifier, so + # we must specify ``chr(46)``: + lineno = "L%s:" % record.lineno + caller = "%s%-s%s" % (lineno.rjust(6), record.module, chr(46)) + maxFuncNameWidth = self.width - 2 - len(caller) + funcName = record.funcName + if len(funcName) > maxFuncNameWidth: + funcName = record.funcName[:maxFuncNameWidth] + caller += "%s()" % (funcName) + callingFunc = caller.ljust(self.width) + + record.callingFunc = callingFunc + return record + + def format(self, record): + """Reformat this log **record** to neatly print thread and function + traces, if configured to do so. + + :type record: :class:`logging.LogRecord` + :param record: A record of an event created by calling a logger. + """ + record = self._formatCallingFuncName(record) + return super(JustifiedLogFormatter, self).format(record)

1 0