[tor-commits] [sbws/master] Logging config overhaul ...
pastly at torproject.org
pastly at torproject.org
Wed Jul 11 15:05:40 UTC 2018
commit 7db1887f1c33e8202a796d2ca38c5242bcd27b9d
Author: Matt Traudt <sirmatt at ksu.edu>
Date: Mon Jun 25 11:18:43 2018 -0400
Logging config overhaul ...
User should configure logging via the [logging] section
in their config.ini.
User can change the log level using --log-level, or in the config
with level, to_file_level, and to_stdout_level.
User can change the log format in the config with format,
to_file_format, and to_stdout_format
An example debugging format is included as a comment in config.default.ini
config.log.default.ini is majorly reorganized. For the most part, if
something doesn't need to be specified in this file, it has been removed.
For example, we don't have to specify any args to handler_to_file. The
beginnings of syslog configuration stuff remain as comments.
As part of configure_logging, force logging to stdout if we don't think
we can log to file.
Actually set some parameters to RotatingFileHandler so it rotates when
files reach 10 MiB and keeps 100 backups.
Each sbws command (like scanner and generate) gets its own log file. I
did this because I didn't want 'sbws scanner' to be running 24/7 and an
'sbws generate' call to log to the same file. That would get confusing.
---
sbws/config.default.ini | 10 +++++--
sbws/config.log.default.ini | 65 +++++++++++++++++------------------------
sbws/sbws.py | 2 +-
sbws/util/config.py | 70 ++++++++++++++++++++++++++++++++++++++++-----
4 files changed, 98 insertions(+), 49 deletions(-)
diff --git a/sbws/config.default.ini b/sbws/config.default.ini
index 10a2fb3..3e49052 100644
--- a/sbws/config.default.ini
+++ b/sbws/config.default.ini
@@ -84,11 +84,17 @@ fraction_relays = 0.05
min_relays = 50
[logging]
-# Level to log at. Debug, info, warning, error.
-level = debug
# Whether or not to log to a rotating file the directory paths.log_dname
to_file = yes
# Whether or not to log to stdout
to_stdout = no
+# Level to log at. Debug, info, warning, error, critical.
+level = info
+to_file_level = ${level}
+to_stdout_level = ${level}
# Format string to use when logging
format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s
+to_file_format = ${format}
+to_stdout_format = ${format}
+# verbose formatter useful for debugging
+#format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s
diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini
index 4bae7aa..606ff16 100644
--- a/sbws/config.log.default.ini
+++ b/sbws/config.log.default.ini
@@ -2,62 +2,49 @@
keys = root,sbws
[handlers]
-keys = sbwsdefault,sbwsdebug,sbwsfile
+keys = to_file,to_stdout
[formatters]
-keys = sbwsdefault,sbwsdebug,sbwssys
+keys = to_file,to_stdout
[logger_root]
level = WARNING
-handlers = sbwsdefault
+handlers = to_file
propagate = 1
qualname=root
[logger_sbws]
-level = INFO
-# add here sbwssys for also logging to system log
-handlers = sbwsdefault,sbwsfile
propagate = 0
qualname=sbws
-[handler_sbwsdefault]
+[handler_to_stdout]
class = StreamHandler
-formatter = sbwsdefault
+formatter = to_stdout
args = (sys.stdout,)
-[formatter_sbwsdefault]
-format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s
-
-# this is not needed, but here to do not modify sbwsdefault
-[handler_sbwsdebug]
-class = StreamHandler
-formatter = sbwsdebug
-args = (sys.stdout,)
-level = DEBUG
-
-# verbose formatter useful for debugging
-[formatter_sbwsdebug]
-format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s
-datefmt = %Y-%m-%d %H:%M:%S
-
-# for logging to system log
-[handler_sbwssys]
-class=handlers.SysLogHandler
-formatter=sbwssys
-level = INFO
-
-# syslog-like formater
-[formatter_sbwssys]
-# hostname should be added here with a context filter, dunno if there is a way
-# to add it here
-format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s
-datefmt = %b %d %H:%M:%S
-
-[handler_sbwsfile]
+[handler_to_file]
class = handlers.RotatingFileHandler
-formatter = sbwssys
+formatter = to_file
# There doesn't seem to be a way to put ${paths:log_filepath} here since the
# logging library eval()s this, which doesn't give the ini parsing library a
# chance to replace the text. So we do it in configure_logging() in
# sbws/util/config.py instead.
-args = ('/nonexist',)
+args =
+
+## for logging to system log
+#[handler_to_syslog]
+#class=handlers.SysLogHandler
+#formatter=to_syslog
+#level = INFO
+#args = ()
+
+[formatter_to_stdout]
+
+[formatter_to_file]
+
+## syslog-like formater
+#[formatter_to_syslog]
+## hostname should be added here with a context filter, dunno if there is a way
+## to add it here
+#format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s
+#datefmt = %b %d %H:%M:%S
diff --git a/sbws/sbws.py b/sbws/sbws.py
index 584ce02..5688cea 100644
--- a/sbws/sbws.py
+++ b/sbws/sbws.py
@@ -28,7 +28,7 @@ def main():
for e in conf_errors:
log.critical(e)
exit(1)
- # configure_logging(conf)
+ configure_logging(args, conf)
def_args = [args, conf]
def_kwargs = {}
known_commands = {
diff --git a/sbws/util/config.py b/sbws/util/config.py
index 25df375..0e6f308 100644
--- a/sbws/util/config.py
+++ b/sbws/util/config.py
@@ -1,4 +1,5 @@
from configparser import (ConfigParser, ExtendedInterpolation)
+from configparser import InterpolationMissingOptionError
import os
import logging
import logging.config
@@ -15,6 +16,8 @@ _SYMBOLS_NO_QUOTES = '!@#$%^&*()-_=+\\|[]{}:;/?.,<>'
_HEX = '0123456789ABCDEF'
+_LOG_LEVELS = ['debug', 'info', 'warning', 'error', 'critical']
+
log = logging.getLogger(__name__)
@@ -72,12 +75,63 @@ def get_user_example_config():
return conf
-def configure_logging(conf):
+def _can_log_to_file(conf):
+ '''
+ Checks all the known reasons for why we might not be able to log to a file,
+ and returns whether or not we think we will be able to do so. This is
+ useful because if we can't log to a file, we might want to force logging to
+ stdout.
+
+ If we can't log to file, return False and the reason. Otherwise return True
+ and an empty string.
+ '''
+ # We won't be able to get paths.log_dname from the config when we are first
+ # initializing sbws because it depends on paths.sbws_home (by default).
+ # If there is an issue getting this option, tell the caller that we can't
+ # log to file.
+ try:
+ conf['paths']['log_dname']
+ except InterpolationMissingOptionError as e:
+ return False, e
+ return True, ''
+
+
+def configure_logging(args, conf):
assert isinstance(conf, ConfigParser)
- # log_filepath is not a variable in config.log.default.ini,
- # so adding it here. Maybe there is a better way to do this
- conf['handler_sbwsfile']['args'] = \
- "('{}',)".format(conf['paths']['log_filepath'])
+ logger = 'logger_sbws'
+ if args.log_level:
+ conf[logger]['level'] = args.log_level.upper()
+ # Set the correct handler(s) based on [logging] options
+ handlers = set()
+ can_log_to_file, reason = _can_log_to_file(conf)
+ if not can_log_to_file or conf.getboolean('logging', 'to_stdout'):
+ # always add to_stdout if we cannot log to file
+ handlers.add('to_stdout')
+ if can_log_to_file and conf.getboolean('logging', 'to_file'):
+ handlers.add('to_file')
+ # Collect the handlers in the appropriate config option
+ conf[logger]['handlers'] = ','.join(handlers)
+ if 'to_file' in handlers:
+ # Because of the way python's standard logging library works, we can't
+ # tell this handler that the file it should log to is
+ # ${paths:log_dname}/foo.log. It evals() the string stored in the args,
+ # therefore not allowing the config parser to change that to
+ # ~/.sbws/log/foo.log. So we set it here.
+ #
+ # Also we set files to rotate at 10 MiB in size and to keep 100 backups
+ dname = conf['paths']['log_dname']
+ os.makedirs(dname, exist_ok=True)
+ fname = os.path.join(dname, '{}.log'.format(args.command or 'sbws'))
+ conf['handler_to_file']['args'] = \
+ "('{}', 'a', 10*1024*1024, 100)".format(fname)
+ # Set some stuff that needs config parser's interpolation
+ conf['formatter_to_file']['format'] = conf['logging']['to_file_format']
+ conf['formatter_to_stdout']['format'] = conf['logging']['to_stdout_format']
+ conf[logger]['level'] = conf['logging']['level'].upper()
+ conf['handler_to_file']['level'] = conf['logging']['to_file_level'].upper()
+ conf['handler_to_stdout']['level'] = \
+ conf['logging']['to_stdout_level'].upper()
+ # Now we configure the standard python logging system
with NamedTemporaryFile('w+t') as fd:
conf.write(fd)
fd.seek(0, 0)
@@ -220,13 +274,15 @@ def _validate_logging(conf):
sec = 'logging'
err_tmpl = Template('$sec/$key ($val): $e')
enums = {
- 'level': {'choices': ['debug', 'info', 'warning', 'error']},
+ 'level': {'choices': _LOG_LEVELS},
+ 'to_file_level': {'choices': _LOG_LEVELS},
+ 'to_stdout_level': {'choices': _LOG_LEVELS},
}
bools = {
'to_file': {},
'to_stdout': {},
}
- unvalidated = ['format']
+ unvalidated = ['format', 'to_file_format', 'to_stdout_format']
all_valid_keys = list(bools.keys()) + list(enums.keys()) + unvalidated
errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl))
errors.extend(_validate_section_bools(conf, sec, bools, err_tmpl))
More information about the tor-commits
mailing list