tor-commits July 2018

tor-commits@lists.torproject.org

17 participants
1737 discussions

[sbws/master] Logging config overhaul ...
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 7db1887f1c33e8202a796d2ca38c5242bcd27b9d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 11:18:43 2018 -0400 Logging config overhaul ... User should configure logging via the [logging] section in their config.ini. User can change the log level using --log-level, or in the config with level, to_file_level, and to_stdout_level. User can change the log format in the config with format, to_file_format, and to_stdout_format An example debugging format is included as a comment in config.default.ini config.log.default.ini is majorly reorganized. For the most part, if something doesn't need to be specified in this file, it has been removed. For example, we don't have to specify any args to handler_to_file. The beginnings of syslog configuration stuff remain as comments. As part of configure_logging, force logging to stdout if we don't think we can log to file. Actually set some parameters to RotatingFileHandler so it rotates when files reach 10 MiB and keeps 100 backups. Each sbws command (like scanner and generate) gets its own log file. I did this because I didn't want 'sbws scanner' to be running 24/7 and an 'sbws generate' call to log to the same file. That would get confusing. --- sbws/config.default.ini | 10 +++++-- sbws/config.log.default.ini | 65 +++++++++++++++++------------------------ sbws/sbws.py | 2 +- sbws/util/config.py | 70 ++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 98 insertions(+), 49 deletions(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index 10a2fb3..3e49052 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -84,11 +84,17 @@ fraction_relays = 0.05 min_relays = 50 [logging] -# Level to log at. Debug, info, warning, error. -level = debug # Whether or not to log to a rotating file the directory paths.log_dname to_file = yes # Whether or not to log to stdout to_stdout = no +# Level to log at. Debug, info, warning, error, critical. +level = info +to_file_level = ${level} +to_stdout_level = ${level} # Format string to use when logging format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s +to_file_format = ${format} +to_stdout_format = ${format} +# verbose formatter useful for debugging +#format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini index 4bae7aa..606ff16 100644 --- a/sbws/config.log.default.ini +++ b/sbws/config.log.default.ini @@ -2,62 +2,49 @@ keys = root,sbws [handlers] -keys = sbwsdefault,sbwsdebug,sbwsfile +keys = to_file,to_stdout [formatters] -keys = sbwsdefault,sbwsdebug,sbwssys +keys = to_file,to_stdout [logger_root] level = WARNING -handlers = sbwsdefault +handlers = to_file propagate = 1 qualname=root [logger_sbws] -level = INFO -# add here sbwssys for also logging to system log -handlers = sbwsdefault,sbwsfile propagate = 0 qualname=sbws -[handler_sbwsdefault] +[handler_to_stdout] class = StreamHandler -formatter = sbwsdefault +formatter = to_stdout args = (sys.stdout,) -[formatter_sbwsdefault] -format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s - -# this is not needed, but here to do not modify sbwsdefault -[handler_sbwsdebug] -class = StreamHandler -formatter = sbwsdebug -args = (sys.stdout,) -level = DEBUG - -# verbose formatter useful for debugging -[formatter_sbwsdebug] -format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s -datefmt = %Y-%m-%d %H:%M:%S - -# for logging to system log -[handler_sbwssys] -class=handlers.SysLogHandler -formatter=sbwssys -level = INFO - -# syslog-like formater -[formatter_sbwssys] -# hostname should be added here with a context filter, dunno if there is a way -# to add it here -format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s -datefmt = %b %d %H:%M:%S - -[handler_sbwsfile] +[handler_to_file] class = handlers.RotatingFileHandler -formatter = sbwssys +formatter = to_file # There doesn't seem to be a way to put ${paths:log_filepath} here since the # logging library eval()s this, which doesn't give the ini parsing library a # chance to replace the text. So we do it in configure_logging() in # sbws/util/config.py instead. -args = ('/nonexist',) +args = + +## for logging to system log +#[handler_to_syslog] +#class=handlers.SysLogHandler +#formatter=to_syslog +#level = INFO +#args = () + +[formatter_to_stdout] + +[formatter_to_file] + +## syslog-like formater +#[formatter_to_syslog] +## hostname should be added here with a context filter, dunno if there is a way +## to add it here +#format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s +#datefmt = %b %d %H:%M:%S diff --git a/sbws/sbws.py b/sbws/sbws.py index 584ce02..5688cea 100644 --- a/sbws/sbws.py +++ b/sbws/sbws.py @@ -28,7 +28,7 @@ def main(): for e in conf_errors: log.critical(e) exit(1) - # configure_logging(conf) + configure_logging(args, conf) def_args = [args, conf] def_kwargs = {} known_commands = { diff --git a/sbws/util/config.py b/sbws/util/config.py index 25df375..0e6f308 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -1,4 +1,5 @@ from configparser import (ConfigParser, ExtendedInterpolation) +from configparser import InterpolationMissingOptionError import os import logging import logging.config @@ -15,6 +16,8 @@ _SYMBOLS_NO_QUOTES = '!@#$%^&*()-_=+\\|[]{}:;/?.,<>' _HEX = '0123456789ABCDEF' +_LOG_LEVELS = ['debug', 'info', 'warning', 'error', 'critical'] + log = logging.getLogger(__name__) @@ -72,12 +75,63 @@ def get_user_example_config(): return conf -def configure_logging(conf): +def _can_log_to_file(conf): + ''' + Checks all the known reasons for why we might not be able to log to a file, + and returns whether or not we think we will be able to do so. This is + useful because if we can't log to a file, we might want to force logging to + stdout. + + If we can't log to file, return False and the reason. Otherwise return True + and an empty string. + ''' + # We won't be able to get paths.log_dname from the config when we are first + # initializing sbws because it depends on paths.sbws_home (by default). + # If there is an issue getting this option, tell the caller that we can't + # log to file. + try: + conf['paths']['log_dname'] + except InterpolationMissingOptionError as e: + return False, e + return True, '' + + +def configure_logging(args, conf): assert isinstance(conf, ConfigParser) - # log_filepath is not a variable in config.log.default.ini, - # so adding it here. Maybe there is a better way to do this - conf['handler_sbwsfile']['args'] = \ - "('{}',)".format(conf['paths']['log_filepath']) + logger = 'logger_sbws' + if args.log_level: + conf[logger]['level'] = args.log_level.upper() + # Set the correct handler(s) based on [logging] options + handlers = set() + can_log_to_file, reason = _can_log_to_file(conf) + if not can_log_to_file or conf.getboolean('logging', 'to_stdout'): + # always add to_stdout if we cannot log to file + handlers.add('to_stdout') + if can_log_to_file and conf.getboolean('logging', 'to_file'): + handlers.add('to_file') + # Collect the handlers in the appropriate config option + conf[logger]['handlers'] = ','.join(handlers) + if 'to_file' in handlers: + # Because of the way python's standard logging library works, we can't + # tell this handler that the file it should log to is + # ${paths:log_dname}/foo.log. It evals() the string stored in the args, + # therefore not allowing the config parser to change that to + # ~/.sbws/log/foo.log. So we set it here. + # + # Also we set files to rotate at 10 MiB in size and to keep 100 backups + dname = conf['paths']['log_dname'] + os.makedirs(dname, exist_ok=True) + fname = os.path.join(dname, '{}.log'.format(args.command or 'sbws')) + conf['handler_to_file']['args'] = \ + "('{}', 'a', 10*1024*1024, 100)".format(fname) + # Set some stuff that needs config parser's interpolation + conf['formatter_to_file']['format'] = conf['logging']['to_file_format'] + conf['formatter_to_stdout']['format'] = conf['logging']['to_stdout_format'] + conf[logger]['level'] = conf['logging']['level'].upper() + conf['handler_to_file']['level'] = conf['logging']['to_file_level'].upper() + conf['handler_to_stdout']['level'] = \ + conf['logging']['to_stdout_level'].upper() + # Now we configure the standard python logging system with NamedTemporaryFile('w+t') as fd: conf.write(fd) fd.seek(0, 0) @@ -220,13 +274,15 @@ def _validate_logging(conf): sec = 'logging' err_tmpl = Template('$sec/$key ($val): $e') enums = { - 'level': {'choices': ['debug', 'info', 'warning', 'error']}, + 'level': {'choices': _LOG_LEVELS}, + 'to_file_level': {'choices': _LOG_LEVELS}, + 'to_stdout_level': {'choices': _LOG_LEVELS}, } bools = { 'to_file': {}, 'to_stdout': {}, } - unvalidated = ['format'] + unvalidated = ['format', 'to_file_format', 'to_stdout_format'] all_valid_keys = list(bools.keys()) + list(enums.keys()) + unvalidated errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) errors.extend(_validate_section_bools(conf, sec, bools, err_tmpl))

1 0

[sbws/master] Add [logging] section to config and validate it
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit f3a487bd8fd950fb2f9e41d1a944bc87aca49e44 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 09:58:30 2018 -0400 Add [logging] section to config and validate it --- sbws/config.default.ini | 11 +++++++++++ sbws/util/config.py | 22 +++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index 80ddc8b..10a2fb3 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -4,6 +4,7 @@ v3bw_dname = ${sbws_home}/v3bw v3bw_fname = ${v3bw_dname}/{}.v3bw started_filepath = ${sbws_home}/started_at log_filepath = ${sbws_home}/sbws.log +log_dname = ${sbws_home}/log [destinations] # How often to check if a destional is usable @@ -81,3 +82,13 @@ measure_authorities = off fraction_relays = 0.05 # The minimum number of best priority relays we are willing to return min_relays = 50 + +[logging] +# Level to log at. Debug, info, warning, error. +level = debug +# Whether or not to log to a rotating file the directory paths.log_dname +to_file = yes +# Whether or not to log to stdout +to_stdout = no +# Format string to use when logging +format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s diff --git a/sbws/util/config.py b/sbws/util/config.py index 0bfa072..c24ac23 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -109,6 +109,7 @@ def validate_config(conf): errors.extend(_validate_paths(conf)) errors.extend(_validate_destinations(conf)) errors.extend(_validate_relayprioritizer(conf)) + errors.extend(_validate_logging(conf)) return len(errors) < 1, errors @@ -156,7 +157,7 @@ def _validate_paths(conf): err_tmpl = Template('$sec/$key ($val): $e') unvalidated_keys = [ 'datadir', 'sbws_home', 'v3bw_fname', 'v3bw_dname', - 'started_filepath', 'log_filepath'] + 'started_filepath', 'log_filepath', 'log_dname'] all_valid_keys = unvalidated_keys allow_missing = ['sbws_home'] errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl, @@ -227,6 +228,25 @@ def _validate_relayprioritizer(conf): return errors +def _validate_logging(conf): + errors = [] + sec = 'logging' + err_tmpl = Template('$sec/$key ($val): $e') + enums = { + 'level': {'choices': ['debug', 'info', 'warning', 'error']}, + } + bools = { + 'to_file': {}, + 'to_stdout': {}, + } + unvalidated = ['format'] + all_valid_keys = list(bools.keys()) + list(enums.keys()) + unvalidated + errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) + errors.extend(_validate_section_bools(conf, sec, bools, err_tmpl)) + errors.extend(_validate_section_enums(conf, sec, enums, err_tmpl)) + return errors + + def _validate_destinations(conf): errors = [] sec = 'destinations'

1 0

[sbws/master] config.log.ini no longer exists
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit dca6e408fd760b294b2402665e1ffa0eaf1c5122 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 11:10:45 2018 -0400 config.log.ini no longer exists --- sbws/globals.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbws/globals.py b/sbws/globals.py index 3ae7d60..a00cfb1 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -35,8 +35,7 @@ def is_initted(d): if not os.path.isdir(d): log.debug('%s not initialized: %s doesn\'t exist', d, d) return False - conf_fnames = [os.path.join(d, 'config.ini'), - os.path.join(d, 'config.log.ini')] + conf_fnames = [os.path.join(d, 'config.ini')] for fname in conf_fnames: if not os.path.isfile(fname): log.debug('%s not initialized: missing %s', d, fname)

1 0

[sbws/master] Change capitalization of --log-level values
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 9fea5c1c8d9c6b10aece0157dbac5dacef1665dc Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Jun 25 11:11:02 2018 -0400 Change capitalization of --log-level values --- sbws/util/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/util/parser.py b/sbws/util/parser.py index bbc6a18..f2fc828 100644 --- a/sbws/util/parser.py +++ b/sbws/util/parser.py @@ -20,7 +20,7 @@ def create_parser(): '--version', action='version', help='sbws version', version='{}'.format(__version__)) p.add_argument('--log-level', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + choices=['debug', 'info', 'warning', 'error', 'critical'], help='Override the sbws log level') p.add_argument('-d', '--directory', default=_default_dot_sbws_dname(), help='Name of the .sbws directory')

1 0

[sbws/master] Update comment: we do more things here now
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit ee720405b0f532138d0ba6a11e6eeb4097991852 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 11:33:07 2018 -0400 Update comment: we do more things here now --- sbws/util/config.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sbws/util/config.py b/sbws/util/config.py index 274e87b..5813878 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -112,18 +112,23 @@ def configure_logging(args, conf): # Collect the handlers in the appropriate config option conf[logger]['handlers'] = ','.join(handlers) if 'to_file' in handlers: - # Because of the way python's standard logging library works, we can't - # tell this handler that the file it should log to is - # ${paths:log_dname}/foo.log. It evals() the string stored in the args, - # therefore not allowing the config parser to change that to - # ~/.sbws/log/foo.log. So we set it here. + # This is weird. # - # Also we set files to rotate at 10 MiB in size and to keep 100 backups + # Python's logging library expects 'args' to be a tuple ... but it has + # to be stored as a string and it evals() the string. + # + # The first argument is the file name to which it should log. Set it to + # the sbws command (like 'scanner' or 'generate') if possible, or to + # 'sbws' failing that. dname = conf['paths']['log_dname'] os.makedirs(dname, exist_ok=True) fname = os.path.join(dname, '{}.log'.format(args.command or 'sbws')) - conf['handler_to_file']['args'] = \ - "('{}', 'a', 10*1024*1024, 100)".format(fname) + # The second argument is the number of backups to keep, and the third + # is the maximum file size (in bytes) each log file should be. + max_bytes = conf.getint('logging', 'to_file_max_bytes') + num_backups = conf.getint('logging', 'to_file_num_backups') + # Now store those things as a string in the config. So dumb. + conf['handler_to_file']['args'] = str((fname, num_backups, max_bytes)) # Set some stuff that needs config parser's interpolation conf['formatter_to_file']['format'] = conf['logging']['to_file_format'] conf['formatter_to_stdout']['format'] = conf['logging']['to_stdout_format']

1 0

[sbws/master] Remove unused import to make flake8 happy
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 5254ea6d75797d63ebf046b7c31aafb36145d627 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 12:01:46 2018 -0400 Remove unused import to make flake8 happy --- sbws/core/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/init.py b/sbws/core/init.py index ea415a3..39d4c17 100644 --- a/sbws/core/init.py +++ b/sbws/core/init.py @@ -1,4 +1,4 @@ -from sbws.globals import (is_initted, fail_hard, touch_file) +from sbws.globals import (is_initted, fail_hard) from sbws.util.config import get_user_example_config from sbws.util.userquery import query_yes_no from argparse import ArgumentDefaultsHelpFormatter

1 0

[sbws/master] With --log-level, override the right thing
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit f52bc5b0ac6e5c5e529ed8655038be4a5dc27f97 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 11:32:50 2018 -0400 With --log-level, override the right thing --- sbws/util/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/util/config.py b/sbws/util/config.py index 8953d92..274e87b 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -100,7 +100,7 @@ def configure_logging(args, conf): assert isinstance(conf, ConfigParser) logger = 'logger_sbws' if args.log_level: - conf[logger]['level'] = args.log_level.upper() + conf['logging']['level'] = args.log_level.upper() # Set the correct handler(s) based on [logging] options handlers = set() can_log_to_file, reason = _can_log_to_file(conf)

1 0

[sbws/master] Update example config.ini
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 813c1198a7048b769f6befb9aa4baed491f7bfd6 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 11:52:18 2018 -0400 Update example config.ini --- docs/source/config.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/config.ini b/docs/source/config.ini index 640b3a9..1b4cf26 100644 --- a/docs/source/config.ini +++ b/docs/source/config.ini @@ -1,6 +1,2 @@ [paths] -datadir = ${sbws_home}/datadir sbws_home = /home/user/.sbws - -[scanner] -nickname = IDidntEditTheSBWSConfig

1 0

[sbws/master] Remove comment that doesn't really apply anymore
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit a316ff667e0128c5ad9252d0e130141bee953129 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 11:31:41 2018 -0400 Remove comment that doesn't really apply anymore --- sbws/config.log.default.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini index 6cfcf88..82e483e 100644 --- a/sbws/config.log.default.ini +++ b/sbws/config.log.default.ini @@ -25,10 +25,6 @@ args = (sys.stdout,) [handler_to_file] class = handlers.RotatingFileHandler formatter = to_file -# There doesn't seem to be a way to put ${paths:log_filepath} here since the -# logging library eval()s this, which doesn't give the ini parsing library a -# chance to replace the text. So we do it in configure_logging() in -# sbws/util/config.py instead. args = ('/dev/null', ) ## for logging to system log

1 0

[sbws/master] This needs a valid file name or sbws can't start
by pastly＠torproject.org 11 Jul '18

11 Jul '18

commit 82ad8983f54630ab19cacd76ef33e76658522778 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jul 4 11:31:09 2018 -0400 This needs a valid file name or sbws can't start --- sbws/config.log.default.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/config.log.default.ini b/sbws/config.log.default.ini index 606ff16..6cfcf88 100644 --- a/sbws/config.log.default.ini +++ b/sbws/config.log.default.ini @@ -29,7 +29,7 @@ formatter = to_file # logging library eval()s this, which doesn't give the ini parsing library a # chance to replace the text. So we do it in configure_logging() in # sbws/util/config.py instead. -args = +args = ('/dev/null', ) ## for logging to system log #[handler_to_syslog]

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits July 2018