[tor-commits] [fallback-scripts/master] script: improve bool env var conversion errors and default handling
teor at torproject.org
teor at torproject.org
Thu Aug 8 00:14:02 UTC 2019
commit ba94afcb7aa76b8a759da5109e582584dbe615a5
Author: teor <teor at torproject.org>
Date: Thu Aug 1 18:27:52 2019 +1000
script: improve bool env var conversion errors and default handling
* Restrict the valid True and False bool values
* use default for unset env var, and (not default) for empty set env var
Closes 31305.
---
updateFallbackDirs.py | 81 +++++++++++++++++++++++++++++++++++++++------------
1 file changed, 63 insertions(+), 18 deletions(-)
diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py
index 00e41f8..02f97e3 100755
--- a/updateFallbackDirs.py
+++ b/updateFallbackDirs.py
@@ -73,12 +73,25 @@ except ImportError:
## Top-Level Configuration
-def getenv_conf(var_name, default_val, type_fn):
+def getenv_conf(var_name, default_val, type_fn, optional=False):
"""Get var_name from the environment, using default_val if it is unset.
Cast the result using type_fn. If conversion fails, log an error and
- exit."""
+ exit.
+ type_fn must not be bool. Instead, use custom_bool, which correctly
+ handles empty env vars and defaults, and bad values."""
try:
- return type_fn(os.getenv(var_name, default_val))
+ original_type_fn = type_fn
+ # Use our custom bool function instead
+ assert type_fn != bool
+ # Make the type function optional
+ if optional:
+ type_fn = opt(type_fn)
+ # Look up and convert the value
+ if original_type_fn == custom_bool:
+ # custom_bool does its own default handling
+ return type_fn(os.getenv(var_name), default_val, var_name)
+ else:
+ return type_fn(os.getenv(var_name, default_val))
except ValueError as e:
# Log a useful message if conversion fails
logging.error('Could not cast env var "{}" using function "{}" and default "{}". ValueError: "{}"'.format(var_name, type_fn, default_val, e))
@@ -94,9 +107,15 @@ def opt(type_fn):
If conversion fails, and var_value is None or the empty string, returns
None.
If conversion fails for any other values, throws a ValueError
- exception, with an error string containing var_name, if present."""
+ exception, with an error string containing var_name, if present.
+ Performs special handling for bool conversion using custom_bool()."""
try:
- return type_fn(var_value)
+ if type_fn == custom_bool:
+ assert default_val is not None
+ assert var_name is not None
+ return custom_bool(var_value, default_val, var_name)
+ else:
+ return type_fn(var_value)
# Make type_fn(None) always return None for types that don't cast None
except TypeError:
return None
@@ -111,6 +130,32 @@ def opt(type_fn):
raise e
return opt_type_fn
+# Permitted True and False values for custom_bool(). Must be lowercase.
+CUSTOM_BOOL_TRUE = ['true', 'yes', '1']
+CUSTOM_BOOL_FALSE = ['false', 'no', '0']
+
+def custom_bool(raw_var_value, default_val, var_name=None):
+ """Custom bool conversion function.
+ If raw_var_value is None, returns default_val.
+ If raw_var_value is the empty string, returns not default_val,
+ Otherwise, checks CUSTOM_BOOL_TRUE and CUSTOM_BOOL_FALSE for
+ raw_var_value, returning True or False respectively.
+ Any other raw_var_value throws a ValueError.
+ If var_name is not None, it is included in the ValueError string."""
+ if raw_var_value is None:
+ return default_val
+ elif raw_var_value == '':
+ return not default_val
+ elif str.lower(raw_var_value) in CUSTOM_BOOL_TRUE:
+ return True
+ elif str.lower(raw_var_value) in CUSTOM_BOOL_FALSE:
+ return False
+ else:
+ error_str = "invalid literal for custom_bool(): '{}', default_val: '{}'".format(raw_var_value, default_val)
+ if var_name is not None:
+ error_str += ", var_name: '{}'".format(var_name)
+ raise ValueError(error_str)
+
# We use semantic versioning: https://semver.org
# In particular:
# * major changes include removing a mandatory field, or anything else that
@@ -133,30 +178,30 @@ MODE = getenv_conf('TOR_FB_MODE',
# Output all candidate fallbacks, or only output selected fallbacks?
OUTPUT_CANDIDATES = getenv_conf('TOR_FB_OUTPUT_CANDIDATES',
- False, bool)
+ False, custom_bool)
# Perform DirPort checks over IPv4?
# Change this to False if IPv4 doesn't work for you, or if you don't want to
# download a consensus for each fallback
# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
PERFORM_IPV4_DIRPORT_CHECKS = getenv_conf('TOR_FB_PERFORM_IPV4_DIRPORT_CHECKS',
- not OUTPUT_CANDIDATES, bool)
+ not OUTPUT_CANDIDATES, custom_bool)
# Perform DirPort checks over IPv6?
# There are no IPv6 DirPorts in the Tor protocol, so we disable this option by
# default. When #18394 is implemented, we'll be able to check IPv6 ORPorts.
PERFORM_IPV6_DIRPORT_CHECKS = getenv_conf('TOR_FB_PERFORM_IPV6_DIRPORT_CHECKS',
- False, bool)
+ False, custom_bool)
# Must relays be running now?
MUST_BE_RUNNING_NOW = getenv_conf('TOR_FB_MUST_BE_RUNNING_NOW',
(PERFORM_IPV4_DIRPORT_CHECKS
- or PERFORM_IPV6_DIRPORT_CHECKS), bool)
+ or PERFORM_IPV6_DIRPORT_CHECKS), custom_bool)
# Clients have been using microdesc consensuses by default for a while now
DOWNLOAD_MICRODESC_CONSENSUS = (
getenv_conf('TOR_FB_DOWNLOAD_MICRODESC_CONSENSUS',
- True, bool))
+ True, custom_bool))
# If a relay delivers an invalid consensus, if it will become valid less than
# this many seconds in the future, or expired less than this many seconds ago,
@@ -185,12 +230,12 @@ REASONABLY_LIVE_TIME = getenv_conf('TOR_FB_REASONABLY_LIVE_TIME',
# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
OUTPUT_COMMENTS = getenv_conf('TOR_FB_OUTPUT_COMMENTS',
- OUTPUT_CANDIDATES, bool)
+ OUTPUT_CANDIDATES, custom_bool)
# Output matching ContactInfo in fallbacks list?
# Useful if you're trying to contact operators
CONTACT_COUNT = getenv_conf('TOR_FB_CONTACT_COUNT',
- OUTPUT_CANDIDATES, bool)
+ OUTPUT_CANDIDATES, custom_bool)
# How the list should be sorted:
# fingerprint: is useful for stable diffs of fallback lists
@@ -211,12 +256,12 @@ ONIONOO = getenv_conf('TOR_FB_ONIONOO',
# None means "all relays".
# Set env TOR_FB_ONIONOO_LIMIT="None" to request all relays.
ONIONOO_LIMIT = getenv_conf('TOR_FB_ONIONOO_LIMIT',
- None, opt(int))
+ None, int, optional=True)
# Don't bother going out to the Internet, just use the files available locally,
# even if they're very old
LOCAL_FILES_ONLY = getenv_conf('TOR_FB_LOCAL_FILES_ONLY',
- False, bool)
+ False, custom_bool)
## Whitelist Filter Settings
@@ -226,7 +271,7 @@ LOCAL_FILES_ONLY = getenv_conf('TOR_FB_LOCAL_FILES_ONLY',
# What happens to entries not in whitelist?
# When True, they are included, when False, they are excluded
INCLUDE_UNLISTED_ENTRIES = getenv_conf('TOR_FB_INCLUDE_UNLISTED_ENTRIES',
- OUTPUT_CANDIDATES, bool)
+ OUTPUT_CANDIDATES, custom_bool)
WHITELIST_FILE_NAME = getenv_conf('TOR_FB_WHITELIST_FILE_NAME',
'fallback.whitelist', str)
@@ -289,14 +334,14 @@ _FB_POG = 0.2
# Set env TOR_FB_FALLBACK_PROPORTION_OF_GUARDS="None" to have no limit.
FALLBACK_PROPORTION_OF_GUARDS = (
getenv_conf('TOR_FB_FALLBACK_PROPORTION_OF_GUARDS',
- None if OUTPUT_CANDIDATES else _FB_POG, opt(float)))
+ None if OUTPUT_CANDIDATES else _FB_POG, float, optional=True))
# Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
# None means no limit on the number of fallbacks.
# Set env TOR_FB_MAX_FALLBACK_COUNT="None" to have no limit.
MAX_FALLBACK_COUNT = (
getenv_conf('TOR_FB_MAX_FALLBACK_COUNT',
- None if OUTPUT_CANDIDATES else 200, opt(int)))
+ None if OUTPUT_CANDIDATES else 200, int, optional=True))
# Emit a C #error if the number of fallbacks is less than expected
# Set to 0 to have no minimum.
MIN_FALLBACK_COUNT = (
@@ -353,7 +398,7 @@ CONSENSUS_DOWNLOAD_SPEED_MAX = (
# If the relay fails a consensus check, retry the download
# This avoids delisting a relay due to transient network conditions
CONSENSUS_DOWNLOAD_RETRY = getenv_conf('TOR_FB_CONSENSUS_DOWNLOAD_RETRY',
- True, bool)
+ True, custom_bool)
## Parsing Functions
More information about the tor-commits
mailing list