commit 92dea956c0f9a18dfeb8b571ecff5633929a858d Author: juga0 juga@riseup.net Date: Wed Feb 6 12:11:20 2019 +0000
config: Add country to scanner and destinations sections
To be able to add them to the bandwidth file so that we can diversify the locations and/or know how the location affects the bandwidth results. --- docs/source/config.example.ini | 7 +++++++ docs/source/man_sbws.ini.rst | 10 ++++++++-- sbws/config.default.ini | 3 +++ sbws/util/config.py | 23 +++++++++++++++++++++-- sbws/util/iso3166.py | 37 +++++++++++++++++++++++++++++++++++++ tests/unit/util/test_config.py | 30 ++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 4 deletions(-)
diff --git a/docs/source/config.example.ini b/docs/source/config.example.ini index f3ca1da..9a0d1ca 100644 --- a/docs/source/config.example.ini +++ b/docs/source/config.example.ini @@ -2,9 +2,16 @@ [scanner] # A human-readable string with chars in a-zA-Z0-9 to identify your scanner nickname = sbws_default +# ISO 3166-1 alpha-2 country code. To be edited. +# Default to a non existing country to detect it was not edited. +country = AA
[destinations] foo = off
[destinations.foo] url = https://example.com/does/not/exist.bin +# ISO 3166-1 alpha-2 country code. To be edited. +# Use ZZ if the destination URL is a domain name and it is in a CDN. +# Default to a non existing country to detect it was not edited. +country = AA diff --git a/docs/source/man_sbws.ini.rst b/docs/source/man_sbws.ini.rst index 8886d62..8f64b0d 100644 --- a/docs/source/man_sbws.ini.rst +++ b/docs/source/man_sbws.ini.rst @@ -8,9 +8,9 @@ Tor bandwidth scanner configuration file.
**sbws** (1) ``scanner`` command requires a configuration file with a "[destinations]" section. +"[destinations]" is the only section that does not have a default value.
-It is the only section that does not have a default value. - +It is also required to configure "country" in the "[scanner]" section. It is recommended, but not required to configure "nickname" in the "[scanner]" section.
@@ -77,6 +77,9 @@ destinations.STR verify = BOOL Whether or not to verify the destination certificate. (Default: True) + country = STR + ISO 3166-1 alpha-2 country code. + Use ZZ if the destination URL is a domain name and it is in a CDN.
tor
@@ -99,6 +102,9 @@ scanner nickname = STR A human-readable string with chars in a-zA-Z0-9 to identify the scanner. (Default: IDidntEditTheSBWSConfig) + country = STR + ISO 3166-1 alpha-2 country code. + (Default: AA, a non existing country to detect it was not edited) download_toofast = INT Limits on what download times are too fast/slow/etc. (Default: 1) download_min = INT diff --git a/sbws/config.default.ini b/sbws/config.default.ini index f666640..81b4b07 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -36,6 +36,9 @@ reset_bw_ipv6_changes = off [scanner] # A human-readable string with chars in a-zA-Z0-9 to identify your scanner nickname = IDidntEditTheSBWSConfig +# ISO 3166-1 alpha-2 country code. To be edited. +# Default to a non existing country to detect it was not edited. +country = AA # Limits on what download times are too fast/slow/etc. download_toofast = 1 download_min = 5 diff --git a/sbws/util/config.py b/sbws/util/config.py index b1ecc4f..6622ba3 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -12,6 +12,8 @@ from sbws.globals import (DEFAULT_CONFIG_PATH, DEFAULT_LOG_CONFIG_PATH, USER_CONFIG_PATH, SUPERVISED_RUN_DPATH, SUPERVISED_USER_CONFIG_PATH)
+from sbws.util.iso3166 import ISO_3166_ALPHA_2 + _ALPHANUM = 'abcdefghijklmnopqrstuvwxyz' _ALPHANUM += _ALPHANUM.upper() _ALPHANUM += '0123456789' @@ -269,6 +271,21 @@ def _validate_paths(conf): return errors
+def _validate_country(conf, sec, key, err_tmpl): + errors = [] + if conf[sec].get(key, None) is None: + errors.append(err_tmpl.substitute( + sec=sec, key=key, val=None, + e="Missing country in configuration file.")) + return errors + valid = conf[sec]['country'] in ISO_3166_ALPHA_2 + if not valid: + errors.append(err_tmpl.substitute( + sec=sec, key=key, val=conf[sec][key], + e="Not a valid ISO 3166 alpha-2 country code.")) + return errors + + def _validate_scanner(conf): errors = [] sec = 'scanner' @@ -288,7 +305,7 @@ def _validate_scanner(conf): 'download_max': {'minimum': 0.001, 'maximum': None}, } all_valid_keys = list(ints.keys()) + list(floats.keys()) + \ - ['nickname'] + ['nickname', 'country'] errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) errors.extend(_validate_section_ints(conf, sec, ints, err_tmpl)) errors.extend(_validate_section_floats(conf, sec, floats, err_tmpl)) @@ -296,6 +313,7 @@ def _validate_scanner(conf): if not valid: errors.append(err_tmpl.substitute( sec=sec, key='nickname', val=conf[sec]['nickname'], e=error_msg)) + errors.extend(_validate_country(conf, sec, 'country', err_tmpl)) return errors
@@ -388,7 +406,7 @@ def _validate_destinations(conf): urls = { 'url': {}, } - all_valid_keys = list(urls.keys()) + ['verify'] + all_valid_keys = list(urls.keys()) + ['verify', 'country'] for sec in dest_sections: if sec not in conf: errors.append('{} is an enabled destination but is not a ' @@ -397,6 +415,7 @@ def _validate_destinations(conf): errors.extend(_validate_section_keys( conf, sec, all_valid_keys, err_tmpl, allow_missing=['verify'])) errors.extend(_validate_section_urls(conf, sec, urls, err_tmpl)) + errors.extend(_validate_country(conf, sec, 'country', err_tmpl)) return errors
diff --git a/sbws/util/iso3166.py b/sbws/util/iso3166.py new file mode 100644 index 0000000..d537c18 --- /dev/null +++ b/sbws/util/iso3166.py @@ -0,0 +1,37 @@ +""" +ISO 3166 alpha-2 countries' codes. +Obtained from https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes. +Last updated 2019/02/05. +ZZ is not the code of any country and it is used to denote any country, +when the destination Web Server is in a CDN. +""" +# When the destination Web Server is in a CDN, the IP could be resolved by +# the exit relay and obtain the country from the IP. + +# It would be better to use some standard location system for geopgraphic areas +# that doesn't depend on political borders. +# It should be possible to obtain IP address location in that system too. + +ISO_3166_ALPHA_2 = [ + 'AF', 'AX', 'AL', 'DZ', 'AS', 'AD', 'AO', 'AI', 'AQ', 'AG', 'AR', 'AM', + 'AW', 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', + 'BM', 'BT', 'BO', 'BQ', 'BA', 'BW', 'BV', 'BR', 'IO', 'BN', 'BG', 'BF', + 'BI', 'CV', 'KH', 'CM', 'CA', 'KY', 'CF', 'TD', 'CL', 'CN', 'CX', 'CC', + 'CO', 'KM', 'CD', 'CG', 'CK', 'CR', 'CI', 'HR', 'CU', 'CW', 'CY', 'CZ', + 'DK', 'DJ', 'DM', 'DO', 'EC', 'EG', 'SV', 'GQ', 'ER', 'EE', 'SZ', 'ET', + 'FK', 'FO', 'FJ', 'FI', 'FR', 'GF', 'PF', 'TF', 'GA', 'GM', 'GE', 'DE', + 'GH', 'GI', 'GR', 'GL', 'GD', 'GP', 'GU', 'GT', 'GG', 'GN', 'GW', 'GY', + 'HT', 'HM', 'VA', 'HN', 'HK', 'HU', 'IS', 'IN', 'ID', 'IR', 'IQ', 'IE', + 'IM', 'IL', 'IT', 'JM', 'JP', 'JE', 'JO', 'KZ', 'KE', 'KI', 'KP', 'KR', + 'KW', 'KG', 'LA', 'LV', 'LB', 'LS', 'LR', 'LY', 'LI', 'LT', 'LU', 'MO', + 'MK', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MH', 'MQ', 'MR', 'MU', 'YT', + 'MX', 'FM', 'MD', 'MC', 'MN', 'ME', 'MS', 'MA', 'MZ', 'MM', 'NA', 'NR', + 'NP', 'NL', 'NC', 'NZ', 'NI', 'NE', 'NG', 'NU', 'NF', 'MP', 'NO', 'OM', + 'PK', 'PW', 'PS', 'PA', 'PG', 'PY', 'PE', 'PH', 'PN', 'PL', 'PT', 'PR', + 'QA', 'RE', 'RO', 'RU', 'RW', 'BL', 'SH', 'KN', 'LC', 'MF', 'PM', 'VC', + 'WS', 'SM', 'ST', 'SA', 'SN', 'RS', 'SC', 'SL', 'SG', 'SX', 'SK', 'SI', + 'SB', 'SO', 'ZA', 'GS', 'SS', 'ES', 'LK', 'SD', 'SR', 'SJ', 'SE', 'CH', + 'SY', 'TW', 'TJ', 'TZ', 'TH', 'TL', 'TG', 'TK', 'TO', 'TT', 'TN', 'TR', + 'TM', 'TC', 'TV', 'UG', 'UA', 'AE', 'GB', 'UM', 'US', 'UY', 'UZ', 'VU', + 'VE', 'VN', 'VG', 'VI', 'WF', 'EH', 'YE', 'ZM', 'ZW', 'ZZ' + ] diff --git a/tests/unit/util/test_config.py b/tests/unit/util/test_config.py index 95dde6f..4ebc703 100644 --- a/tests/unit/util/test_config.py +++ b/tests/unit/util/test_config.py @@ -235,6 +235,36 @@ def test_nickname(): assert not valid, reason
+def test_country(conf): + from string import Template + err_tmpl = Template('$sec/$key ($val): $e') + + # Invalid default country code in scanner section + errors = con._validate_country(conf, 'scanner', 'country', err_tmpl) + assert errors[0] == \ + 'scanner/country (AA): Not a valid ISO 3166 alpha-2 country code.' + + # Valid country code in scanner section + conf['scanner']['country'] = 'US' + errors = con._validate_country(conf, 'scanner', 'country', err_tmpl) + assert not errors + + # No country in destinations.foo section + conf['destinations']['foo'] = 'on' + conf['destinations.foo'] = {} + conf['destinations.foo']['url'] = 'https://foo.bar' + errors = con._validate_country( + conf, 'destinations.foo', 'country', err_tmpl) + assert errors[0] == \ + 'destinations.foo/country (None): ' \ + 'Missing country in configuration file.' + + # Valid country in destinations.foo section + conf['destinations.foo']['url'] = 'US' + errors = con._validate_country(conf, 'scanner', 'country', err_tmpl) + assert not errors + + def test_config_arg_provided_but_no_found(args, conf): args.config = 'non_existing_file' user_conf = con._get_user_config(args, conf)