February 2021 - tor-commits - lists.torproject.org

[sbws/master] chg: config: Add option to connect to external tor
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit f6e300c83508508a6f38abe76390f9c6c3de00a2 Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:14:48 2020 +0000 chg: config: Add option to connect to external tor via control port. --- docs/source/man_sbws.ini.rst | 4 ++++ sbws/config.default.ini | 1 + sbws/util/config.py | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/man_sbws.ini.rst b/docs/source/man_sbws.ini.rst index f878ce8..e2127ab 100644 --- a/docs/source/man_sbws.ini.rst +++ b/docs/source/man_sbws.ini.rst @@ -93,6 +93,10 @@ tor sbws's owned tor pid file. (Default: ~/.sbws/tor/sbws/tor.pid) log = STR sbws's owned tor directory log files. (Default: ~/.sbws/tor/log) + external_control_port = INT + tor control port to connect to. Useful to run integration tests with + chutney. + (Default: not set. If set, it takes preference over the control socket) extra_lines = sbws's tor extra configuration. (Default: None) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index cb943d0..0105527 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -66,6 +66,7 @@ control_socket = ${tor:run_dpath}/control pid = ${tor:run_dpath}/tor.pid # note this is a directory log = ${tor:datadir}/log +external_control_port = extra_lines = [cleanup] diff --git a/sbws/util/config.py b/sbws/util/config.py index e1bc429..5ba3fd5 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -323,7 +323,9 @@ def _validate_tor(conf): sec = 'tor' err_tmpl = Template('$sec/$key ($val): $e') unvalidated_keys = [ - 'datadir', 'run_dpath', 'control_socket', 'pid', 'log', 'extra_lines'] + 'datadir', 'run_dpath', 'control_socket', 'pid', 'log', + 'external_control_port', 'extra_lines', + ] all_valid_keys = unvalidated_keys errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) return errors

1 0

[sbws/master] fix: stem: Remove unused code
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 553f4a8fa35a35b686c27a808671b0c1f0f76807 Author: juga0 <juga(a)riseup.net> Date: Wed Dec 16 15:27:55 2020 +0000 fix: stem: Remove unused code --- sbws/util/stem.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sbws/util/stem.py b/sbws/util/stem.py index d2c596f..746fa88 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -80,13 +80,6 @@ def init_controller(port=None, path=None, set_custom_stream_settings=True): if not c: return None, 'Unable to reach tor on control socket' assert c is not None - if set_custom_stream_settings: - # These options are also set in launch_tor. - # In a future refactor they could be set in the case they are not - # already in the running instance. This way the controller_port - # could also be used. - set_torrc_options_can_fail(c) - set_torrc_runtime_options(c) return c, ''

1 0

[sbws/master] Merge branch 'bug_40034' into 'maint-1.1'
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 713d40045a98a9a453a4f24a1ed3830e7fb15e6c Merge: 187aa96 7ab6176 Author: juga <juga(a)riseup.net> Date: Mon Dec 21 09:25:37 2020 +0000 Merge branch 'bug_40034' into 'maint-1.1' fix: doc: Update default values in man_sbws.ini.rst Closes #40034 See merge request tpo/network-health/sbws!46 docs/source/man_sbws.ini.rst | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-)

1 0

[sbws/master] fix: doc: Update default values in man_sbws.ini.rst
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 7ab61765f018adf5002acce2c03773eba7eee2ae Author: Georg Koppen <gk(a)torproject.org> Date: Fri Dec 18 15:12:07 2020 +0000 fix: doc: Update default values in man_sbws.ini.rst Closes #40034. --- docs/source/man_sbws.ini.rst | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/source/man_sbws.ini.rst b/docs/source/man_sbws.ini.rst index 22512b8..f878ce8 100644 --- a/docs/source/man_sbws.ini.rst +++ b/docs/source/man_sbws.ini.rst @@ -22,12 +22,12 @@ general http_timeout = INT Timeout in seconds to give to the python Requests library. (Default: 10) circuit_timeout = INT - Timeout in seconds to create circuits. (Default: 10) + Timeout in seconds to create circuits. (Default: 60) reset_bw_ipv4_changes = {on, off} Whether or not to reset the bandwidth measurements when the relay's IP address changes. If it changes, we only consider results for the relay that we obtained while the relay was located at its most recent IP address. - (Default: on) + (Default: off) reset_bw_ipv6_changes = off NOT implemented for IPv6. @@ -47,13 +47,13 @@ paths (Default: ~/.sbws/v3bw) v3bw_fname = STR File names of the bandwidth list files. - The latest bandwidth file is symlinked by ``latests.v3bw`` - started_filepath = STR + The latest bandwidth file is symlinked by ``latest.v3bw`` + state_fname = STR File path to store the timestamp when the scanner was last started. - (Default: ~/.sbws/started_at) + (Default: ~/.sbws/state.dat) log_dname = STR Directory where to store log files when logging to files is enabled. - (Default ~/.sbws/log) + (Default: ~/.sbws/log) destinations @@ -66,7 +66,7 @@ destinations download files in order to measure bandwidths. usability_test_interval = INT - How often to check if a destination is usable + How often to check if a destination is usable (Default: 300) destinations.STR url = STR @@ -112,7 +112,7 @@ scanner download_max = INT Limits on what download times are too fast/slow/etc. (Default: 10) num_rtts = INT - How many RTT measurements to make. (Default: 10) + How many RTT measurements to make. (Default: 0) num_downloads = INT Number of downloads with acceptable times we must have for a relay before moving on. (Default: 5) @@ -138,15 +138,19 @@ relayprioritizer (Default: 50) cleanup - stale_days = INT + data_files_compress_after_days = INT After this many days, compress data files. (Default: 10) - rotten_days = INT + data_files_delete_after_days = INT After this many days, delete data files. (Default: 90) + v3bw_files_compress_after_days = INT + After this many days, compress v3bw files. (Default: 1) + v3bw_files_delete_after_days = INT + After this many days, delete v3bw files. (Default: 7) logging to_file = {yes, no} Whether or not to log to a rotating file the directory paths.log_dname. - (Default: no) + (Default: yes) to_stdout = {yes, no} Whether or not to log to stdout. (Default: yes) to_syslog = {yes, no} @@ -161,23 +165,24 @@ logging If logging to file, how many backups to keep. If zero or max bytes is zero, never rotate the log file. (Default: 50) level = {debug, info, warning, error, critical} - Level to log at. (Default: info) + Level to log at. (Default: debug) to_file_level = {debug, info, warning, error, critical} - Level to log at when using files. (Default: info) + Level to log at when using files. (Default: debug) to_stdout_level = {debug, info, warning, error, critical} Level to log at when using stdout. (Default: info) to_syslog_level = {debug, info, warning, error, critical} Level to log at when using syslog. (Default: info) format = STR Format string to use when logging. - (Default: [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s) - to_file_format = STR - Format string to use when logging to files. (Default: $format) + (Default: %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s) to_stdout_format = STR - Format string to use when logging to stdout. (Default: $format) + Format string to use when logging to stdout. (Default: ${format}) to_syslog_format = STR Format string to use when logging to syslog. (Default: %(module)s[%(process)s]: <%(levelname)s> %(message)s) + to_file_format = STR + Format string to use when logging to files. + (Default: %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s) EXAMPLES --------

1 0

[sbws/master] fix: doc: Add bwauthealth tool
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 3ddc9f1e4648bcab2ed59059e6f6bdf29f3ce982 Author: juga0 <juga(a)riseup.net> Date: Sun Sep 27 10:40:58 2020 +0000 fix: doc: Add bwauthealth tool --- docs/source/monitoring_bandwidth.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/monitoring_bandwidth.rst b/docs/source/monitoring_bandwidth.rst index dedf90d..09326c4 100644 --- a/docs/source/monitoring_bandwidth.rst +++ b/docs/source/monitoring_bandwidth.rst @@ -24,6 +24,13 @@ It should be approximately equal for all bwauths. https://metrics.torproject.org/totalcw.html + +Not measured relays and descriptors and consensus updates +--------------------------------------------------------- + +Run the tool https://gitlab.torproject.org/juga/bwauthealth. + + Total bandwidth ---------------

1 0

[sbws/master] fix: doc: Add the bwauths timeline wiki
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 81d2bea0008a4ef2fc6ded2ee0bd56a6aa7cbdd7 Author: juga0 <juga(a)riseup.net> Date: Sun Sep 27 10:43:53 2020 +0000 fix: doc: Add the bwauths timeline wiki Closes #40013. --- docs/source/monitoring_bandwidth.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/monitoring_bandwidth.rst b/docs/source/monitoring_bandwidth.rst index 09326c4..d9ebdf2 100644 --- a/docs/source/monitoring_bandwidth.rst +++ b/docs/source/monitoring_bandwidth.rst @@ -1,6 +1,15 @@ Monitoring bandwidth changes in the Tor Network ================================================ +Bandwidth authorities timeline +------------------------------ + +Events that can affect the data generated by the bwauths: + +https://gitlab.torproject.org/tpo/network-health/sbws/-/wikis/bandwidth%20authorities%20timeline + +This page might be moved to a different location. + Bwauths number of measured relays ---------------------------------

1 0

[sbws/master] fix: doc: Add consensus health page
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 6e6a8f3ba534cbd93b830fe3ffd5ce40abe8e77d Author: juga0 <juga(a)riseup.net> Date: Sun Sep 27 10:32:12 2020 +0000 fix: doc: Add consensus health page about bwauths measured relays. --- docs/source/monitoring_bandwidth.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/monitoring_bandwidth.rst b/docs/source/monitoring_bandwidth.rst index 54d1219..dedf90d 100644 --- a/docs/source/monitoring_bandwidth.rst +++ b/docs/source/monitoring_bandwidth.rst @@ -1,6 +1,18 @@ Monitoring bandwidth changes in the Tor Network ================================================ +Bwauths number of measured relays +--------------------------------- + +It should be approximately equal for all bwauths. + +.. image:: images/20200927_consensushealth_bwauths.png + :alt: bwauths measured relays + +https://consensus-health.torproject.org/graphs.html#votedaboutgraphs + +http://tgnv2pssfumdedyw.onion/graphs.html#votedaboutgraphs + Total consensus weights across bandwidth authorities ----------------------------------------------------

1 0

[sbws/master] Merge branch 'maint-1.1' into master
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit c8d1cc4f5d8b68688fd7dca48491531a8a9d4c30 Merge: 2e38dcb 81d2bea Author: Georg Koppen <gk(a)torproject.org> Date: Mon Oct 5 07:12:08 2020 +0000 Merge branch 'maint-1.1' into master docs/source/monitoring_bandwidth.rst | 49 ++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-)

1 0

[sbws/master] fix: doc: Move consensus weight to top
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 75af5c3575670121cb933135ab79ce49e7eb0f7b Author: juga0 <juga(a)riseup.net> Date: Sun Sep 27 10:08:15 2020 +0000 fix: doc: Move consensus weight to top and explain what to check. --- docs/source/monitoring_bandwidth.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/source/monitoring_bandwidth.rst b/docs/source/monitoring_bandwidth.rst index 888d544..54d1219 100644 --- a/docs/source/monitoring_bandwidth.rst +++ b/docs/source/monitoring_bandwidth.rst @@ -1,6 +1,17 @@ Monitoring bandwidth changes in the Tor Network ================================================ +Total consensus weights across bandwidth authorities +---------------------------------------------------- + +It should be approximately equal for all bwauths. + +.. image:: images/totalcw.png + :alt: total consensus weight + + +https://metrics.torproject.org/totalcw.html + Total bandwidth --------------- @@ -23,13 +34,3 @@ Should not increase. https://metrics.torproject.org/torperf.html - - -Total consensus weights across bandwidth authorities ----------------------------------------------------- - -.. image:: images/totalcw.png - :alt: total consensus weight - - -https://metrics.torproject.org/totalcw.html

1 0

[sbws/master] fix: v3bwfile: Tor version added in bandwidth v1.4.0
by juga＠torproject.org 19 Feb '21

19 Feb '21

commit 50821110c908e5da9f1aab3b14883e39bb226614 Author: juga0 <juga(a)riseup.net> Date: Mon Dec 7 11:28:42 2020 +0000 fix: v3bwfile: Tor version added in bandwidth v1.4.0 since, by mistake, the bandwidth file version here was never updated to v1.5.0. This patch only changes the constants names, but logic remains the same. Related to torspec#35. --- sbws/lib/v3bwfile.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 45563b6..553265b 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -125,13 +125,9 @@ HEADER_KEYS_V1_4 = [ 'time_to_report_half_network', ] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS -# KeyValues added in the Bandwidth File v1.5.0 -# XXX: Change SPEC_VERSION when all the v1.5.0 keys are added, before a new -# sbws release. # Tor version will be obtained from the state file, so it won't be pass as an # argument, but will be self-initialized. -HEADER_KEYS_V1_5_TO_INIT = ['tor_version'] -HEADER_KEYS_V1_5 = HEADER_KEYS_V1_5_TO_INIT +HEADER_KEYS_V1_4_TO_INIT = ['tor_version'] # KeyValues that are initialized from arguments, not self-initialized. HEADER_INIT_KEYS = ( @@ -139,7 +135,7 @@ HEADER_INIT_KEYS = ( + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 - + HEADER_KEYS_V1_5_TO_INIT + + HEADER_KEYS_V1_4_TO_INIT ) HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 @@ -150,7 +146,7 @@ HEADER_UNORDERED_KEYS = ( + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 - + HEADER_KEYS_V1_5 + + HEADER_KEYS_V1_4_TO_INIT ) # List of all the KeyValues currently being used to generate the file HEADER_ALL_KEYS = HEADER_KEYS_V1_1_ORDERED + HEADER_UNORDERED_KEYS

1 0