September 2018 - tor-commits - lists.torproject.org

[sbws/master] Fix syntax
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit f0c873e6cb84c627c0aa6f6706cc4f8890d57c71 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 08:01:12 2018 +0000 Fix syntax --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index 962e4ab..67540b0 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -90,7 +90,7 @@ To build the manual (``man``) pages:: cd docs/ && make man -The generated man pages will be in `docs/man/`. +The generated man pages will be in ``docs/man/``. To build the documentation diagrams::

1 0

[sbws/master] Fix syntax
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit bda5d187ca99a5a96ae6b7844b47f4a6f7ce94b0 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 08:01:04 2018 +0000 Fix syntax --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index 73d960f..962e4ab 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -84,7 +84,7 @@ To build the documentation as HTML:: cd docs/ && make html -The generated HTML will be in `docs/build/`. +The generated HTML will be in ``docs/build/``. To build the manual (``man``) pages::

1 0

[sbws/master] Exit on OSError when building circuit
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit 949e67637772597621a6060673270a6df1ecdec4 Author: juga0 <juga(a)riseup.net> Date: Sun Jul 15 14:28:13 2018 +0000 Exit on OSError when building circuit --- sbws/core/scanner.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 4e59498..0fe7908 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -198,7 +198,16 @@ def measure_relay(args, conf, destinations, cb, rl, relay): our_nick = conf['scanner']['nickname'] circ_id = cb.build_circuit(circ_fps) if not circ_id: - log.warning('Could not build circuit involving %s', relay.nickname) + try: + log.warning('Could not build circuit involving %s', relay.nickname) + except OSError as e: + # if the error is no space left (logging to file system) + # can not log the error, so print it + # this can happen in any log call, here would happen when measuring + # new relay starts + if e.code == 28: # No space left on device + print("ERROR: " + str(e)) + exit(1) msg = 'Unable to complete circuit' return [ ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, msg=msg),

1 0

[sbws/master] Add functions to check disk space
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit 2f39df44fb4b539498cbabe1d072b8c100a7da63 Author: juga0 <juga(a)riseup.net> Date: Sun Jul 15 20:35:23 2018 +0000 Add functions to check disk space --- sbws/util/fs.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/sbws/util/fs.py b/sbws/util/fs.py new file mode 100644 index 0000000..45fd468 --- /dev/null +++ b/sbws/util/fs.py @@ -0,0 +1,50 @@ +"""Utils file system functions""" + +import logging +import shutil + +log = logging.getLogger(__name__) + + +def sbws_required_disk_space(conf): + """Disk space required by sbws files. + Rough calculations. + + :param ConfigParser conf: sbws configuration + :returns: int, size in MiB + """ + # Number of relays per line average size in Bytes + size_v3bw_file = 7500 * 220 + num_v3bw_files = int(conf['general']['data_period']) + # not counting compressed files + space_v3bw_files = size_v3bw_file * num_v3bw_files + # not counted rotated files and assuming that when it is not rotated the + # size will be aproximately 10MiB + size_log_file = (int(conf['logging']['to_file_max_bytes']) or 10485760) \ + if conf['logging']['to_stdout'] == 'yes' else 0 + # roughly... + space_result_files = space_v3bw_files + # duplicate everything to warn early + size_total = (space_v3bw_files + size_log_file + space_result_files) * 2 + size_total_mb = round(size_total / (1024 ** 2)) + return size_total_mb + + +def df(path): + """Return space left on device where path is.""" + return round(shutil.disk_usage(path).free / (1024 ** 2)) + + +def is_low_space(conf): + """Warn and return True when the space left on the device is less than + what is needed for sbws and False otherwise needs. + """ + disk_required_mb = sbws_required_disk_space(conf) + disk_avail_mb = df(conf['paths']['sbws_home']) + if disk_avail_mb < disk_required_mb: + log.warn("The space left on the device (%s MiB) is less than " + "the minimum recommented to run sbws (%s MiB)." + "Run sbws cleanup to delete old sbws generated files.", + disk_avail_mb, disk_required_mb) + return True + return False

1 0

[sbws/master] Merge branch 'ticket27341'
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit eca0d2685cac7bc20571d3d476eda7fcdf48f0a7 Merge: 48715b7 9239f69 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Sep 5 10:35:31 2018 -0400 Merge branch 'ticket27341' CHANGELOG.md | 12 +++ DEPLOY.rst | 84 ++++-------------- INSTALL.rst | 198 +++++++++++++++---------------------------- README.md | 17 ---- docs/source/glossary.rst | 52 +++++++----- docs/source/man_sbws.ini.rst | 39 +++++++-- docs/source/man_sbws.rst | 6 ++ 7 files changed, 169 insertions(+), 239 deletions(-) diff --cc CHANGELOG.md index b760644,f45182e..1de8368 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -7,11 -7,18 +7,23 @@@ and this project adheres to [Semantic V ## [Unreleased] +### Fixed + +- Broken environment variable in default sbws config. To use envvar $FOO, write + $$FOO in the config. + + ### Changed + + - sbws install doc is confusing (#27341) + - Include system and Python dependencies in `INSTALL`. + - Include dependencies for docs and tests in `INSTALL`. + - Point to `DEPLOY` to run sbws. + - Remove obsolete sections in `INSTALL` + - Simplify `DEPLOY`, reuse terms in the `glossary`. + - Remove obsolete ``sbws init`` from `DEPLOY`. + - Point to config documentation. + - Add, unify and reuse terms in `glossary`. + ## [0.7.0] - 2018-08-09 **Important changes**:

1 0

[sbws/master] Check disk space before running generate
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit 7aaf0a5b4342840ae07827d3becfcd45b6351a01 Author: juga0 <juga(a)riseup.net> Date: Sun Jul 15 20:37:02 2018 +0000 Check disk space before running generate --- sbws/core/scanner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 0fe7908..8b13816 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -21,6 +21,9 @@ import logging import requests import random +from ..util.fs import is_low_space + + rng = random.SystemRandom() end_event = Event() log = logging.getLogger(__name__) @@ -377,6 +380,9 @@ def gen_parser(sub): def main(args, conf): + if is_low_space(conf): + exit(1) + if conf.getint('scanner', 'measurement_threads') < 1: fail_hard('Number of measurement threads must be larger than 1')

1 0

[sbws/master] Calculate space for result files and tor dir
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit d92f85aeef967846f3489d582fd9194ceba13a3a Author: juga0 <juga(a)riseup.net> Date: Fri Aug 3 12:52:05 2018 +0000 Calculate space for result files and tor dir also replace conf[''] for conf.getint/conf.getboolean --- sbws/util/fs.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index 45fd468..111e109 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -15,15 +15,21 @@ def sbws_required_disk_space(conf): """ # Number of relays per line average size in Bytes size_v3bw_file = 7500 * 220 - num_v3bw_files = int(conf['general']['data_period']) + # the minimum number of required v3bw files + num_v3bw_files = 2 + # ~1000 is the length of a line when the result is successfull + # ~4550 is the number of lines of the biggest result file + size_result_file = 4550 * 1000 + num_result_files = int(conf['general']['data_period']) # not counting compressed files space_v3bw_files = size_v3bw_file * num_v3bw_files + space_result_files = size_result_file * num_result_files # not counted rotated files and assuming that when it is not rotated the # size will be aproximately 10MiB - size_log_file = (int(conf['logging']['to_file_max_bytes']) or 10485760) \ - if conf['logging']['to_stdout'] == 'yes' else 0 - # roughly... - space_result_files = space_v3bw_files + size_log_file = conf.getint('logging', 'to_file_max_bytes') or 10485760 \ + if conf.getboolean('logging', 'to_stdout') else 0 + # roughly, size of a current tor dir + space_tor_dir = 19828000 # duplicate everything to warn early size_total = (space_v3bw_files + size_log_file + space_result_files) * 2 size_total_mb = round(size_total / (1024 ** 2))

1 0

[sbws/master] Specify the unit is MiB
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit b5002e7e921a04bb32f8f4a0210fbcc58bdfcbcd Author: juga0 <juga(a)riseup.net> Date: Fri Aug 3 12:54:06 2018 +0000 Specify the unit is MiB --- sbws/util/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index 5b48f3b..0f20d0a 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -39,7 +39,7 @@ def sbws_required_disk_space(conf): def df(path): - """Return space left on device where path is.""" + """Return space left on device where path is in MiB.""" return round(shutil.disk_usage(path).free / (1024 ** 2))

1 0

[sbws/master] Add unit tests for disk space functions
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit 07140b3433e32906bdeab75a669ded41356ff216 Author: juga0 <juga(a)riseup.net> Date: Sun Jul 15 20:35:58 2018 +0000 Add unit tests for disk space functions --- tests/unit/util/test_fs.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/util/test_fs.py b/tests/unit/util/test_fs.py new file mode 100644 index 0000000..0f4c35a --- /dev/null +++ b/tests/unit/util/test_fs.py @@ -0,0 +1,23 @@ +"""Unit tests for fs module""" +from unittest.mock import patch + +from sbws.util import fs + + +def mock_df_zero(path): + return 0 + + +def mock_df_enough(path): + return 32 + + +@patch('sbws.util.fs.df', mock_df_zero) +def test_is_low_space_true(caplog, conf): + assert fs.is_low_space(conf) is True + assert ' is less than ' in caplog.records[-1].getMessage() + + +@patch('sbws.util.fs.df', mock_df_enough) +def test_is_low_space_false(conf): + assert fs.is_low_space(conf) is False

1 0

[sbws/master] Update the total
by pastly＠torproject.org 05 Sep '18

05 Sep '18

commit a602a54e12f200566751e417ed6614a8900aead2 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 3 12:53:39 2018 +0000 Update the total and add comment about MiB --- sbws/util/fs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index 111e109..5b48f3b 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -31,7 +31,9 @@ def sbws_required_disk_space(conf): # roughly, size of a current tor dir space_tor_dir = 19828000 # duplicate everything to warn early - size_total = (space_v3bw_files + size_log_file + space_result_files) * 2 + size_total = (space_v3bw_files + size_log_file + space_result_files + + space_tor_dir) * 2 + # convert to MiB size_total_mb = round(size_total / (1024 ** 2)) return size_total_mb

1 0