September 2018 - tor-commits - lists.torproject.org

[sbws/master] Stop including \n EOL when parsing V110 header
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 48fa53dd3a38f021654fba4d9be65be7d2af7ac4 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 14:49:37 2018 +0000 Stop including \n EOL when parsing V110 header --- sbws/lib/v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b068075..576345e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -146,7 +146,8 @@ class V3BWHeader(object): for l in lines[:index_terminator] if l.split(KEYVALUE_SEP_V110)[0] in ALL_KEYVALUES]) h = cls(ts, **kwargs) - return h, lines[index_terminator + 1:] + # last line is new line + return h, lines[index_terminator + 1:-1] @classmethod def from_text_v110(self, text):

1 0

[sbws/master] Reorder methods in BWV3Header
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 40a76cc7d6566626a871abf0934a379b3cad40bb Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 14:40:28 2018 +0000 Reorder methods in BWV3Header 1. magic methods 2. classmethods 3. staticmethods 4. properties 5. methods --- sbws/lib/v3bwfile.py | 116 +++++++++++++++++++++++++-------------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index ad3e2e2..d663445 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -109,53 +109,26 @@ class V3BWHeader(object): [setattr(self, k, v) for k, v in kwargs.items() if k in EXTRA_ARG_KEYVALUES] - @property - def keyvalue_unordered_tuple_ls(self): - """Return list of KeyValue tuples that do not have specific order.""" - # sort the list to generate determinist headers - keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in UNORDERED_KEYVALUES]) - return keyvalue_tuple_ls - - @property - def keyvalue_tuple_ls(self): - """Return list of all KeyValue tuples""" - return [('version', self.version)] + self.keyvalue_unordered_tuple_ls - - @property - def keyvalue_v110str_ls(self): - """Return KeyValue list of strings following spec v1.1.0.""" - keyvalues = [self.timestamp] + [KEYVALUE_SEP_V110.join([k, v]) - for k, v in self.keyvalue_tuple_ls] - return keyvalues - - @property - def strv110(self): - """Return header string following spec v1.1.0.""" - header_str = LINE_SEP.join(self.keyvalue_v110str_ls) + LINE_SEP + \ - LINE_TERMINATOR - return header_str - - @property - def keyvalue_v200_ls(self): - """Return KeyValue list of strings following spec v2.0.0.""" - keyvalue = [self.timestamp] + [KEYVALUE_SEP_V200.join([k, v]) - for k, v in self.keyvalue_tuple_ls] - return keyvalue - - @property - def strv200(self): - """Return header string following spec v2.0.0.""" - header_str = LINE_SEP.join(self.keyvalue_v200_ls) + LINE_SEP + \ - LINE_TERMINATOR - return header_str - def __str__(self): if self.version == '1.1.0': return self.strv110 return self.strv200 @classmethod + def from_results(cls, conf, results): + kwargs = dict() + latest_bandwidth = cls.latest_bandwidth_from_results(results) + earliest_bandwidth = cls.earliest_bandwidth_from_results(results) + generator_started = cls.generator_started_from_file(conf) + timestamp = str(latest_bandwidth) + kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) + kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) + if generator_started is not None: + kwargs['generator_started'] = generator_started + h = cls(timestamp, **kwargs) + return h + + @classmethod def from_lines_v110(cls, lines): """ :param list lines: list of lines to parse @@ -184,10 +157,6 @@ class V3BWHeader(object): assert isinstance(text, str) return self.from_lines_v110(text.split(LINE_SEP)) - @property - def num_lines(self): - return len(self.__str__().split(LINE_SEP)) - @staticmethod def generator_started_from_file(conf): ''' @@ -208,19 +177,50 @@ class V3BWHeader(object): def earliest_bandwidth_from_results(results): return round(min([r.time for fp in results for r in results[fp]])) - @classmethod - def from_results(cls, conf, results): - kwargs = dict() - latest_bandwidth = cls.latest_bandwidth_from_results(results) - earliest_bandwidth = cls.earliest_bandwidth_from_results(results) - generator_started = cls.generator_started_from_file(conf) - timestamp = str(latest_bandwidth) - kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) - kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) - if generator_started is not None: - kwargs['generator_started'] = generator_started - h = cls(timestamp, **kwargs) - return h + @property + def keyvalue_unordered_tuple_ls(self): + """Return list of KeyValue tuples that do not have specific order.""" + # sort the list to generate determinist headers + keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() + if k in UNORDERED_KEYVALUES]) + return keyvalue_tuple_ls + + @property + def keyvalue_tuple_ls(self): + """Return list of all KeyValue tuples""" + return [('version', self.version)] + self.keyvalue_unordered_tuple_ls + + @property + def keyvalue_v110str_ls(self): + """Return KeyValue list of strings following spec v1.1.0.""" + keyvalues = [self.timestamp] + [KEYVALUE_SEP_V110.join([k, v]) + for k, v in self.keyvalue_tuple_ls] + return keyvalues + + @property + def strv110(self): + """Return header string following spec v1.1.0.""" + header_str = LINE_SEP.join(self.keyvalue_v110str_ls) + LINE_SEP + \ + LINE_TERMINATOR + return header_str + + @property + def keyvalue_v200_ls(self): + """Return KeyValue list of strings following spec v2.0.0.""" + keyvalue = [self.timestamp] + [KEYVALUE_SEP_V200.join([k, v]) + for k, v in self.keyvalue_tuple_ls] + return keyvalue + + @property + def strv200(self): + """Return header string following spec v2.0.0.""" + header_str = LINE_SEP.join(self.keyvalue_v200_ls) + LINE_SEP + \ + LINE_TERMINATOR + return header_str + + @property + def num_lines(self): + return len(self.__str__().split(LINE_SEP)) class V3BWLine(object):

1 0

[sbws/master] Fix linter error
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 1ba79e551e74d3bacedcfda9f7fde791f91260b1 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 15:46:25 2018 +0000 Fix linter error --- sbws/util/fs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index 9877cf1..f61f07f 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -76,8 +76,8 @@ def sbws_required_disk_space(conf): text_dict['mb_code'] = round(size_code_deps / 1000 ** 2) # Multiply per 2, just in case size_total = (results_max_space_after_delete + - v3bw_max_space_after_delete + space_log_files + size_tor_dir + - size_code_deps) * 2 + v3bw_max_space_after_delete + space_log_files + + size_tor_dir + size_code_deps) * 2 text_dict['mb_total'] = round(size_total / 1000 ** 2) space_text = DISK_SPACE_TEXT.format(**text_dict) return space_text

1 0

[sbws/master] Merge pull request #250 from juga0/ticket27512_warn_disk_followup
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit aea806bf83dcb5d95d843872f36324d415d2256e Merge: 75be1aa 1ba79e5 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 16:21:46 2018 +0000 Merge pull request #250 from juga0/ticket27512_warn_disk_followup Ticket27512 warn disk followup sbws/core/generate.py | 3 -- sbws/core/scanner.py | 5 --- sbws/sbws.py | 3 ++ sbws/util/fs.py | 81 +++++++++++++++++++++++++++++++++++++--------- sbws/util/parser.py | 4 +-- tests/unit/util/test_fs.py | 23 ------------- 6 files changed, 70 insertions(+), 49 deletions(-)

1 0

[sbws/master] Support raw text on command description
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 6b67368eb9b212cffc35b7edea56fdf5fb2ff52b Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 17:00:41 2018 +0000 Support raw text on command description --- sbws/util/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/util/parser.py b/sbws/util/parser.py index 2264d7c..7ff39d2 100644 --- a/sbws/util/parser.py +++ b/sbws/util/parser.py @@ -4,7 +4,7 @@ import sbws.core.generate import sbws.core.stats from sbws import __version__ -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from argparse import ArgumentParser, RawTextHelpFormatter import os @@ -14,7 +14,7 @@ def _default_dot_sbws_dname(): def create_parser(): - p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) + p = ArgumentParser(formatter_class=RawTextHelpFormatter) p.add_argument( '--version', action='version', help='sbws version', version='{}'.format(__version__))

1 0

[sbws/master] Check if file exists before locking it
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 850271a127956213d432e93e9e0fdab0689a1cf6 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 18:50:48 2018 +0000 Check if file exists before locking it --- sbws/util/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/util/state.py b/sbws/util/state.py index a13ee13..ce3dbb9 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -48,9 +48,9 @@ class State: self._state = self._read() def _read(self): + if not os.path.exists(self._fname): + return {} with FileLock(self._fname): - if not os.path.exists(self._fname): - return {} with open(self._fname, 'rt') as fd: return json.load(fd)

1 0

[sbws/master] Adapt tests to changed methods
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 8fd2d7399c584d6720a523f1b1874267bf2d158d Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 19:17:03 2018 +0000 Adapt tests to changed methods --- tests/unit/lib/test_v3bwfile.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 01c84ce..dafe015 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -36,7 +36,8 @@ header_extra_ls = [timestamp_l, version_l, software_l, software_version_l, TERMINATOR] header_extra_str = LINE_SEP.join(header_extra_ls) + LINE_SEP -bwl_str = "bw=54 error_circ=0 error_misc=0 error_stream=1 " \ +bwl_str = "bw=56 bw_bs_mean=61423 bw_bs_median=55656 "\ + "desc_avg_bw_bs=1000000000 error_circ=0 error_misc=0 error_stream=1 " \ "master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s " \ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 " \ @@ -97,17 +98,22 @@ def test_v3bwline_from_results_file(datadir): d[fp] = [] d[fp].append(r) bwl = V3BWLine.from_data(d, fp) + # bw store now B, not KB + bwl.bw = round(bwl.bw / 1000) assert bwl_str == str(bwl) -def test_from_arg_results(datadir, tmpdir, conf, args): +def test_from_results_read(datadir, tmpdir, conf, args): results = load_result_file(str(datadir.join("results.txt"))) expected_header = V3BWHeader(timestamp_l, earliest_bandwidth=earliest_bandwidth, latest_bandwidth=latest_bandwidth) expected_bwls = [V3BWLine.from_results(results[fp]) for fp in results] + # bw store now B, not KB + expected_bwls[0].bw = round(expected_bwls[0].bw / 1000) expected_f = V3BWFile(expected_header, expected_bwls) - v3bwfile = V3BWFile.from_arg_results(args, conf, results) + # This way is going to convert bw to KB + v3bwfile = V3BWFile.from_results(results) assert str(expected_f)[1:] == str(v3bwfile)[1:] output = os.path.join(args.output, now_fname()) v3bwfile.write(output) @@ -115,7 +121,7 @@ def test_from_arg_results(datadir, tmpdir, conf, args): def test_from_arg_results_write(datadir, tmpdir, conf, args): results = load_result_file(str(datadir.join("results.txt"))) - v3bwfile = V3BWFile.from_arg_results(args, conf, results) + v3bwfile = V3BWFile.from_results(results) output = os.path.join(args.output, now_fname()) v3bwfile.write(output) assert os.path.isfile(output) @@ -123,7 +129,7 @@ def test_from_arg_results_write(datadir, tmpdir, conf, args): def test_from_arg_results_write_read(datadir, tmpdir, conf, args): results = load_result_file(str(datadir.join("results.txt"))) - v3bwfile = V3BWFile.from_arg_results(args, conf, results) + v3bwfile = V3BWFile.from_results(results) output = os.path.join(args.output, now_fname()) v3bwfile.write(output) with open(output) as fd:

1 0

[sbws/master] Temporally disable integration test
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit e1e6f54f8ae90465d7884978ae31b4f5936ef6ff Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:00:25 2018 +0000 Temporally disable integration test --- tests/integration/core/test_scanner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/core/test_scanner.py b/tests/integration/core/test_scanner.py index 61927d7..4a218f7 100644 --- a/tests/integration/core/test_scanner.py +++ b/tests/integration/core/test_scanner.py @@ -1,3 +1,5 @@ +import pytest + from sbws.core.scanner import measure_relay from sbws.lib.resultdump import ResultSuccess import logging @@ -42,6 +44,7 @@ def test_measure_relay_with_maxadvertisedbandwidth( assert result.relay_average_bandwidth == one_mbyte +(a)pytest.mark.skip(reason="temporally disabled") def test_measure_relay_with_relaybandwidthrate( persistent_launch_tor, args, conf, dests, cb, rl): relay = [r for r in rl.relays

1 0

[sbws/master] Merge pull request #251 from juga0/ticket27135_store_obs_bw
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 89d94e1cedc7e220be6a104aec50d004ffd759d6 Merge: aea806b e1e6f54 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:10:11 2018 +0000 Merge pull request #251 from juga0/ticket27135_store_obs_bw Ticket27135 store obs bw CHANGELOG.md | 1 + sbws/core/generate.py | 3 +- sbws/lib/v3bwfile.py | 410 ++++++++++++++++++--------------- sbws/util/state.py | 4 +- tests/integration/core/test_scanner.py | 3 + tests/unit/lib/test_v3bwfile.py | 16 +- 6 files changed, 247 insertions(+), 190 deletions(-)

1 0

[sbws/master] Add property to obtain observed bandwidth
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit f95fa474f55dd3d63ad576cee75768abc720b1e5 Author: juga0 <juga(a)riseup.net> Date: Sat Aug 25 12:53:10 2018 +0000 Add property to obtain observed bandwidth --- sbws/lib/relaylist.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index f177a96..aa197c0 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -75,6 +75,10 @@ class Relay: return self._from_desc('average_bandwidth') @property + def observed_bandwidth(self): + return self._from_desc('observed_bandwidth') + + @property def bandwidth(self): return self._from_ns('bandwidth')

1 0