commit 5372eb7d09ca1cdcbd8418a38e5dda236cc7e93e Author: juga0 <juga@riseup.net> Date: Sat May 26 17:39:46 2018 +0000 Add class to manage bw files * Move logic in generate to class * Add test * Move 7500 to global constant * Make read_started_at not fail when conf is None --- sbws/core/generate.py | 17 ++++-------- sbws/globals.py | 2 ++ sbws/lib/v3bwfile.py | 58 ++++++++++++++++++++++++++++++++++++++++- tests/lib/data/v3bw.txt | 10 +++++++ tests/unit/lib/test_v3bwfile.py | 16 +++++++++--- 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index e7dd037..4921f57 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,6 +1,5 @@ -from sbws.globals import (fail_hard, is_initted) -from sbws.lib.v3bwfile import V3BwHeader, V3BWLine -from sbws.lib.resultdump import ResultSuccess +from sbws.globals import (fail_hard, is_initted, SCALE_CONSTANT) +from sbws.lib.v3bwfile import V3BwFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter import os @@ -22,7 +21,7 @@ def gen_parser(sub): # time, torflow happened to generate output that averaged to 7500 bw units # per relay. We wanted the ability to try to be like torflow. See # https://lists.torproject.org/pipermail/tor-dev/2018-March/013049.html - p.add_argument('--scale-constant', default=7500, type=int, + p.add_argument('--scale-constant', default=SCALE_CONSTANT, type=int, help='When scaling bw weights, scale them using this const ' 'multiplied by the number of measured relays') p.add_argument('--scale', action='store_true', @@ -32,13 +31,6 @@ def gen_parser(sub): 'out, and we do so proportionally') -def log_stats(data_lines): - assert len(data_lines) > 0 - total_bw = sum([l.bw for l in data_lines]) - bw_per_line = total_bw / len(data_lines) - log.info('Mean bandwidth per line: %f "KiB"', bw_per_line) - - def main(args, conf): if not is_initted(args.directory): fail_hard('Sbws isn\'t initialized. Try sbws init') @@ -56,4 +48,5 @@ def main(args, conf): log.warning('No recent results, so not generating anything. (Have you ' 'ran sbws scanner recently?)') return - + bw_file = V3BwFile.from_arg_results(args, conf, results) + log.info('Mean bandwidth per line: %f "KiB"', bw_file.avg_bw) diff --git a/sbws/globals.py b/sbws/globals.py index 0026ba2..602075a 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -24,6 +24,8 @@ TORRC_STARTING_POINT = { 'UseEntryGuards': '0', } +SCALE_CONSTANT = 7500 + def is_initted(d): if not os.path.isdir(d): diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index d7a5d2b..4b6e6bc 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -70,7 +70,10 @@ def read_started_ts(conf): :param ConfigParser conf: configuration :returns: str, ISO formated timestamp """ - filepath = conf['paths']['started_filepath'] + try: + filepath = conf['paths']['started_filepath'] + except TypeError as e: + return '' try: with FileLock(filepath): with open(filepath, 'r') as fd: @@ -336,3 +339,56 @@ class V3BWLine(object): assert fingerprint in data return cls.from_results(data[fingerprint]) + +class V3BwFile(object): + def __init__(self, v3bwheader, v3bwlines): + """ + :param V3BWHeader v3bwheader: + :param list v3bwlines: + """ + self.header = v3bwheader + self.bw_lines = v3bwlines + + def __str__(self): + return str(self.header) + ''.join([str(bw_line) + for bw_line in self.bw_lines]) + + @property + def total_bw(self): + return total_bw(self.bw_lines) + + @property + def num_lines(self): + return len(self.bw_lines) + + @property + def avg_bw(self): + return self.total_bw / self.num_lines + + @classmethod + def from_results(cls, conf, output, results): + bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] + bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) + header = V3BwHeader.from_results(conf, results) + f = cls(header, bw_lines) + f.write(output) + return f + + @classmethod + def from_arg_results(cls, args, conf, results): + bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] + bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) + if args.scale: + bw_lines = scale_lines(bw_lines, args.scale_constant) + header = V3BwHeader.from_results(conf, results) + f = cls(header, bw_lines) + output = args.output or conf['paths']['v3bw_fname'] + f.write(output) + return f + + def write(self, output): + log.info('Writing v3bw file to %s', output) + with open(output, 'wt') as fd: + fd.write(str(self.header)) + for line in self.bw_lines: + fd.write(str(line)) diff --git a/tests/lib/data/v3bw.txt b/tests/lib/data/v3bw.txt new file mode 100644 index 0000000..09f4a6f --- /dev/null +++ b/tests/lib/data/v3bw.txt @@ -0,0 +1,10 @@ +1523974147 +version=1.1.0 +earliest_bandwidth=2018-04-16T14:09:07 +file_created=2018-04-25T13:10:57 +generator_started=2018-04-16T14:09:05 +latest_bandwidth=2018-04-17T14:09:07 +software=sbws +software_version=0.3.1-dev +==== +bw=54 error_auth=0 error_circ=0 error_misc=0 error_stream=1 last_time=2018-04-17T14:09:07 nick=A node_id=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA success=1 diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 8af0869..3127a92 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- """Test generation of bandwidth measurements document (v3bw)""" import json -import os.path from sbws import __version__ as version from sbws.globals import SPEC_VERSION from sbws.lib.resultdump import Result, load_result_file from sbws.lib.v3bwfile import (V3BwHeader, V3BWLine, TERMINATOR, LINE_SEP, - KEYVALUE_SEP_V110, num_results_of_type) + KEYVALUE_SEP_V110, num_results_of_type, + V3BwFile) timestamp = 1523974147 timestamp_l = str(timestamp) @@ -152,4 +152,14 @@ def test_v3bwline_from_results_file(datadir): def test_v3bwfile(datadir, tmpdir): """Test generate v3bw file (including relay_lines).""" - pass + v3bw = datadir.read('v3bw.txt') + results = load_result_file(str(datadir.join("results.txt"))) + header = V3BwHeader(timestamp_l, + file_created=file_created, + generator_started=generator_started, + earliest_bandwidth=earliest_bandwidth) + bwls = [V3BWLine.from_results(results[fp]) for fp in results] + f = V3BwFile(header, bwls) + # f = V3BwFile.from_results(None, str(tmpdir.join("v3bw.txt")), results) + + assert v3bw == str(f)