[tor-commits] [sbws/master] Add class to manage bw files
pastly at torproject.org
pastly at torproject.org
Thu Jun 14 13:29:50 UTC 2018
commit 5372eb7d09ca1cdcbd8418a38e5dda236cc7e93e
Author: juga0 <juga at riseup.net>
Date: Sat May 26 17:39:46 2018 +0000
Add class to manage bw files
* Move logic in generate to class
* Add test
* Move 7500 to global constant
* Make read_started_at not fail when conf is None
---
sbws/core/generate.py | 17 ++++--------
sbws/globals.py | 2 ++
sbws/lib/v3bwfile.py | 58 ++++++++++++++++++++++++++++++++++++++++-
tests/lib/data/v3bw.txt | 10 +++++++
tests/unit/lib/test_v3bwfile.py | 16 +++++++++---
5 files changed, 87 insertions(+), 16 deletions(-)
diff --git a/sbws/core/generate.py b/sbws/core/generate.py
index e7dd037..4921f57 100644
--- a/sbws/core/generate.py
+++ b/sbws/core/generate.py
@@ -1,6 +1,5 @@
-from sbws.globals import (fail_hard, is_initted)
-from sbws.lib.v3bwfile import V3BwHeader, V3BWLine
-from sbws.lib.resultdump import ResultSuccess
+from sbws.globals import (fail_hard, is_initted, SCALE_CONSTANT)
+from sbws.lib.v3bwfile import V3BwFile
from sbws.lib.resultdump import load_recent_results_in_datadir
from argparse import ArgumentDefaultsHelpFormatter
import os
@@ -22,7 +21,7 @@ def gen_parser(sub):
# time, torflow happened to generate output that averaged to 7500 bw units
# per relay. We wanted the ability to try to be like torflow. See
# https://lists.torproject.org/pipermail/tor-dev/2018-March/013049.html
- p.add_argument('--scale-constant', default=7500, type=int,
+ p.add_argument('--scale-constant', default=SCALE_CONSTANT, type=int,
help='When scaling bw weights, scale them using this const '
'multiplied by the number of measured relays')
p.add_argument('--scale', action='store_true',
@@ -32,13 +31,6 @@ def gen_parser(sub):
'out, and we do so proportionally')
-def log_stats(data_lines):
- assert len(data_lines) > 0
- total_bw = sum([l.bw for l in data_lines])
- bw_per_line = total_bw / len(data_lines)
- log.info('Mean bandwidth per line: %f "KiB"', bw_per_line)
-
-
def main(args, conf):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init')
@@ -56,4 +48,5 @@ def main(args, conf):
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
-
+ bw_file = V3BwFile.from_arg_results(args, conf, results)
+ log.info('Mean bandwidth per line: %f "KiB"', bw_file.avg_bw)
diff --git a/sbws/globals.py b/sbws/globals.py
index 0026ba2..602075a 100644
--- a/sbws/globals.py
+++ b/sbws/globals.py
@@ -24,6 +24,8 @@ TORRC_STARTING_POINT = {
'UseEntryGuards': '0',
}
+SCALE_CONSTANT = 7500
+
def is_initted(d):
if not os.path.isdir(d):
diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py
index d7a5d2b..4b6e6bc 100644
--- a/sbws/lib/v3bwfile.py
+++ b/sbws/lib/v3bwfile.py
@@ -70,7 +70,10 @@ def read_started_ts(conf):
:param ConfigParser conf: configuration
:returns: str, ISO formated timestamp
"""
- filepath = conf['paths']['started_filepath']
+ try:
+ filepath = conf['paths']['started_filepath']
+ except TypeError as e:
+ return ''
try:
with FileLock(filepath):
with open(filepath, 'r') as fd:
@@ -336,3 +339,56 @@ class V3BWLine(object):
assert fingerprint in data
return cls.from_results(data[fingerprint])
+
+class V3BwFile(object):
+ def __init__(self, v3bwheader, v3bwlines):
+ """
+ :param V3BWHeader v3bwheader:
+ :param list v3bwlines:
+ """
+ self.header = v3bwheader
+ self.bw_lines = v3bwlines
+
+ def __str__(self):
+ return str(self.header) + ''.join([str(bw_line)
+ for bw_line in self.bw_lines])
+
+ @property
+ def total_bw(self):
+ return total_bw(self.bw_lines)
+
+ @property
+ def num_lines(self):
+ return len(self.bw_lines)
+
+ @property
+ def avg_bw(self):
+ return self.total_bw / self.num_lines
+
+ @classmethod
+ def from_results(cls, conf, output, results):
+ bw_lines = [V3BWLine.from_results(results[fp]) for fp in results]
+ bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True)
+ header = V3BwHeader.from_results(conf, results)
+ f = cls(header, bw_lines)
+ f.write(output)
+ return f
+
+ @classmethod
+ def from_arg_results(cls, args, conf, results):
+ bw_lines = [V3BWLine.from_results(results[fp]) for fp in results]
+ bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True)
+ if args.scale:
+ bw_lines = scale_lines(bw_lines, args.scale_constant)
+ header = V3BwHeader.from_results(conf, results)
+ f = cls(header, bw_lines)
+ output = args.output or conf['paths']['v3bw_fname']
+ f.write(output)
+ return f
+
+ def write(self, output):
+ log.info('Writing v3bw file to %s', output)
+ with open(output, 'wt') as fd:
+ fd.write(str(self.header))
+ for line in self.bw_lines:
+ fd.write(str(line))
diff --git a/tests/lib/data/v3bw.txt b/tests/lib/data/v3bw.txt
new file mode 100644
index 0000000..09f4a6f
--- /dev/null
+++ b/tests/lib/data/v3bw.txt
@@ -0,0 +1,10 @@
+1523974147
+version=1.1.0
+earliest_bandwidth=2018-04-16T14:09:07
+file_created=2018-04-25T13:10:57
+generator_started=2018-04-16T14:09:05
+latest_bandwidth=2018-04-17T14:09:07
+software=sbws
+software_version=0.3.1-dev
+====
+bw=54 error_auth=0 error_circ=0 error_misc=0 error_stream=1 last_time=2018-04-17T14:09:07 nick=A node_id=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA success=1
diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py
index 8af0869..3127a92 100644
--- a/tests/unit/lib/test_v3bwfile.py
+++ b/tests/unit/lib/test_v3bwfile.py
@@ -1,13 +1,13 @@
# -*- coding: utf-8 -*-
"""Test generation of bandwidth measurements document (v3bw)"""
import json
-import os.path
from sbws import __version__ as version
from sbws.globals import SPEC_VERSION
from sbws.lib.resultdump import Result, load_result_file
from sbws.lib.v3bwfile import (V3BwHeader, V3BWLine, TERMINATOR, LINE_SEP,
- KEYVALUE_SEP_V110, num_results_of_type)
+ KEYVALUE_SEP_V110, num_results_of_type,
+ V3BwFile)
timestamp = 1523974147
timestamp_l = str(timestamp)
@@ -152,4 +152,14 @@ def test_v3bwline_from_results_file(datadir):
def test_v3bwfile(datadir, tmpdir):
"""Test generate v3bw file (including relay_lines)."""
- pass
+ v3bw = datadir.read('v3bw.txt')
+ results = load_result_file(str(datadir.join("results.txt")))
+ header = V3BwHeader(timestamp_l,
+ file_created=file_created,
+ generator_started=generator_started,
+ earliest_bandwidth=earliest_bandwidth)
+ bwls = [V3BWLine.from_results(results[fp]) for fp in results]
+ f = V3BwFile(header, bwls)
+ # f = V3BwFile.from_results(None, str(tmpdir.join("v3bw.txt")), results)
+
+ assert v3bw == str(f)
More information about the tor-commits
mailing list