commit ecfdba922026cc56ca451d58e7933b42fab9ecc7 Author: Damian Johnson atagar@torproject.org Date: Sat Jan 19 16:11:36 2019 -0800
Parse bandwidth file headers
Reading all the header metadata fields. Honestly I'm a bit fuzzy on the definition of 'eligible', but that's fine. Taking my best guess at describing these.
00:08 <+atagar> juga: Minor thing that is crossing my mind while implementing stem's bandwidth-file parser is "how does the 'eligible' count differ from the number of relay measurements we list?". In other words, why does the header provide a 'consensus, eligible, and mininum eligible' count rather than a 'consensus, measured, and minimum measured' count. 'Eligible' is defined as 'having enough measurements' but not necessarily having a metric included, so I'm guessing we chose this term because there are relays that have metrics (making it 'eligible') but are excluded from the results, and need to count toward the 'minimum percent of the consensus'.
00:09 <+atagar> Anywho, unimportant to parsing. Just a spot in the spec that might benefit from a little clarification. --- stem/descriptor/bandwidth_metric.py | 94 ++++++++++++++++++++++++++------ test/unit/descriptor/bandwidth_metric.py | 40 ++++++++++++++ 2 files changed, 116 insertions(+), 18 deletions(-)
diff --git a/stem/descriptor/bandwidth_metric.py b/stem/descriptor/bandwidth_metric.py index bb732f4a..900a9d2e 100644 --- a/stem/descriptor/bandwidth_metric.py +++ b/stem/descriptor/bandwidth_metric.py @@ -16,9 +16,52 @@ Parsing for Bandwidth Authority metrics as described in Tor's
import datetime
-from stem.descriptor import ( - Descriptor, -) +import stem.util.str_tools + +from stem.descriptor import Descriptor + + +# Converters header attributes to a given type. Malformed fields should be +# ignored according to the spec. + +def _str(val): + return val # already a str + + +def _int(val): + return int(val) if (val and val.isdigit()) else None + + +def _date(val): + try: + return stem.util.str_tools._parse_iso_timestamp(val) + except ValueError: + return None # not an iso formatted date + + +# mapping of attributes => (header, type) + +HEADER_ATTR = { + 'version': ('version', _str), + + 'software': ('software', _str), + 'software_version': ('software_version', _str), + + 'earliest_bandwidth': ('earliest_bandwidth', _date), + 'latest_bandwidth': ('latest_bandwidth', _date), + 'created_at': ('file_created', _date), + 'generated_at': ('generator_started', _date), + + 'consensus_size': ('number_consensus_relays', _int), + 'eligible_count': ('number_eligible_relays', _int), + 'eligible_percent': ('percent_eligible_relays', _int), + 'min_count': ('minimum_number_eligible_relays', _int), + 'min_percent': ('minimum_percent_eligible_relays', _int), +} + +HEADER_DEFAULT = { + 'version': '1.0.0', # version field was added in 1.1.0 +}
def _parse_file(descriptor_file, validate = False, **kwargs): @@ -42,8 +85,14 @@ def _parse_file(descriptor_file, validate = False, **kwargs):
def _parse_header(descriptor, entries): header = {} + lines = str(descriptor).split('\n') + + # skip the first line, which should be the timestamp + + if lines and lines[0].isdigit(): + lines = lines[1:]
- for line in str(descriptor).split('\n'): + for line in lines: if line == '=====': break elif line.startswith('node_id='): @@ -51,15 +100,15 @@ def _parse_header(descriptor, entries):
if '=' in line: key, value = line.split('=', 1) - elif line.isdigit() and 'timestamp' not in header: - key, value = 'timestamp', line + header[key] = value else: raise ValueError("Header expected to be key=value pairs, but had '%s'" % line)
- header[key] = value - descriptor.header = header
+ for attr, (keyword, cls) in HEADER_ATTR.items(): + setattr(descriptor, attr, cls(header.get(keyword, HEADER_DEFAULT.get(attr)))) +
def _parse_timestamp(descriptor, entries): first_line = str(descriptor).split('\n', 1)[0] @@ -70,33 +119,42 @@ def _parse_timestamp(descriptor, entries): raise ValueError("First line should be a unix timestamp, but was '%s'" % first_line)
-def _header_attr(name): - def _parse(descriptor, entries): - val = descriptor.header.get(name, None) - setattr(descriptor, name, val) - - return _parse - - class BandwidthMetric(Descriptor): """ Tor bandwidth authroity measurements.
:var datetime timestamp: ***** time when these metrics were published + :var str version: ***** document format version + + :var str software: application that generated these metrics + :var str software_version: version of the application that generated these metrics
- :var dict header: ***** header metadata attributes + :var datetime earliest_bandwidth: time of the first sampling + :var datetime latest_bandwidth: time of the last sampling + :var datetime created_at: time when this file was created + :var datetime generated_at: time when collection of these metrics started + + :var int consensus_size: number of relays in the consensus + :var int eligible_count: relays with enough measurements to be included + :var int eligible_percent: percentage of consensus with enough measurements + :var int min_count: minimum eligible relays for results to be provided + :var int min_percent: minimum measured percentage of the consensus + + :var dict header: ***** header metadata
***** attribute is either required when we're parsed with validation or has a default value, others are left as **None** if undefined """
- TYPE_ANNOTATION_NAME = 'badnwidth-file' # TODO: needs an official @type + TYPE_ANNOTATION_NAME = 'badnwidth-file' # TODO: needs an official @type, https://trac.torproject.org/projects/tor/ticket/28615
ATTRIBUTES = { 'timestamp': (None, _parse_timestamp), 'header': ({}, _parse_header), }
+ ATTRIBUTES.update(dict([(k, (None, _parse_header)) for k in HEADER_ATTR.keys()])) + def __init__(self, raw_content, validate = False): super(BandwidthMetric, self).__init__(raw_content, lazy_load = not validate)
diff --git a/test/unit/descriptor/bandwidth_metric.py b/test/unit/descriptor/bandwidth_metric.py index 489f4d41..45739ed7 100644 --- a/test/unit/descriptor/bandwidth_metric.py +++ b/test/unit/descriptor/bandwidth_metric.py @@ -18,4 +18,44 @@ class TestBandwidthMetric(unittest.TestCase): """
desc = list(stem.descriptor.parse_file(test.unit.descriptor.get_resource('bwauth_v1.0'), 'badnwidth-file 1.0'))[0] + self.assertEqual(datetime.datetime(2019, 1, 14, 17, 41, 29), desc.timestamp) + self.assertEqual('1.0.0', desc.version) + + self.assertEqual(None, desc.software) + self.assertEqual(None, desc.software_version) + + self.assertEqual(None, desc.earliest_bandwidth) + self.assertEqual(None, desc.latest_bandwidth) + self.assertEqual(None, desc.created_at) + self.assertEqual(None, desc.generated_at) + + self.assertEqual(None, desc.consensus_size) + self.assertEqual(None, desc.eligible_count) + self.assertEqual(None, desc.eligible_percent) + self.assertEqual(None, desc.min_count) + self.assertEqual(None, desc.min_percent) + + def test_format_v1_2(self): + """ + Parse version 1.2 formatted metrics. + """ + + desc = list(stem.descriptor.parse_file(test.unit.descriptor.get_resource('bwauth_v1.2'), 'badnwidth-file 1.2'))[0] + + self.assertEqual(datetime.datetime(2019, 1, 14, 5, 34, 59), desc.timestamp) + self.assertEqual('1.2.0', desc.version) + + self.assertEqual('sbws', desc.software) + self.assertEqual('1.0.2', desc.software_version) + + self.assertEqual(datetime.datetime(2019, 1, 4, 5, 35, 29), desc.earliest_bandwidth) + self.assertEqual(datetime.datetime(2019, 1, 14, 5, 34, 59), desc.latest_bandwidth) + self.assertEqual(datetime.datetime(2019, 1, 14, 5, 35, 6), desc.created_at) + self.assertEqual(datetime.datetime(2019, 1, 3, 22, 45, 8), desc.generated_at) + + self.assertEqual(6514, desc.consensus_size) + self.assertEqual(6256, desc.eligible_count) + self.assertEqual(96, desc.eligible_percent) + self.assertEqual(3908, desc.min_count) + self.assertEqual(60, desc.min_percent)