commit f7575c6a1fb8b755a744c20318195c659a85d060 Author: Damian Johnson <atagar@torproject.org> Date: Sun Jan 20 17:09:22 2019 -0800 Parse BandwidthFile as streams String parsing was a great spot to start, but highly memory inefficient. Parsing internally created multiple copies of our bandwidth file content as processed rather than working from a single copy of the bytes. --- stem/descriptor/bandwidth_file.py | 50 ++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/stem/descriptor/bandwidth_file.py b/stem/descriptor/bandwidth_file.py index 9c87385c..2e9fc216 100644 --- a/stem/descriptor/bandwidth_file.py +++ b/stem/descriptor/bandwidth_file.py @@ -15,6 +15,7 @@ Parsing for Bandwidth Authority metrics as described in Tor's """ import datetime +import io import time import stem.util.str_tools @@ -91,16 +92,17 @@ def _parse_file(descriptor_file, validate = False, **kwargs): def _parse_header(descriptor, entries): header = {} - lines = str(descriptor).split('\n') + content = io.BytesIO(descriptor.get_bytes()) - # skip the first line, which should be the timestamp + content.readline() # skip the first line, which should be the timestamp - if lines and lines[0].isdigit(): - lines = lines[1:] + while True: + line = content.readline().strip() - for line in lines: - if line == HEADER_DIV: - break + if not line: + break # end of the content + elif line == HEADER_DIV: + break # end of header elif line.startswith('node_id='): break # version 1.0 measurement @@ -117,7 +119,7 @@ def _parse_header(descriptor, entries): def _parse_timestamp(descriptor, entries): - first_line = str(descriptor).split('\n', 1)[0] + first_line = io.BytesIO(descriptor.get_bytes()).readline().strip() if first_line.isdigit(): descriptor.timestamp = datetime.datetime.utcfromtimestamp(int(first_line)) @@ -129,30 +131,30 @@ def _parse_body(descriptor, entries): # In version 1.0.0 the body is everything after the first line. Otherwise # it's everything after the header's divider. - div = '\n' if descriptor.version == '1.0.0' else HEADER_DIV + content = io.BytesIO(descriptor.get_bytes()) - if div in str(descriptor): - body = str(descriptor).split(div, 1)[1].strip() + if descriptor.version == '1.0.0': + content.readline() # skip the first line else: - body = '' + while content.readline().strip() != HEADER_DIV: + pass # skip the header measurements = {} - if body: - for line in body.split('\n'): - attr = dict(_mappings_for('measurement', line)) + for line in content.readlines(): + attr = dict(_mappings_for('measurement', line.strip())) - if 'node_id' not in attr: - raise ValueError("Every meaurement must include 'node_id': %s" % line) - elif attr['node_id'] in measurements: - # Relay is listed multiple times. This is a bug for the bandwidth - # authority that made this descriptor, but according to the spec - # should be ignored by parsers. + if 'node_id' not in attr: + raise ValueError("Every meaurement must include 'node_id': %s" % line.strip()) + elif attr['node_id'] in measurements: + # Relay is listed multiple times. This is a bug for the bandwidth + # authority that made this descriptor, but according to the spec + # should be ignored by parsers. - continue + continue - fingerprint = attr['node_id'].lstrip('$') # bwauths prefix fingerprints with '$' - measurements[fingerprint] = attr + fingerprint = attr['node_id'].lstrip('$') # bwauths prefix fingerprints with '$' + measurements[fingerprint] = attr descriptor.measurements = measurements