commit 7b767465f665fc9a9de56bac698db04021c90f10 Author: juga0 juga@riseup.net Date: Sun Feb 9 16:34:34 2020 +0000

fix: v3bwfile: !refactor, Rename constants

Rename constants to make their names more consistent with each other. All header constants start now by HEADER_ and all Bandwidth Lines by BWLINE_. The constants that are a list of keys are now named KEYS instead of KEY_VALUE or KEY_VALUE.

These are the substitutions made:

s/BW_KEYVALUE_SEP_V1/BWLINE_KEYVALUES_SEP_V1/ s/BW_KEYVALUES_BASIC/BWLINE_KEYS_V0/ s/BW_KEYVALUES_FILE/BWLINE_KEYS_V1_1/ s/BW_KEYVALUES_EXTRA_BWS/BWLINE_KEYS_V1_2/ s/BANDWIDTH_LINE_KEY_VALUES_MONITOR/BWLINE_KEYS_V1_4/ s/BW_KEYVALUES_EXTRA/BWLINE_KEYS_V1/ s/BW_KEYVALUES_INT/BWLINE_INT_KEYS/ s/BW_KEYVALUES/BWLINE_ALL_KEYS/ s/EXTRA_ARG_KEYVALUES/HEADER_KEYS_V1X/ s/STATS_KEYVALUES/HEADER_KEYS_V1_2/ s/BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED/HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS/ s/BW_HEADER_KEYVALUES_MONITOR/HEADER_KEYS_V1_4/ s/BANDWIDTH_HEADER_KEY_VALUES_INIT/HEADER_INIT_KEYS/ s/KEYVALUES_INT/HEADER_INT_KEYS/ s/UNORDERED_KEYVALUES/HEADER_UNORDERED_KEYS/ s/ALL_KEYVALUES/HEADER_ALL_KEYS/

Part of #30196 --- sbws/lib/v3bwfile.py | 76 ++++++++++++++++++++--------------------- tests/unit/lib/test_v3bwfile.py | 4 +-- 2 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index beed451..8537652 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -33,21 +33,21 @@ KEYVALUE_SEP_V2 = ' ' # Header KeyValues # ================= # List of the extra KeyValues accepted by the class -EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', +HEADER_KEYS_V1X = ['software', 'software_version', 'file_created', 'earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries'] # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent # with the other KeyValues. -STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', +HEADER_KEYS_V1_2 = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays']

# KeyValues that count the number of relays that are in the bandwidth file, # but ignored by Tor when voting, because they do not have a # measured bandwidth. -BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ +HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS = [ # Number of relays that were measured but all the measurements failed # because of network failures or it was # not found a suitable helper relay @@ -70,7 +70,7 @@ BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ # recent_measurement_attempt_count and recent_priority_relay_count # are not reset when the scanner is stop. # They will accumulate the values since the scanner was ever started. -BW_HEADER_KEYVALUES_MONITOR = [ +HEADER_KEYS_V1_4 = [ # 1.1 header: the number of different consensuses, that sbws has seen, # since the last 5 days 'recent_consensus_count', @@ -91,34 +91,34 @@ BW_HEADER_KEYVALUES_MONITOR = [ 'recent_measurement_failure_count', # The time it took to report about half of the network. 'time_to_report_half_network', -] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED -BANDWIDTH_HEADER_KEY_VALUES_INIT = \ +] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS +HEADER_INIT_KEYS = \ ['earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries']\ - + STATS_KEYVALUES \ - + BW_HEADER_KEYVALUES_MONITOR + + HEADER_KEYS_V1_2 \ + + HEADER_KEYS_V1_4

-KEYVALUES_INT = STATS_KEYVALUES + BW_HEADER_KEYVALUES_MONITOR +HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 # List of all unordered KeyValues currently being used to generate the file -UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + STATS_KEYVALUES + \ +HEADER_UNORDERED_KEYS = HEADER_KEYS_V1X + HEADER_KEYS_V1_2 + \ ['latest_bandwidth'] + \ - BW_HEADER_KEYVALUES_MONITOR + HEADER_KEYS_V1_4 # List of all the KeyValues currently being used to generate the file -ALL_KEYVALUES = ['version'] + UNORDERED_KEYVALUES +HEADER_ALL_KEYS = ['version'] + HEADER_UNORDERED_KEYS

TERMINATOR = '====='

# Bandwidth Lines KeyValues # ========================= # Num header lines in v1.X.X using all the KeyValues -NUM_LINES_HEADER_V1 = len(ALL_KEYVALUES) + 2 +NUM_LINES_HEADER_V1 = len(HEADER_ALL_KEYS) + 2 LINE_TERMINATOR = TERMINATOR + LINE_SEP

# KeyValue separator in Bandwidth Lines -BW_KEYVALUE_SEP_V1 = ' ' +BWLINE_KEYVALUES_SEP_V1 = ' ' # not inclding in the files the extra bws for now -BW_KEYVALUES_BASIC = ['node_id', 'bw'] -BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ +BWLINE_KEYS_V0 = ['node_id', 'bw'] +BWLINE_KEYS_V1_1 = BWLINE_KEYS_V0 + \ ['master_key_ed25519', 'nick', 'rtt', 'time', 'success', 'error_stream', 'error_circ', 'error_misc', # `vote=0` is used for the relays that were excluded to @@ -136,13 +136,13 @@ BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ 'under_min_report', # Added in #292951 'error_second_relay', 'error_destination'] -BW_KEYVALUES_EXTRA_BWS = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', +BWLINE_KEYS_V1_2 = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', 'desc_bw_obs_last', 'desc_bw_obs_mean', 'consensus_bandwidth', 'consensus_bandwidth_is_unmeasured']

# Added in #292951 -BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ +BWLINE_KEYS_V1_4 = [ # 1.2 relay: the number of different consensuses, that sbws has seen, # since the last 5 days, that have this relay 'relay_in_recent_consensus_count', @@ -179,16 +179,16 @@ BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ # (by default 2). 'relay_recent_measurements_excluded_few_count', ] -BW_KEYVALUES_EXTRA = BW_KEYVALUES_FILE + BW_KEYVALUES_EXTRA_BWS \ - + BANDWIDTH_LINE_KEY_VALUES_MONITOR +BWLINE_KEYS_V1 = BWLINE_KEYS_V1_1 + BWLINE_KEYS_V1_2 \ + + BWLINE_KEYS_V1_4 # NOTE: tech-debt: assign boolean type to vote and unmeasured, # when the attributes are defined with a type, as stem does. -BW_KEYVALUES_INT = ['bw', 'rtt', 'success', 'error_stream', +BWLINE_INT_KEYS = ['bw', 'rtt', 'success', 'error_stream', 'error_circ', 'error_misc', 'vote', 'unmeasured', 'under_min_report'] \ - + BW_KEYVALUES_EXTRA_BWS \ - + BANDWIDTH_LINE_KEY_VALUES_MONITOR -BW_KEYVALUES = BW_KEYVALUES_BASIC + BW_KEYVALUES_EXTRA + + BWLINE_KEYS_V1_2 \ + + BWLINE_KEYS_V1_4 +BWLINE_ALL_KEYS = BWLINE_KEYS_V0 + BWLINE_KEYS_V1

def round_sig_dig(n, digits=PROP276_ROUND_DIG): @@ -263,7 +263,7 @@ class V3BWHeader(object): # same as timestamp self.latest_bandwidth = unixts_to_isodt_str(timestamp) [setattr(self, k, v) for k, v in kwargs.items() - if k in BANDWIDTH_HEADER_KEY_VALUES_INIT] + if k in HEADER_INIT_KEYS]

def __str__(self): if self.version.startswith('1.'): @@ -339,7 +339,7 @@ class V3BWHeader(object): ts = lines[0] kwargs = dict([l.split(KEYVALUE_SEP_V1) for l in lines[:index_terminator] - if l.split(KEYVALUE_SEP_V1)[0] in ALL_KEYVALUES]) + if l.split(KEYVALUE_SEP_V1)[0] in HEADER_ALL_KEYS]) h = cls(ts, **kwargs) # last line is new line return h, lines[index_terminator + 1:-1] @@ -427,7 +427,7 @@ class V3BWHeader(object): """Return list of KeyValue tuples that do not have specific order.""" # sort the list to generate determinist headers keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in UNORDERED_KEYVALUES]) + if k in HEADER_UNORDERED_KEYS]) return keyvalue_tuple_ls

@property @@ -470,7 +470,7 @@ class V3BWHeader(object): def add_stats(self, **kwargs): # Using kwargs because attributes might chage. [setattr(self, k, str(v)) for k, v in kwargs.items() - if k in STATS_KEYVALUES] + if k in HEADER_KEYS_V1_2]

def add_time_report_half_network(self): """Add to the header the time it took to measure half of the network. @@ -553,7 +553,7 @@ class V3BWLine(object): self.node_id = node_id self.bw = bw [setattr(self, k, v) for k, v in kwargs.items() - if k in BW_KEYVALUES_EXTRA] + if k in BWLINE_KEYS_V1]

def __str__(self): return self.bw_strv1 @@ -604,8 +604,8 @@ class V3BWLine(object): # NOTE: The following 4 conditions exclude relays from the bandwidth # file when the measurements does not satisfy some rules, what makes # the relay non-`eligible`. - # In BANDWIDTH_LINE_KEY_VALUES_MONITOR it is explained what they mean. - # In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is also + # In BWLINE_KEYS_V1_4 it is explained what they mean. + # In HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS it is also # explained the what it means the strings returned. # They rules were introduced in #28061 and #27338 # In #28565 we introduce the KeyValues to know why they're excluded. @@ -708,10 +708,10 @@ class V3BWLine(object): def from_bw_line_v1(cls, line): assert isinstance(line, str) kwargs = dict([kv.split(KEYVALUE_SEP_V1) - for kv in line.split(BW_KEYVALUE_SEP_V1) - if kv.split(KEYVALUE_SEP_V1)[0] in BW_KEYVALUES]) + for kv in line.split(BWLINE_KEYVALUES_SEP_V1) + if kv.split(KEYVALUE_SEP_V1)[0] in BWLINE_ALL_KEYS]) for k, v in kwargs.items(): - if k in BW_KEYVALUES_INT: + if k in BWLINE_INT_KEYS: kwargs[k] = int(v) node_id = kwargs['node_id'] bw = kwargs['bw'] @@ -829,7 +829,7 @@ class V3BWLine(object): """Return list of KeyValue Bandwidth Line tuples.""" # sort the list to generate determinist headers keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in BW_KEYVALUES]) + if k in BWLINE_ALL_KEYS]) return keyvalue_tuple_ls

@property @@ -844,7 +844,7 @@ class V3BWLine(object): @property def bw_strv1(self): """Return Bandwidth Line string following spec v1.X.X.""" - bw_line_str = BW_KEYVALUE_SEP_V1.join( + bw_line_str = BWLINE_KEYVALUES_SEP_V1.join( self.bw_keyvalue_v1str_ls) + LINE_SEP if len(bw_line_str) > BW_LINE_SIZE: # if this is the case, probably there are too many KeyValues, @@ -908,11 +908,11 @@ class V3BWFile(object):

# Create a dictionary with the number of relays excluded by any of the # of the filtering rules that makes relays non-`eligible`. - # NOTE: In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is + # NOTE: In HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS it is # explained what are the KeyValues. # See also the comments in `from_results`. exclusion_dict = dict( - [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + [(k, 0) for k in HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS] ) for fp, values in results.items(): # log.debug("Relay fp %s", fp) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 720cd4b..40ffe16 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -14,7 +14,7 @@ from sbws.lib.v3bwfile import ( V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V1, num_results_of_type, V3BWFile, round_sig_dig, - BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS ) from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts

@@ -258,7 +258,7 @@ def test_from_results_read(datadir, tmpdir, conf, args): earliest_bandwidth=earliest_bandwidth, latest_bandwidth=latest_bandwidth) exclusion_dict = dict( - [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + [(k, 0) for k in HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS] ) expected_header.add_relays_excluded_counters(exclusion_dict) raw_bwls = [V3BWLine.from_results(results[fp])[0] for fp in results]