March 2020 - tor-commits - lists.torproject.org

[sbws/maint-1.1] fix: v3bwfile: !refactor, Rename constants
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 7b767465f665fc9a9de56bac698db04021c90f10 Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 16:34:34 2020 +0000 fix: v3bwfile: !refactor, Rename constants Rename constants to make their names more consistent with each other. All header constants start now by HEADER_ and all Bandwidth Lines by BWLINE_. The constants that are a list of keys are now named KEYS instead of KEY_VALUE or KEY_VALUE. These are the substitutions made: s/BW_KEYVALUE_SEP_V1/BWLINE_KEYVALUES_SEP_V1/ s/BW_KEYVALUES_BASIC/BWLINE_KEYS_V0/ s/BW_KEYVALUES_FILE/BWLINE_KEYS_V1_1/ s/BW_KEYVALUES_EXTRA_BWS/BWLINE_KEYS_V1_2/ s/BANDWIDTH_LINE_KEY_VALUES_MONITOR/BWLINE_KEYS_V1_4/ s/BW_KEYVALUES_EXTRA/BWLINE_KEYS_V1/ s/BW_KEYVALUES_INT/BWLINE_INT_KEYS/ s/BW_KEYVALUES/BWLINE_ALL_KEYS/ s/EXTRA_ARG_KEYVALUES/HEADER_KEYS_V1X/ s/STATS_KEYVALUES/HEADER_KEYS_V1_2/ s/BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED/HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS/ s/BW_HEADER_KEYVALUES_MONITOR/HEADER_KEYS_V1_4/ s/BANDWIDTH_HEADER_KEY_VALUES_INIT/HEADER_INIT_KEYS/ s/KEYVALUES_INT/HEADER_INT_KEYS/ s/UNORDERED_KEYVALUES/HEADER_UNORDERED_KEYS/ s/ALL_KEYVALUES/HEADER_ALL_KEYS/ Part of #30196 --- sbws/lib/v3bwfile.py | 76 ++++++++++++++++++++--------------------- tests/unit/lib/test_v3bwfile.py | 4 +-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index beed451..8537652 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -33,21 +33,21 @@ KEYVALUE_SEP_V2 = ' ' # Header KeyValues # ================= # List of the extra KeyValues accepted by the class -EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', +HEADER_KEYS_V1X = ['software', 'software_version', 'file_created', 'earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries'] # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent # with the other KeyValues. -STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', +HEADER_KEYS_V1_2 = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] # KeyValues that count the number of relays that are in the bandwidth file, # but ignored by Tor when voting, because they do not have a # measured bandwidth. -BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ +HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS = [ # Number of relays that were measured but all the measurements failed # because of network failures or it was # not found a suitable helper relay @@ -70,7 +70,7 @@ BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ # recent_measurement_attempt_count and recent_priority_relay_count # are not reset when the scanner is stop. # They will accumulate the values since the scanner was ever started. -BW_HEADER_KEYVALUES_MONITOR = [ +HEADER_KEYS_V1_4 = [ # 1.1 header: the number of different consensuses, that sbws has seen, # since the last 5 days 'recent_consensus_count', @@ -91,34 +91,34 @@ BW_HEADER_KEYVALUES_MONITOR = [ 'recent_measurement_failure_count', # The time it took to report about half of the network. 'time_to_report_half_network', -] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED -BANDWIDTH_HEADER_KEY_VALUES_INIT = \ +] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS +HEADER_INIT_KEYS = \ ['earliest_bandwidth', 'generator_started', 'scanner_country', 'destinations_countries']\ - + STATS_KEYVALUES \ - + BW_HEADER_KEYVALUES_MONITOR + + HEADER_KEYS_V1_2 \ + + HEADER_KEYS_V1_4 -KEYVALUES_INT = STATS_KEYVALUES + BW_HEADER_KEYVALUES_MONITOR +HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 # List of all unordered KeyValues currently being used to generate the file -UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + STATS_KEYVALUES + \ +HEADER_UNORDERED_KEYS = HEADER_KEYS_V1X + HEADER_KEYS_V1_2 + \ ['latest_bandwidth'] + \ - BW_HEADER_KEYVALUES_MONITOR + HEADER_KEYS_V1_4 # List of all the KeyValues currently being used to generate the file -ALL_KEYVALUES = ['version'] + UNORDERED_KEYVALUES +HEADER_ALL_KEYS = ['version'] + HEADER_UNORDERED_KEYS TERMINATOR = '=====' # Bandwidth Lines KeyValues # ========================= # Num header lines in v1.X.X using all the KeyValues -NUM_LINES_HEADER_V1 = len(ALL_KEYVALUES) + 2 +NUM_LINES_HEADER_V1 = len(HEADER_ALL_KEYS) + 2 LINE_TERMINATOR = TERMINATOR + LINE_SEP # KeyValue separator in Bandwidth Lines -BW_KEYVALUE_SEP_V1 = ' ' +BWLINE_KEYVALUES_SEP_V1 = ' ' # not inclding in the files the extra bws for now -BW_KEYVALUES_BASIC = ['node_id', 'bw'] -BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ +BWLINE_KEYS_V0 = ['node_id', 'bw'] +BWLINE_KEYS_V1_1 = BWLINE_KEYS_V0 + \ ['master_key_ed25519', 'nick', 'rtt', 'time', 'success', 'error_stream', 'error_circ', 'error_misc', # `vote=0` is used for the relays that were excluded to @@ -136,13 +136,13 @@ BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ 'under_min_report', # Added in #292951 'error_second_relay', 'error_destination'] -BW_KEYVALUES_EXTRA_BWS = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', +BWLINE_KEYS_V1_2 = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', 'desc_bw_obs_last', 'desc_bw_obs_mean', 'consensus_bandwidth', 'consensus_bandwidth_is_unmeasured'] # Added in #292951 -BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ +BWLINE_KEYS_V1_4 = [ # 1.2 relay: the number of different consensuses, that sbws has seen, # since the last 5 days, that have this relay 'relay_in_recent_consensus_count', @@ -179,16 +179,16 @@ BANDWIDTH_LINE_KEY_VALUES_MONITOR = [ # (by default 2). 'relay_recent_measurements_excluded_few_count', ] -BW_KEYVALUES_EXTRA = BW_KEYVALUES_FILE + BW_KEYVALUES_EXTRA_BWS \ - + BANDWIDTH_LINE_KEY_VALUES_MONITOR +BWLINE_KEYS_V1 = BWLINE_KEYS_V1_1 + BWLINE_KEYS_V1_2 \ + + BWLINE_KEYS_V1_4 # NOTE: tech-debt: assign boolean type to vote and unmeasured, # when the attributes are defined with a type, as stem does. -BW_KEYVALUES_INT = ['bw', 'rtt', 'success', 'error_stream', +BWLINE_INT_KEYS = ['bw', 'rtt', 'success', 'error_stream', 'error_circ', 'error_misc', 'vote', 'unmeasured', 'under_min_report'] \ - + BW_KEYVALUES_EXTRA_BWS \ - + BANDWIDTH_LINE_KEY_VALUES_MONITOR -BW_KEYVALUES = BW_KEYVALUES_BASIC + BW_KEYVALUES_EXTRA + + BWLINE_KEYS_V1_2 \ + + BWLINE_KEYS_V1_4 +BWLINE_ALL_KEYS = BWLINE_KEYS_V0 + BWLINE_KEYS_V1 def round_sig_dig(n, digits=PROP276_ROUND_DIG): @@ -263,7 +263,7 @@ class V3BWHeader(object): # same as timestamp self.latest_bandwidth = unixts_to_isodt_str(timestamp) [setattr(self, k, v) for k, v in kwargs.items() - if k in BANDWIDTH_HEADER_KEY_VALUES_INIT] + if k in HEADER_INIT_KEYS] def __str__(self): if self.version.startswith('1.'): @@ -339,7 +339,7 @@ class V3BWHeader(object): ts = lines[0] kwargs = dict([l.split(KEYVALUE_SEP_V1) for l in lines[:index_terminator] - if l.split(KEYVALUE_SEP_V1)[0] in ALL_KEYVALUES]) + if l.split(KEYVALUE_SEP_V1)[0] in HEADER_ALL_KEYS]) h = cls(ts, **kwargs) # last line is new line return h, lines[index_terminator + 1:-1] @@ -427,7 +427,7 @@ class V3BWHeader(object): """Return list of KeyValue tuples that do not have specific order.""" # sort the list to generate determinist headers keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in UNORDERED_KEYVALUES]) + if k in HEADER_UNORDERED_KEYS]) return keyvalue_tuple_ls @property @@ -470,7 +470,7 @@ class V3BWHeader(object): def add_stats(self, **kwargs): # Using kwargs because attributes might chage. [setattr(self, k, str(v)) for k, v in kwargs.items() - if k in STATS_KEYVALUES] + if k in HEADER_KEYS_V1_2] def add_time_report_half_network(self): """Add to the header the time it took to measure half of the network. @@ -553,7 +553,7 @@ class V3BWLine(object): self.node_id = node_id self.bw = bw [setattr(self, k, v) for k, v in kwargs.items() - if k in BW_KEYVALUES_EXTRA] + if k in BWLINE_KEYS_V1] def __str__(self): return self.bw_strv1 @@ -604,8 +604,8 @@ class V3BWLine(object): # NOTE: The following 4 conditions exclude relays from the bandwidth # file when the measurements does not satisfy some rules, what makes # the relay non-`eligible`. - # In BANDWIDTH_LINE_KEY_VALUES_MONITOR it is explained what they mean. - # In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is also + # In BWLINE_KEYS_V1_4 it is explained what they mean. + # In HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS it is also # explained the what it means the strings returned. # They rules were introduced in #28061 and #27338 # In #28565 we introduce the KeyValues to know why they're excluded. @@ -708,10 +708,10 @@ class V3BWLine(object): def from_bw_line_v1(cls, line): assert isinstance(line, str) kwargs = dict([kv.split(KEYVALUE_SEP_V1) - for kv in line.split(BW_KEYVALUE_SEP_V1) - if kv.split(KEYVALUE_SEP_V1)[0] in BW_KEYVALUES]) + for kv in line.split(BWLINE_KEYVALUES_SEP_V1) + if kv.split(KEYVALUE_SEP_V1)[0] in BWLINE_ALL_KEYS]) for k, v in kwargs.items(): - if k in BW_KEYVALUES_INT: + if k in BWLINE_INT_KEYS: kwargs[k] = int(v) node_id = kwargs['node_id'] bw = kwargs['bw'] @@ -829,7 +829,7 @@ class V3BWLine(object): """Return list of KeyValue Bandwidth Line tuples.""" # sort the list to generate determinist headers keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in BW_KEYVALUES]) + if k in BWLINE_ALL_KEYS]) return keyvalue_tuple_ls @property @@ -844,7 +844,7 @@ class V3BWLine(object): @property def bw_strv1(self): """Return Bandwidth Line string following spec v1.X.X.""" - bw_line_str = BW_KEYVALUE_SEP_V1.join( + bw_line_str = BWLINE_KEYVALUES_SEP_V1.join( self.bw_keyvalue_v1str_ls) + LINE_SEP if len(bw_line_str) > BW_LINE_SIZE: # if this is the case, probably there are too many KeyValues, @@ -908,11 +908,11 @@ class V3BWFile(object): # Create a dictionary with the number of relays excluded by any of the # of the filtering rules that makes relays non-`eligible`. - # NOTE: In BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED it is + # NOTE: In HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS it is # explained what are the KeyValues. # See also the comments in `from_results`. exclusion_dict = dict( - [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + [(k, 0) for k in HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS] ) for fp, values in results.items(): # log.debug("Relay fp %s", fp) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 720cd4b..40ffe16 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -14,7 +14,7 @@ from sbws.lib.v3bwfile import ( V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V1, num_results_of_type, V3BWFile, round_sig_dig, - BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS ) from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts @@ -258,7 +258,7 @@ def test_from_results_read(datadir, tmpdir, conf, args): earliest_bandwidth=earliest_bandwidth, latest_bandwidth=latest_bandwidth) exclusion_dict = dict( - [(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED] + [(k, 0) for k in HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS] ) expected_header.add_relays_excluded_counters(exclusion_dict) raw_bwls = [V3BWLine.from_results(results[fp])[0] for fp in results]

1 0

[sbws/maint-1.1] fix: v3bwfile: Add constant for ordered key/values
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 017c7c100d39007176edb080128a5bf96e61080d Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 16:46:28 2020 +0000 fix: v3bwfile: Add constant for ordered key/values to build the list of all keys from it and ensure no key is missing. --- sbws/lib/v3bwfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index dc85a37..a232093 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -32,6 +32,8 @@ KEYVALUE_SEP_V2 = ' ' # Header KeyValues # ================= +# KeyValues that need to be in a specific order in the Bandwidth File. +HEADER_KEYS_V1_1_ORDERED = ['version'] # List of the extra KeyValues accepted by the class HEADER_KEYS_V1X = [ "software", @@ -114,7 +116,7 @@ HEADER_UNORDERED_KEYS = HEADER_KEYS_V1X + HEADER_KEYS_V1_2 + \ ['latest_bandwidth'] + \ HEADER_KEYS_V1_4 # List of all the KeyValues currently being used to generate the file -HEADER_ALL_KEYS = ['version'] + HEADER_UNORDERED_KEYS +HEADER_ALL_KEYS = HEADER_KEYS_V1_1_ORDERED + HEADER_UNORDERED_KEYS TERMINATOR = '====='

1 0

[sbws/maint-1.1] fix: v3bwfile: Reformat to don't get flake8 errors
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit be0c5ea45c6d24cb7ffe5902576680600800c48e Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 16:43:38 2020 +0000 fix: v3bwfile: Reformat to don't get flake8 errors After the automatic constants renaming, fix the flake8 errors by reformatting automatically with `black`, only the lines that had errors. Part of #30196 --- sbws/lib/v3bwfile.py | 56 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 8537652..dc85a37 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -33,16 +33,26 @@ KEYVALUE_SEP_V2 = ' ' # Header KeyValues # ================= # List of the extra KeyValues accepted by the class -HEADER_KEYS_V1X = ['software', 'software_version', 'file_created', - 'earliest_bandwidth', 'generator_started', - 'scanner_country', 'destinations_countries'] +HEADER_KEYS_V1X = [ + "software", + "software_version", + "file_created", + "earliest_bandwidth", + "generator_started", + "scanner_country", + "destinations_countries", +] # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent # with the other KeyValues. -HEADER_KEYS_V1_2 = ['number_eligible_relays', 'minimum_number_eligible_relays', - 'number_consensus_relays', 'percent_eligible_relays', - 'minimum_percent_eligible_relays'] +HEADER_KEYS_V1_2 = [ + "number_eligible_relays", + "minimum_number_eligible_relays", + "number_consensus_relays", + "percent_eligible_relays", + "minimum_percent_eligible_relays", +] # KeyValues that count the number of relays that are in the bandwidth file, # but ignored by Tor when voting, because they do not have a @@ -136,10 +146,16 @@ BWLINE_KEYS_V1_1 = BWLINE_KEYS_V0 + \ 'under_min_report', # Added in #292951 'error_second_relay', 'error_destination'] -BWLINE_KEYS_V1_2 = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', - 'desc_bw_obs_last', 'desc_bw_obs_mean', - 'consensus_bandwidth', - 'consensus_bandwidth_is_unmeasured'] +BWLINE_KEYS_V1_2 = [ + "bw_median", + "bw_mean", + "desc_bw_avg", + "desc_bw_bur", + "desc_bw_obs_last", + "desc_bw_obs_mean", + "consensus_bandwidth", + "consensus_bandwidth_is_unmeasured", +] # Added in #292951 BWLINE_KEYS_V1_4 = [ @@ -183,11 +199,21 @@ BWLINE_KEYS_V1 = BWLINE_KEYS_V1_1 + BWLINE_KEYS_V1_2 \ + BWLINE_KEYS_V1_4 # NOTE: tech-debt: assign boolean type to vote and unmeasured, # when the attributes are defined with a type, as stem does. -BWLINE_INT_KEYS = ['bw', 'rtt', 'success', 'error_stream', - 'error_circ', 'error_misc', 'vote', 'unmeasured', - 'under_min_report'] \ - + BWLINE_KEYS_V1_2 \ - + BWLINE_KEYS_V1_4 +BWLINE_INT_KEYS = ( + [ + "bw", + "rtt", + "success", + "error_stream", + "error_circ", + "error_misc", + "vote", + "unmeasured", + "under_min_report", + ] + + BWLINE_KEYS_V1_2 + + BWLINE_KEYS_V1_4 +) BWLINE_ALL_KEYS = BWLINE_KEYS_V0 + BWLINE_KEYS_V1

1 0

[sbws/maint-1.1] fix: v3bwfile: !refactor, replace constants
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 27e2112adfc114b6931ad87d54cf69b94c9a8e8e Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 16:51:55 2020 +0000 fix: v3bwfile: !refactor, replace constants - Add new KeyValues constants - Remove HEADER_KEYS_V1X constant, only used once - Replace HEADER_UNORDERED_KEYS with the new constants Part of #30196. --- sbws/lib/v3bwfile.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index a232093..b3a4775 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -34,16 +34,32 @@ KEYVALUE_SEP_V2 = ' ' # ================= # KeyValues that need to be in a specific order in the Bandwidth File. HEADER_KEYS_V1_1_ORDERED = ['version'] -# List of the extra KeyValues accepted by the class -HEADER_KEYS_V1X = [ +# KeyValues that are not initialized from the state file nor the measurements. +# They can also be pass as an argument to `Header` to overwrite default values, +# what is done in unit tests. +# `latest bandwidth` is special cause it gets its value from timestamp, which +# is not a KeyValue, but it's always pass as an agument. +# It could be separaed in other list, but so far there is no need, cause: +# 1. when it's pass to the Header to initialize it, it's just ignored. +# 2. when the file is created, it's took into account. +HEADER_KEYS_V1_1_SELF_INITIALIZED = [ "software", "software_version", "file_created", + "latest_bandwidth", +] +# KeyValues that are initialized from arguments. +HEADER_KEYS_V1_1_TO_INIT = [ "earliest_bandwidth", "generator_started", +] + +# KeyValues added in the Bandwidth File v1.3.0 +HEADER_KEYS_V1_3 = [ "scanner_country", "destinations_countries", ] + # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent @@ -112,9 +128,13 @@ HEADER_INIT_KEYS = \ HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 # List of all unordered KeyValues currently being used to generate the file -HEADER_UNORDERED_KEYS = HEADER_KEYS_V1X + HEADER_KEYS_V1_2 + \ - ['latest_bandwidth'] + \ - HEADER_KEYS_V1_4 +HEADER_UNORDERED_KEYS = ( + HEADER_KEYS_V1_1_SELF_INITIALIZED + + HEADER_KEYS_V1_1_TO_INIT + + HEADER_KEYS_V1_3 + + HEADER_KEYS_V1_2 + + HEADER_KEYS_V1_4 +) # List of all the KeyValues currently being used to generate the file HEADER_ALL_KEYS = HEADER_KEYS_V1_1_ORDERED + HEADER_UNORDERED_KEYS

1 0

[sbws/maint-1.1] fix: v3bwfile: !refactor, Replace constant
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit ed24b53d3f604c3da784743d6592e28716854ca3 Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 16:53:57 2020 +0000 fix: v3bwfile: !refactor, Replace constant Replace HEADER_INIT_KEYS constant KeyValues with other constants created in the previous commit. Part of #30196. --- sbws/lib/v3bwfile.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b3a4775..49015c0 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -120,11 +120,14 @@ HEADER_KEYS_V1_4 = [ # The time it took to report about half of the network. 'time_to_report_half_network', ] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS -HEADER_INIT_KEYS = \ - ['earliest_bandwidth', 'generator_started', - 'scanner_country', 'destinations_countries']\ - + HEADER_KEYS_V1_2 \ + +# KeyValues that are initialized from arguments, not self-initialized. +HEADER_INIT_KEYS = ( + HEADER_KEYS_V1_1_TO_INIT + + HEADER_KEYS_V1_3 + + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 +) HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 # List of all unordered KeyValues currently being used to generate the file

1 0

[sbws/maint-1.1] fix: v3bwfile: Add comment about bwlines v1.3
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit fa59e991609a61fd1a0770d8fb5c73f744a36ece Author: juga0 <juga(a)riseup.net> Date: Mon Feb 24 14:10:12 2020 +0000 fix: v3bwfile: Add comment about bwlines v1.3 Part of #30196. --- sbws/lib/v3bwfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 0826127..7805fe2 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -192,6 +192,8 @@ BWLINE_KEYS_V1_2 = [ "consensus_bandwidth_is_unmeasured", ] +# There were no bandwidth lines key added in the specification version 1.3 + # Added in #292951 BWLINE_KEYS_V1_4 = [ # 1.2 relay: the number of different consensuses, that sbws has seen,

1 0

[sbws/maint-1.1] fix: v3bwfile: !refactor, Reorder header constants
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit e9182a41a95b9aef8af0700b9c4566d9eba38122 Author: juga0 <juga(a)riseup.net> Date: Mon Feb 24 14:06:52 2020 +0000 fix: v3bwfile: !refactor, Reorder header constants by version. Part of #30196. --- sbws/lib/v3bwfile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 2f34376..0826127 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -54,12 +54,6 @@ HEADER_KEYS_V1_1_TO_INIT = [ "generator_started", ] -# KeyValues added in the Bandwidth File v1.3.0 -HEADER_KEYS_V1_3 = [ - "scanner_country", - "destinations_countries", -] - # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent @@ -72,6 +66,12 @@ HEADER_KEYS_V1_2 = [ "minimum_percent_eligible_relays", ] +# KeyValues added in the Bandwidth File v1.3.0 +HEADER_KEYS_V1_3 = [ + "scanner_country", + "destinations_countries", +] + # KeyValues that count the number of relays that are in the bandwidth file, # but ignored by Tor when voting, because they do not have a # measured bandwidth.

1 0

[sbws/maint-1.1] fix: v3bwfile: Add tor_version KeyValue
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 6843145a2887fb2496a2b5cef30ffabae2402cda Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 17:03:13 2020 +0000 fix: v3bwfile: Add tor_version KeyValue - Create new KeyValues constants for the new v1.5.0 KeyValues - Instanciate State in Header.from_results so that there is no need to create new methods for all the header KeyValues that are read from the state file - Add tor_version to the kwargs to initialize the Header - Write tor_version in the state file when the scanner is started Closes: #30196. --- sbws/core/scanner.py | 7 ++++++- sbws/lib/v3bwfile.py | 19 +++++++++++++++++++ tests/unit/lib/test_v3bwfile.py | 15 +++++++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index d3ed5b1..3178594 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -679,9 +679,14 @@ def run_speedtest(args, conf): # When there will be a refactor where conf is global, this can be removed # from here. state = State(conf.getpath('paths', 'state_fname')) + # XXX: tech-debt: create new function to obtain the controller and to + # write the state, so that a unit test to check the state tor version can + # be created + # Store tor version whenever the scanner starts. + state['tor_version'] = str(controller.get_version()) # Call only once to initialize http_headers settings.init_http_headers(conf.get('scanner', 'nickname'), state['uuid'], - str(controller.get_version())) + state['tor_version']) # To do not have to pass args and conf to RelayList, pass an extra # argument with the data_period measurements_period = conf.getint('general', 'data_period') diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 49015c0..2f34376 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -121,12 +121,21 @@ HEADER_KEYS_V1_4 = [ 'time_to_report_half_network', ] + HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS +# KeyValues added in the Bandwidth File v1.5.0 +# XXX: Change SPEC_VERSION when all the v1.5.0 keys are added, before a new +# sbws release. +# Tor version will be obtained from the state file, so it won't be pass as an +# argument, but will be self-initialized. +HEADER_KEYS_V1_5_TO_INIT = ['tor_version'] +HEADER_KEYS_V1_5 = HEADER_KEYS_V1_5_TO_INIT + # KeyValues that are initialized from arguments, not self-initialized. HEADER_INIT_KEYS = ( HEADER_KEYS_V1_1_TO_INIT + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 + + HEADER_KEYS_V1_5_TO_INIT ) HEADER_INT_KEYS = HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 @@ -137,6 +146,7 @@ HEADER_UNORDERED_KEYS = ( + HEADER_KEYS_V1_3 + HEADER_KEYS_V1_2 + HEADER_KEYS_V1_4 + + HEADER_KEYS_V1_5 ) # List of all the KeyValues currently being used to generate the file HEADER_ALL_KEYS = HEADER_KEYS_V1_1_ORDERED + HEADER_UNORDERED_KEYS @@ -331,6 +341,15 @@ class V3BWHeader(object): generator_started = cls.generator_started_from_file(state_fpath) recent_consensus_count = cls.consensus_count_from_file(state_fpath) timestamp = str(latest_bandwidth) + + # XXX: tech-debt: obtain the other values from the state file using + # this state variable. + # Store the state as an attribute of the object? + state = State(state_fpath) + tor_version = state.get('tor_version', None) + if tor_version: + kwargs['tor_version'] = tor_version + kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) if generator_started is not None: diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 40ffe16..566cc8d 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -50,10 +50,14 @@ earliest_bandwidth_l = KEYVALUE_SEP_V1.join(['earliest_bandwidth', generator_started = '2018-04-16T14:09:05' generator_started_l = KEYVALUE_SEP_V1.join(['generator_started', generator_started]) +tor_version = '0.4.2.5' +tor_version_l = KEYVALUE_SEP_V1.join(['tor_version', tor_version]) + header_extra_ls = [timestamp_l, version_l, earliest_bandwidth_l, file_created_l, generator_started_l, latest_bandwidth_l, - software_l, software_version_l, TERMINATOR] + software_l, software_version_l, tor_version_l, + TERMINATOR] header_extra_str = LINE_SEP.join(header_extra_ls) + LINE_SEP # Line produced without any scaling. @@ -90,7 +94,8 @@ def test_v3bwheader_extra_str(): header = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, - earliest_bandwidth=earliest_bandwidth) + earliest_bandwidth=earliest_bandwidth, + tor_version=tor_version) assert header_extra_str == str(header) @@ -98,7 +103,8 @@ def test_v3bwheader_from_lines(): header_obj = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, - earliest_bandwidth=earliest_bandwidth) + earliest_bandwidth=earliest_bandwidth, + tor_version=tor_version) header, _ = V3BWHeader.from_lines_v1(header_extra_ls) assert str(header_obj) == str(header) @@ -107,7 +113,8 @@ def test_v3bwheader_from_text(): header_obj = V3BWHeader(timestamp_l, file_created=file_created, generator_started=generator_started, - earliest_bandwidth=earliest_bandwidth) + earliest_bandwidth=earliest_bandwidth, + tor_version=tor_version) header, _ = V3BWHeader.from_text_v1(header_extra_str) assert str(header_obj) == str(header)

1 0

[sbws/maint-1.1] fix: v3bwfile: Move keys to correct constant
by juga＠torproject.org 09 Mar '20

09 Mar '20

commit 713ca6a121af2ab72862517de7bfd485cc415e15 Author: juga0 <juga(a)riseup.net> Date: Mon Feb 24 14:33:03 2020 +0000 fix: v3bwfile: Move keys to correct constant Part of #30196. --- sbws/lib/v3bwfile.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 7805fe2..04cc067 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -166,19 +166,6 @@ BWLINE_KEYS_V0 = ['node_id', 'bw'] BWLINE_KEYS_V1_1 = BWLINE_KEYS_V0 + \ ['master_key_ed25519', 'nick', 'rtt', 'time', 'success', 'error_stream', 'error_circ', 'error_misc', - # `vote=0` is used for the relays that were excluded to - # be reported in the bandwidth file and now they are - # reported. - # It tells Tor to do not vote on the relay. - # `unmeasured=1` is used for the same relays and it is - # added in case Tor would vote on them in future versions. - # Maybe these keys should not be included for the relays - # in which vote=1 and unmeasured=0. - 'vote', 'unmeasured', - # When there not enough eligible relays (not excluded) - # under_min_report is 1, `vote` is 0. - # Added in #29853. - 'under_min_report', # Added in #292951 'error_second_relay', 'error_destination'] BWLINE_KEYS_V1_2 = [ @@ -231,6 +218,19 @@ BWLINE_KEYS_V1_4 = [ # The number of measurements excluded because they are not at least X # (by default 2). 'relay_recent_measurements_excluded_few_count', + # `vote=0` is used for the relays that were excluded to + # be reported in the bandwidth file and now they are + # reported. + # It tells Tor to do not vote on the relay. + # `unmeasured=1` is used for the same relays and it is + # added in case Tor would vote on them in future versions. + # Maybe these keys should not be included for the relays + # in which vote=1 and unmeasured=0. + 'vote', 'unmeasured', + # When there not enough eligible relays (not excluded) + # under_min_report is 1, `vote` is 0. + # Added in #29853. + 'under_min_report', ] BWLINE_KEYS_V1 = BWLINE_KEYS_V1_1 + BWLINE_KEYS_V1_2 \ + BWLINE_KEYS_V1_4 @@ -244,9 +244,6 @@ BWLINE_INT_KEYS = ( "error_stream", "error_circ", "error_misc", - "vote", - "unmeasured", - "under_min_report", ] + BWLINE_KEYS_V1_2 + BWLINE_KEYS_V1_4

1 0

[translation/tor-launcher-properties_completed] https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-properties_completed
by translation＠torproject.org 09 Mar '20

09 Mar '20

commit 3cd6218192f52df3d58be2fd8de0d9875fbb97dc Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 9 14:23:18 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-proper… --- mk/torlauncher.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mk/torlauncher.properties b/mk/torlauncher.properties index 4055e2407b..f7071713fc 100644 --- a/mk/torlauncher.properties +++ b/mk/torlauncher.properties @@ -1,4 +1,4 @@ -### Copyright (c) 2016, The Tor Project, Inc. +### Copyright (c) 2020, The Tor Project, Inc. ### See LICENSE for licensing information. torlauncher.error_title=Tor стартувач @@ -16,6 +16,7 @@ torlauncher.unable_to_start_tor=Не може да се стартува Tor.\n\ torlauncher.tor_missing=Извршната датотека на Tor недостасува. torlauncher.torrc_missing=torrc датотеката недостасува и не може да биде создадена. torlauncher.datadir_missing=Tor податочниот директориум не постои и не може да биде создаден. +torlauncher.onionauthdir_missing=Директориумот за Tor onion автентификација не постои или не може да биде создаден. torlauncher.password_hash_missing=Неуспешно добивање на хаш-лозинка. torlauncher.failed_to_get_settings=Невозможно е да се добијат Tor поставки.\n\n%S

1 0