October 2018 - tor-commits - lists.torproject.org

[sbws/master] Create method to add new header attrs
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit c5b342d25b989f7bbdd8b22b5e6871dbe4eb735a Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:53:24 2018 +0000 Create method to add new header attrs --- sbws/lib/v3bwfile.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index e33a7ec..83df447 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -229,6 +229,11 @@ class V3BWHeader(object): def num_lines(self): return len(self.__str__().split(LINE_SEP)) + def add_stats(self, **kwargs): + # Using kwargs because attributes might chage. + [setattr(self, k, str(v)) for k, v in kwargs.items() + if k in STATS_KEYVALUES] + class V3BWLine(object): """

1 0

[sbws/master] Create bwfile passing consensus path
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 06af6d34c22006f1f26fc10e2d78a825a7c7d883 Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:48:38 2018 +0000 Create bwfile passing consensus path and rm condition about whether the min percentage of measured relays was reached. It will be checked when writing the file. Also pass the arg rm_link to write. --- sbws/core/generate.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index f147993..66c9d7d 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,7 +1,7 @@ from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_ROUND_DIG, DAY_SECS, NUM_MIN_RESULTS) -from sbws.lib.v3bwfile import V3BWFile, is_min_percent_measured +from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter import os @@ -95,21 +95,18 @@ def main(args, conf): 'ran sbws scanner recently?)') return state_fpath = conf.getpath('paths', 'state_fname') + consensus_path = os.path.join(conf.getpath('tor', 'datadir'), + "cached-consensus") bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant, scaling_method, torflow_cap=args.torflow_bw_margin, torflow_round_digs=args.torflow_round_digs, secs_recent=args.secs_recent, secs_away=args.secs_away, - min_num=args.min_num) + min_num=args.min_num, + consensus_path=consensus_path) output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) - # Check percentage of measured relays. - consensus_path = os.path.join(conf.getpath('tor', 'datadir'), - "cached-consensus") - if is_min_percent_measured(consensus_path): - bw_file.write(output) - else: - bw_file.rm_link(output) + bw_file.write(output, args.rm_link) bw_file.info_stats

1 0

[sbws/master] Add new constants to use as header attributes
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit a53611826fd5ddbc9b8e72cd719f8f7f57d4b382 Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:51:34 2018 +0000 Add new constants to use as header attributes --- sbws/lib/v3bwfile.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index c69e674..e67e556 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -27,8 +27,13 @@ KEYVALUE_SEP_V200 = ' ' # List of the extra KeyValues accepted by the class EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', 'earliest_bandwidth', 'generator_started'] +STATS_KEYVALUES = ['num_measured_relays', 'num_target_relays', + 'num_net_relays', 'perc_measured_relays', + 'perc_measured_targed'] +KEYVALUES_INT = STATS_KEYVALUES # List of all unordered KeyValues currently being used to generate the file -UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + ['latest_bandwidth'] +UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + STATS_KEYVALUES + \ + ['latest_bandwidth'] # List of all the KeyValues currently being used to generate the file ALL_KEYVALUES = ['version'] + UNORDERED_KEYVALUES TERMINATOR = '===='

1 0

[sbws/master] Move function to check percentage to classmethod
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 5003c6bc4a56563d05ffc6345661df8955e07f3f Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:52:30 2018 +0000 Move function to check percentage to classmethod --- sbws/lib/v3bwfile.py | 56 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index e67e556..e33a7ec 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -71,22 +71,6 @@ def result_type_to_key(type_str): return type_str.replace('-', '_') -def is_min_percent_measured(consensus_path): - """""" - # cached-consensus should be updated every time that scanner get the - # network status or descriptors? - # It will not be updated to the last consensus, but the list of measured - # relays is not either. - descs = parse_file(consensus_path) - num_relays_net = len(list(descs)) - if num_relays_net * MIN_REPORT / 100: - log.warning('The percentage of the measured relays is less than the %s' - '%% of the relays in the network (%s).', - MIN_REPORT, num_relays_net) - return False - return True - - class V3BWHeader(object): """ Create a bandwidth measurements (V3bw) header @@ -796,6 +780,46 @@ class V3BWFile(object): l.bw = max(round(bw_new), 1) return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse) + @staticmethod + def measured_progress_stats(bw_lines, consensus_path): + """ Statistics about measurements progress, + to be included in the header. + + :param list bw_lines: the bw_lines after scaling and applying filters. + :param str consensus_path: the path to the cached consensus file. + :returns dict, bool: Statistics about the progress made with + measurements and whether the percentage of measured relays has been + reached. + + """ + # cached-consensus should be updated every time that scanner get the + # network status or descriptors? + # It will not be updated to the last consensus, but the list of + # measured relays is not either. + assert isinstance(consensus_path, str) + assert isinstance(bw_lines, list) + statsd = {} + statsd['num_measured_relays'] = len(bw_lines) + statsd['num_net_relays'] = len(list(parse_file(consensus_path))) + statsd['num_target_relays'] = round(statsd['num_net_relays'] + * MIN_REPORT / 100) + statsd['perc_measured_relays'] = round(len(bw_lines) * 100 + / statsd['num_target_relays']) + statsd['perc_measured_targed'] = MIN_REPORT + if statsd['num_measured_relays'] < statsd['num_target_relays']: + log.warning('The percentage of the measured relays is less than' + ' the %s%% of the relays in the network (%s).', + MIN_REPORT, statsd['num_net_relays']) + return statsd, False + return statsd, True + + @property + def is_min_perc(self): + if getattr(self.header, 'num_measured_relays', 0) \ + < getattr(self.header, 'num_target_relays', 0): + return False + return True + @property def sum_bw(self): return sum([l.bw for l in self.bw_lines])

1 0

[sbws/master] Add entry about bw file header statistics
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 6276cdaaca084a4c070445c5b9196dae90a5b102 Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:58:48 2018 +0000 Add entry about bw file header statistics --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 778b5f6..6c13dd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Write bw file only when the percentage of measured relays is bigger than 60% (#28062) +- Add option to remove link when the percentage of measured relays is less than + the 60% and instead include some statistics in the header (#28076) ## [0.8.0] - 2018-10-08

1 0

[sbws/master] Don't assume we can resolve a domain name
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit a7dce5ce0c191cce9e3dc8e9e7e34a2c1ee91cfd Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Oct 22 10:13:43 2018 -0400 Don't assume we can resolve a domain name Even for a valid domain that usually resolves, sometimes the local DNS resolver won't be able to resolve it (duh). Catch the exception that gets thrown in that instance and fail gracefully by returning an empty list. --- CHANGELOG.md | 1 + sbws/globals.py | 10 +++++++++- sbws/lib/relaylist.py | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90da685..f8160af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Update python minimal version in setup (#28043) +- Catch unhandled exception when we fail to resolve a domain name (#28141) ### Changed diff --git a/sbws/globals.py b/sbws/globals.py index 52c31bf..e6ba3f5 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -67,8 +67,16 @@ def touch_file(fname, times=None): def resolve(hostname, ipv4_only=False, ipv6_only=False): assert not (ipv4_only and ipv6_only) + results = [] + try: + results = socket.getaddrinfo(hostname, 0): + except socket.gaierror: + log.warn( + 'Unable to resolve %s hostname. Returning empty list of addresses', + hostname) + return [] ret = set() - for result in socket.getaddrinfo(hostname, 0): + for result in results: fam, _, _, _, addr = result if fam == socket.AddressFamily.AF_INET6 and not ipv4_only: ret.add(addr[0]) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 58cae75..9402574 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -120,7 +120,10 @@ class Relay: # # Also, only use the first ipv4/6 we get even if there is more than # one. - host = resolve(host)[0] + results = resolve(host) + if not len(results): + return False + host = results[0] assert is_valid_ipv4_address(host) or is_valid_ipv6_address(host) return self.exit_policy.can_exit_to(host, port)

1 0

[sbws/master] Fix syntax error
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 8787741b7fcebdc9fb18b68ea27bb8f858d93efc Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Mon Oct 22 10:29:47 2018 -0400 Fix syntax error --- sbws/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/globals.py b/sbws/globals.py index e6ba3f5..5d1e667 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -69,7 +69,7 @@ def resolve(hostname, ipv4_only=False, ipv6_only=False): assert not (ipv4_only and ipv6_only) results = [] try: - results = socket.getaddrinfo(hostname, 0): + results = socket.getaddrinfo(hostname, 0) except socket.gaierror: log.warn( 'Unable to resolve %s hostname. Returning empty list of addresses',

1 0

[tor-launcher/master] Revert "Bug 28082: Add locales cs, el, hu, ka"
by gk＠torproject.org 24 Oct '18

24 Oct '18

commit 670882ad2936a906f3e2d84230714446e98a214c Author: Georg Koppen <gk(a)torproject.org> Date: Wed Oct 24 08:50:09 2018 +0000 Revert "Bug 28082: Add locales cs, el, hu, ka" This reverts commit 3aa20983122b4aaed6ae77cfc5bdb48107aa33a4. --- src/chrome/content/network-settings.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/chrome/content/network-settings.js b/src/chrome/content/network-settings.js index 1599c28..aaa91ac 100644 --- a/src/chrome/content/network-settings.js +++ b/src/chrome/content/network-settings.js @@ -306,21 +306,17 @@ function populateLocaleList(aLangPackAddons) "ar" : "\u0627\u0644\u0639\u0631\u0628\u064a\u0629", "bn-BD" : "\u09ac\u09be\u0982\u09b2\u09be", "ca" : "català", - "cs" : "\u010De\u0161tina", "da" : "Dansk", "de" : "Deutsch", - "el" : "\u03B5\u03BB\u03BB\u03B7\u03BD\u03B9\u03BA\u03AC", "es-ES" : "Espa\u00f1ol", "fa" : "\u0641\u0627\u0631\u0633\u06cc", "fr" : "Fran\u00e7ais", "ga" : "Gaeilge (Irish)", "he" : "\u05e2\u05d1\u05e8\u05d9\u05ea", - "hu" : "Magyar nyelv", "id" : "bahasa Indonesia", "is" : "\u0355slenska", "it" : "Italiano", "ja" : "\u65e5\u672c\u8a9e", - "ka" : "\u10E5\u10D0\u10E0\u10D7\u10E3\u10DA\u10D8", "ko" : "\ud55c\uad6d\uc5b4", "nl" : "Nederlands", "nb" : "Norsk bokmå\u0345l",

1 0

[torbutton/master] Revert "Bug 28082: Add locales cs, el, hu, ka"
by gk＠torproject.org 24 Oct '18

24 Oct '18

commit 8f1ff9017e9f793156d1edce38dde8cfcf590b2b Author: Georg Koppen <gk(a)torproject.org> Date: Wed Oct 24 08:49:40 2018 +0000 Revert "Bug 28082: Add locales cs, el, hu, ka" This reverts commit c94a5589ffeabf71ea95c3c5c1a3396bdbe14da5. --- trans_tools/import-translations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trans_tools/import-translations.sh b/trans_tools/import-translations.sh index 8f562208..d13f8259 100755 --- a/trans_tools/import-translations.sh +++ b/trans_tools/import-translations.sh @@ -2,7 +2,7 @@ # This var comes from the TBB locale list. # XXX: Find some way to keep this, tor-launcher, and Tor Browser in sync -BUNDLE_LOCALES="ar ca cs da de el es fa fr ga he hu id is it ja ka ko nb nl pl pt-BR ru sv tr vi zh-CN zh-TW" +BUNDLE_LOCALES="ar ca da de es fa fr ga he id is it ja ko nb nl pl pt-BR ru sv tr vi zh-CN zh-TW" # XXX: Basque (eu) by request in #10687. # This is not used for official builds, but should remain

1 0

[tor-browser-build/master] Revert "Bug 28082: Add locales cs, el, hu, ka"
by gk＠torproject.org 24 Oct '18

24 Oct '18

commit 6803d9a8a4e7c851629cab47342a6579a60f64cd Author: Georg Koppen <gk(a)torproject.org> Date: Wed Oct 24 08:48:35 2018 +0000 Revert "Bug 28082: Add locales cs, el, hu, ka" This reverts commit fbb72c65a24ac70619062ebcd16c67a9afb4feae. --- rbm.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rbm.conf b/rbm.conf index 5e981ac..23f3598 100644 --- a/rbm.conf +++ b/rbm.conf @@ -53,21 +53,17 @@ var: locales: - ar - ca - - cs - da - de - - el - es-ES - fa - fr - ga-IE - he - - hu - id - is - it - '[% c("var/locale_ja") %]' - - ka - ko - nb-NO - nl

1 0