February 2020 - tor-commits - lists.torproject.org

[bridgedb/master] Replace has_key method call
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit adc0f16639e9f61c06c7d0d49c51199c115488c6 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:18:18 2020 -0800 Replace has_key method call Python's has_key method was always pointless since "x in my_dict" does the same. The method was removed in python 3 so this fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 255, in test_HTTPSDistributor_getBridges_with_varied_blocked_bridges bridge.setBlockedIn('cn', methodname='vanilla') File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 1501, in setBlockedIn self._addBlockByKey(key, countryCode) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 1381, in _addBlockByKey if self._blockedIn.has_key(key): builtins.AttributeError: 'dict' object has no attribute 'has_key' Test results changed as follows... before: FAILED (skips=114, failures=15, errors=140, successes=716) after: FAILED (skips=114, failures=15, errors=133, successes=723) --- bridgedb/bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py index 827a156..cdd1ae8 100644 --- a/bridgedb/bridges.py +++ b/bridgedb/bridges.py @@ -1377,7 +1377,7 @@ class Bridge(BridgeBackwardsCompatibility): :meth:`_getBlockKey`. :param str countryCode: A two-character country code specifier. """ - if self._blockedIn.has_key(key): + if key in self._blockedIn: self._blockedIn[key].append(countryCode.lower()) else: self._blockedIn[key] = [countryCode.lower(),]

1 0

[bridgedb/master] Replace func_name method
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 1a1cc3fbd56ce4f6ef05eb82226b49745221c4cf Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:14:54 2020 -0800 Replace func_name method This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 154, in test_HTTPSDistributor_prepopulateRings_without_proxies dist.prepopulateRings() File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/distributor.py", line 263, in prepopulateRings populate_from=self.hashring.bridges) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/Bridges.py", line 673, in addRing filterNames = self.extractFilterNames(ringname) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/Bridges.py", line 630, in extractFilterNames for filterName in [x.func_name for x in list(ringname)]: File "/home/atagar/Desktop/tor/bridgedb/bridgedb/Bridges.py", line 630, in <listcomp> for filterName in [x.func_name for x in list(ringname)]: builtins.AttributeError: 'function' object has no attribute 'func_name' Test results changed as follows... before: FAILED (skips=114, failures=14, errors=143, successes=714) after: FAILED (skips=114, failures=15, errors=140, successes=716) --- bridgedb/Bridges.py | 4 ++-- bridgedb/distributors/https/distributor.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index 20a78cb..135174c 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -627,8 +627,8 @@ class FilteredBridgeSplitter(object): """ filterNames = [] - for filterName in [x.func_name for x in list(ringname)]: - # Using `assignBridgesToSubring.func_name` gives us a messy + for filterName in [x.__name__ for x in list(ringname)]: + # Using `assignBridgesToSubring.__name__` gives us a messy # string which includes all parameters and memory addresses. Get # rid of this by partitioning at the first `(`: realFilterName = filterName.partition('(')[0] diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py index 7bce8d9..03aa970 100644 --- a/bridgedb/distributors/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -326,7 +326,7 @@ class HTTPSDistributor(Distributor): logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) logging.debug("Total bridges: %d" % len(self.hashring)) - logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) + logging.debug("Bridge filters: %s" % ' '.join([x.__name__ for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.hashring.filterRings.keys():

1 0

[bridgedb/master] Fix debug hexify statement
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 442e38c2cd5f077592aa9fea19d8a62276b14f3f Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:22:46 2020 -0800 Fix debug hexify statement Oops, missed that there was other instances of hex conversion... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 235, in test_HTTPSDistributor_getBridges_with_some_blocked_bridges bridges = dist.getBridges(clientRequest1, 1) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/distributor.py", line 327, in getBridges logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) builtins.AttributeError: 'bytes' object has no attribute 'encode' Test results changed as follows... before: FAILED (skips=114, failures=15, errors=133, successes=723) after: FAILED (skips=114, failures=15, errors=122, successes=734) --- bridgedb/Bridges.py | 3 ++- bridgedb/distributors/https/distributor.py | 3 ++- bridgedb/test/test_crypto.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index 135174c..46b7afe 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -10,6 +10,7 @@ them into hashrings for distributors. """ +import binascii import bisect import logging import re @@ -329,7 +330,7 @@ class BridgeRing(object): else: logging.debug( "Got duplicate bridge %r in main hashring for position %r." - % (logSafely(k.encode('hex')), pos.encode('hex'))) + % (logSafely(binascii.hexlify(k).decode('utf-8')), binascii.hexlify(pos).decode('utf-8'))) keys.sort() if filterBySubnet: diff --git a/bridgedb/distributors/https/distributor.py b/bridgedb/distributors/https/distributor.py index 03aa970..f3e2e9b 100644 --- a/bridgedb/distributors/https/distributor.py +++ b/bridgedb/distributors/https/distributor.py @@ -20,6 +20,7 @@ A Distributor that hands out bridges through a web interface. :parts: 1 """ +import binascii import ipaddr import logging @@ -324,7 +325,7 @@ class HTTPSDistributor(Distributor): logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) - logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) + logging.debug("Assigned client to subhashring position: %s" % binascii.hexlify(position).decode('utf-8')) logging.debug("Total bridges: %d" % len(self.hashring)) logging.debug("Bridge filters: %s" % ' '.join([x.__name__ for x in filters])) diff --git a/bridgedb/test/test_crypto.py b/bridgedb/test/test_crypto.py index 000ca0f..9255c30 100644 --- a/bridgedb/test/test_crypto.py +++ b/bridgedb/test/test_crypto.py @@ -15,6 +15,7 @@ from __future__ import print_function from __future__ import unicode_literals import base64 +import binascii import io import logging import math @@ -87,7 +88,7 @@ class GetKeyTests(unittest.TestCase): """The example key and the one read from file differ! key (in hex): %s SEKRIT_KEY (in hex): %s""" - % (key.encode('hex'), SEKRIT_KEY.encode('hex'))) + % (binascii.hexlify(key).decode('utf-8'), binascii.hexlify(SEKRIT_KEY).decode('utf-8'))) class InitializeGnuPGTests(unittest.TestCase):

1 0

[bridgedb/master] Replace dict's iteritems with items
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit a7bb3fbab43ca01455e283d29a3f68cbb9a0a2fe Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:58:14 2020 -0800 Replace dict's iteritems with items Python's methods for keys, values, and items now provide iterators so there is no longer a separate iteritems method. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_metrics.py", line 108, in test_export metrics.export(pseudo_fh, 0) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/metrics.py", line 131, in export httpsLines = httpsMetrix.getMetrics() File "/home/atagar/Desktop/tor/bridgedb/bridgedb/metrics.py", line 235, in getMetrics for key, value in self.coldMetrics.iteritems(): builtins.AttributeError: 'dict' object has no attribute 'iteritems' This changes test results as follows... before: FAILED (skips=114, failures=15, errors=111, successes=745) after: FAILED (skips=114, failures=15, errors=109, successes=747) --- bridgedb/Bridges.py | 2 +- bridgedb/metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index 46b7afe..93d0190 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -539,7 +539,7 @@ class BridgeSplitter(object): logging.info("Current rings: %s" % " ".join(self.ringsByName)) def dumpAssignments(self, f, description=""): - for name,ring in self.ringsByName.iteritems(): + for name,ring in self.ringsByName.items(): ring.dumpAssignments(f, "%s %s" % (description, name)) diff --git a/bridgedb/metrics.py b/bridgedb/metrics.py index 5d8bda3..2785874 100644 --- a/bridgedb/metrics.py +++ b/bridgedb/metrics.py @@ -232,7 +232,7 @@ class Metrics(metaclass=Singleton): :returns: A list of metric lines. """ lines = [] - for key, value in self.coldMetrics.iteritems(): + for key, value in self.coldMetrics.items(): # Round up our value to the nearest multiple of self.binSize to # reduce the accuracy of our real values. if (value % self.binSize) > 0:

1 0

[bridgedb/master] Fix singleton metaclass definition
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit e3fc34053f98990d28a2121dabd93f4ee5f6edb3 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:48:34 2020 -0800 Fix singleton metaclass definition This fixes the following... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_metrics.py", line 41, in setUp type(metrics.HTTPSMetrics()).clear() builtins.AttributeError: type object 'HTTPSMetrics' has no attribute 'clear' Test results changed as follows... before: FAILED (skips=114, failures=15, errors=116, successes=740) after: FAILED (skips=114, failures=15, errors=111, successes=745) --- bridgedb/metrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bridgedb/metrics.py b/bridgedb/metrics.py index 6b2e5cb..5d8bda3 100644 --- a/bridgedb/metrics.py +++ b/bridgedb/metrics.py @@ -184,16 +184,13 @@ class Singleton(type): pass -class Metrics(object): +class Metrics(metaclass=Singleton): """Base class representing metrics. This class provides functionality that our three distribution mechanisms share. """ - # We're using a meta class to implement a singleton for Metrics. - __metaclass__ = Singleton - def __init__(self, binSize=BIN_SIZE): logging.debug("Instantiating metrics class.") self.binSize = binSize

1 0

[tor-browser-build/master] Bug 33283: Pick up RBM caching logic
by sysrqb＠torproject.org 19 Feb '20

19 Feb '20

commit ffc6f844dbbf1985f5d10bd8dab2b15d9c8c2e75 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Wed Feb 19 17:50:41 2020 +0000 Bug 33283: Pick up RBM caching logic --- rbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbm b/rbm index 94fce31..e4f12ab 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 94fce31e54d3fd491cdb3e3b9401919c0e841360 +Subproject commit e4f12abe9ed81050994b5345c21b988005259396

1 0

[rbm/master] Bug 33283: Add caching for the exec template function
by sysrqb＠torproject.org 19 Feb '20

19 Feb '20

commit ce6b9136fc9e006536b34844946d67aa92cc0c21 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Feb 12 11:16:22 2020 +0100 Bug 33283: Add caching for the exec template function --- lib/RBM.pm | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/RBM.pm b/lib/RBM.pm index 4db951a..f419c24 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -475,10 +475,14 @@ sub run_script { sub execute { my ($project, $cmd, $options) = @_; + CORE::state %cache; + my $res_name; my $old_cwd = getcwd; if (project_config($project, 'git_url', $options)) { my $git_hash = project_config($project, 'git_hash', $options) || exit_error "No git_hash specified for project $project"; + $res_name = "git-$git_hash-$cmd"; + return $cache{$res_name} if exists $cache{$res_name}; git_clone_fetch_chdir($project, $options); my ($stdout, $stderr, $success, $exit_code) = capture_exec('git', 'checkout', $git_hash); @@ -494,6 +498,8 @@ sub execute { } elsif (project_config($project, 'hg_url', $options)) { my $hg_hash = project_config($project, 'hg_hash', $options) || exit_error "No hg_hash specified for project $project"; + $res_name = "hg-$hg_hash-$cmd"; + return $cache{$res_name} if exists $cache{$res_name}; hg_clone_fetch_chdir($project, $options); my ($stdout, $stderr, $success, $exit_code) = capture_exec('hg', 'update', '-C', $hg_hash); @@ -503,7 +509,8 @@ sub execute { = run_script($project, $cmd, \&capture_exec); chdir($old_cwd); chomp $stdout; - return $success ? $stdout : undef; + $cache{$res_name} = $success ? $stdout : undef; + return $cache{$res_name}; } sub gpg_id {

1 0

[rbm/master] Bug 33283: Avoid computing sha256sum of files multiple times
by sysrqb＠torproject.org 19 Feb '20

19 Feb '20

commit e4f12abe9ed81050994b5345c21b988005259396 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Feb 12 16:09:00 2020 +0100 Bug 33283: Avoid computing sha256sum of files multiple times Instead, use the sha256file functon, which keeps the sha256sum of files to avoid computing it twice. The function can now take a HASH as a second argument, which can contain the key remove_cache indicating that the sha256sum of a file should be cleared: we use this when we find that a file has the wrong checksum and needs to be downloaded again. --- lib/RBM.pm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/RBM.pm b/lib/RBM.pm index f419c24..375648f 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -610,6 +610,11 @@ sub maketar { sub sha256file { CORE::state %res; my $f = rbm_path(shift); + my $opt = shift; + if (ref $opt eq 'HASH' && $opt->{remove_cache}) { + delete $res{$f}; + return; + } return $res{$f} if exists $res{$f}; return $res{$f} = -f $f ? sha256_hex(path($f)->slurp_raw) : ''; } @@ -724,7 +729,8 @@ sub input_file_need_dl { return undef if $action eq 'getfpaths'; if ($fname && $input_file->{sha256sum} - && $t->('sha256sum') ne sha256_hex(path($fname)->slurp_raw)) { + && $t->('sha256sum') ne sha256file($fname)) { + sha256file($fname, { remove_cache => 1 }); $fname = undef; } if ($action eq 'input_files_id') { @@ -928,7 +934,7 @@ sub input_files { } exit_error "Missing file $name" unless $fname; if ($t->('sha256sum') - && $t->('sha256sum') ne sha256_hex(path($fname)->slurp_raw)) { + && $t->('sha256sum') ne sha256file($fname)) { exit_error "Can't have sha256sum on directory: $fname" if -d $fname; exit_error "Wrong sha256sum for $fname.\n" . "Expected sha256sum: " . $t->('sha256sum');

1 0

[tor-browser-build/master] Bug 33380: Add *.json to sha256sums-unsigned-build.txt
by sysrqb＠torproject.org 19 Feb '20

19 Feb '20

commit 19497b09ae5f68768a1d2690249b51b6fca89b41 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Feb 19 12:51:44 2020 +0100 Bug 33380: Add *.json to sha256sums-unsigned-build.txt --- projects/release/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/release/build b/projects/release/build index c1badd6..c859828 100644 --- a/projects/release/build +++ b/projects/release/build @@ -46,7 +46,7 @@ RewriteRule ^sha256sums.txt.asc$ sha256sums-unsigned-build.txt.asc RewriteRule ^sha256sums.incrementals.txt$ sha256sums-unsigned-build.incrementals.txt RewriteRule ^sha256sums.incrementals.txt.asc$ sha256sums-unsigned-build.incrementals.txt.asc EOF -sha256sum $(ls -1 *.exe *.tar.xz *.dmg *.mar *.zip *.tar.gz *.apk | grep -v '\.incremental\.mar$' | sort) > sha256sums-unsigned-build.txt +sha256sum $(ls -1 *.exe *.tar.xz *.dmg *.mar *.zip *.tar.gz *.apk *.json | grep -v '\.incremental\.mar$' | sort) > sha256sums-unsigned-build.txt [% IF c("var/sign_build") -%] gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-unsigned-build.txt [% END -%]

1 0

[tor/master] Do not build selftest.c when running relay mode is disabled.
by nickm＠torproject.org 19 Feb '20

19 Feb '20

commit e85f46b7be0e658f3d5cd76dff3f1a98b426f22e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Feb 18 13:30:20 2020 -0500 Do not build selftest.c when running relay mode is disabled. --- changes/ticket33370 | 3 +++ src/feature/relay/include.am | 4 ++-- src/feature/relay/selftest.h | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/changes/ticket33370 b/changes/ticket33370 new file mode 100644 index 000000000..41e03357f --- /dev/null +++ b/changes/ticket33370 @@ -0,0 +1,3 @@ + o Minor features (client-only compilation): + - Disable more of our self-testing code when support for relay mode is + disabled. Closes ticket 33370. diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am index a4c025ae1..2f730a685 100644 --- a/src/feature/relay/include.am +++ b/src/feature/relay/include.am @@ -5,8 +5,7 @@ LIBTOR_APP_A_SOURCES += \ src/feature/relay/ext_orport.c \ src/feature/relay/onion_queue.c \ src/feature/relay/router.c \ - src/feature/relay/routerkeys.c \ - src/feature/relay/selftest.c + src/feature/relay/routerkeys.c # The Relay module. @@ -16,6 +15,7 @@ MODULE_RELAY_SOURCES = \ src/feature/relay/relay_config.c \ src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ + src/feature/relay/selftest.c \ src/feature/relay/transport_config.c # ADD_C_FILE: INSERT HEADERS HERE. diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h index 94f305f20..c7987cf4a 100644 --- a/src/feature/relay/selftest.h +++ b/src/feature/relay/selftest.h @@ -12,6 +12,7 @@ #ifndef TOR_SELFTEST_H #define TOR_SELFTEST_H +#ifdef HAVE_MODULE_RELAY struct or_options_t; int check_whether_orport_reachable(const struct or_options_t *options); int check_whether_dirport_reachable(const struct or_options_t *options); @@ -20,5 +21,33 @@ void router_do_reachability_checks(int test_or, int test_dir); void router_orport_found_reachable(void); void router_dirport_found_reachable(void); void router_perform_bandwidth_test(int num_circs, time_t now); +#else /* !defined(HAVE_MODULE_RELAY) */ + +#define check_whether_orport_reachable(opts) \ + ((void)(opts), 0) +#define check_whether_dirport_reachable(opts) \ + ((void)(opts), 0) + +#define router_orport_found_reachable() \ + STMT_NIL +#define router_dirport_found_reachable() \ + STMT_NIL + +static inline void +router_do_reachability_checks(int test_or, int test_dir) +{ + (void)test_or; + (void)test_dir; + tor_assert_nonfatal_unreached(); +} +static inline void +router_perform_bandwidth_test(int num_circs, time_t now) +{ + (void)num_circs; + (void)now; + tor_assert_nonfatal_unreached(); +} + +#endif /* defined(HAVE_MODULE_RELAY) */ #endif /* !defined(TOR_SELFTEST_H) */

1 0