January 2020 - tor-commits - lists.torproject.org

[tor/master] Merge remote-tracking branch 'tor-github/pr/1671'
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 0b82b5ee4248d8dce60556524b08419cec90d214 Merge: 697527764 c8b6392b4 Author: teor <teor(a)torproject.org> Date: Mon Jan 20 15:40:42 2020 +1000 Merge remote-tracking branch 'tor-github/pr/1671' changes/bug32778 | 3 ++ changes/bug32883 | 5 ---- src/app/main/main.c | 11 ++----- src/app/main/main.h | 3 ++ src/app/main/ntmain.c | 59 +++++++++++++++++++++++++++----------- src/feature/api/tor_api_internal.h | 5 ---- 6 files changed, 50 insertions(+), 36 deletions(-)

1 0

[tor/master] Update routerstatus_has_changed tests.
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit c3633e6b295f9dbafd4c86795dfa79ed784c3bd2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 17 09:20:00 2020 -0500 Update routerstatus_has_changed tests. Now they also check whether output of routerstatus_format_entry() has changed. --- src/test/test_nodelist.c | 49 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 8a36de58d..d47cfc090 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -18,6 +18,8 @@ #include "feature/nodelist/torcert.h" #include "core/or/extend_info_st.h" +#include "feature/dirauth/dirvote.h" +#include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" @@ -1252,6 +1254,7 @@ test_nodelist_routerstatus_has_changed(void *arg) { (void)arg; routerstatus_t rs_orig, rs; + char *fmt_orig = NULL, *fmt = NULL; memset(&rs_orig, 0, sizeof(rs_orig)); strlcpy(rs_orig.nickname, "friendly", sizeof(rs_orig.nickname)); memcpy(rs_orig.identity_digest, "abcdefghijklmnopqrst", 20); @@ -1259,16 +1262,41 @@ test_nodelist_routerstatus_has_changed(void *arg) rs_orig.addr = 0x7f000001; rs_orig.or_port = 3; rs_orig.published_on = time(NULL); + rs_orig.has_bandwidth = 1; + rs_orig.bandwidth_kb = 20; #define COPY() memcpy(&rs, &rs_orig, sizeof(rs)) +#define FORMAT() \ + STMT_BEGIN \ + tor_free(fmt_orig); \ + tor_free(fmt); \ + fmt_orig = routerstatus_format_entry(&rs_orig, NULL, NULL, \ + NS_CONTROL_PORT, \ + ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD, \ + NULL); \ + fmt = routerstatus_format_entry(&rs, NULL, NULL, NS_CONTROL_PORT, \ + ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD, \ + NULL); \ + tt_assert(fmt_orig); \ + tt_assert(fmt); \ + STMT_END #define ASSERT_SAME() \ STMT_BEGIN \ tt_assert(! routerstatus_has_changed(&rs_orig, &rs)); \ + FORMAT(); \ + tt_str_op(fmt_orig, OP_EQ, fmt); \ COPY(); \ STMT_END #define ASSERT_CHANGED() \ STMT_BEGIN \ tt_assert(routerstatus_has_changed(&rs_orig, &rs)); \ + FORMAT(); \ + tt_str_op(fmt_orig, OP_NE, fmt); \ + COPY(); \ + STMT_END +#define ASSERT_CHANGED_NO_FORMAT() \ + STMT_BEGIN \ + tt_assert(routerstatus_has_changed(&rs_orig, &rs)); \ COPY(); \ STMT_END @@ -1296,6 +1324,9 @@ test_nodelist_routerstatus_has_changed(void *arg) tor_addr_parse(&rs.ipv6_addr, "1234::56"); ASSERT_CHANGED(); + tor_addr_parse(&rs_orig.ipv6_addr, "1234::56"); + rs_orig.ipv6_orport = 99; + COPY(); rs.ipv6_orport = 22; ASSERT_CHANGED(); @@ -1314,13 +1345,13 @@ test_nodelist_routerstatus_has_changed(void *arg) rs.is_flagged_running = 1; ASSERT_CHANGED(); - // Isn't this obsolete? + // This option is obsolete and not actually formatted. rs.is_named = 1; - ASSERT_CHANGED(); + ASSERT_CHANGED_NO_FORMAT(); - // Isn't this obsolete? + // This option is obsolete and not actually formatted. rs.is_unnamed = 1; - ASSERT_CHANGED(); + ASSERT_CHANGED_NO_FORMAT(); rs.is_valid = 1; ASSERT_CHANGED(); @@ -1340,10 +1371,12 @@ test_nodelist_routerstatus_has_changed(void *arg) rs.is_staledesc = 1; ASSERT_CHANGED(); - rs.has_bandwidth = 1; - ASSERT_CHANGED(); + // Setting this to zero crashes us with an assertion failure in + // routerstatus_format_entry() if we don't have a descriptor. + rs.has_bandwidth = 0; + ASSERT_CHANGED_NO_FORMAT(); - // Does not actually matter unless exitsummary changes. + // Does not actually matter; not visible to controller. rs.has_exitsummary = 1; ASSERT_SAME(); @@ -1374,6 +1407,8 @@ test_nodelist_routerstatus_has_changed(void *arg) #undef COPY #undef ASSERT_SAME #undef ASSERT_CHANGED + tor_free(fmt_orig); + tor_free(fmt); return; }

1 0

[tor/master] Merge remote-tracking branch 'tor-github/pr/1673'
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit b23b8fa4a18e7830fcbf95a2621dbc6096d68da5 Merge: 0b82b5ee4 af30be951 Author: teor <teor(a)torproject.org> Date: Mon Jan 20 15:52:40 2020 +1000 Merge remote-tracking branch 'tor-github/pr/1673' changes/ticket20218 | 3 + src/feature/nodelist/fmt_routerstatus.c | 2 + src/feature/nodelist/networkstatus.c | 28 ++++-- src/feature/nodelist/networkstatus.h | 2 + src/feature/nodelist/routerstatus_st.h | 4 + src/test/test_nodelist.c | 165 ++++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 8 deletions(-)

1 0

[tor/master] Rename test to match new identifier.
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 4541289f2a7e7a79da8cea38a94af5afd9ac9f72 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 17 09:27:58 2020 -0500 Rename test to match new identifier. --- src/test/test_nodelist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index c568e3698..388cd74f3 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -1250,7 +1250,7 @@ test_nodelist_router_get_verbose_nickname(void *arg) } static void -test_nodelist_routerstatus_has_changed(void *arg) +test_nodelist_routerstatus_has_visibly_changed(void *arg) { (void)arg; routerstatus_t rs_orig, rs;

1 0

[tor/master] Merge branch 'ticket20218_rebased_squashed' into ticket20218_merged
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 3851128e886906aee5afa790c96f5b7dc2940454 Merge: 4f88eb849 4541289f2 Author: teor <teor(a)torproject.org> Date: Mon Jan 20 15:50:54 2020 +1000 Merge branch 'ticket20218_rebased_squashed' into ticket20218_merged * ticket 32695 removed networkstatus_consensus_has_ipv6(), keep that change in master. * ticket 20218 modifies the function name and comment for routerstatus_has_visibly_changed(), keep that change in ticket20218_rebased_squashed. changes/ticket20218 | 3 + src/feature/nodelist/fmt_routerstatus.c | 2 + src/feature/nodelist/networkstatus.c | 28 ++++-- src/feature/nodelist/networkstatus.h | 2 + src/feature/nodelist/routerstatus_st.h | 4 + src/test/test_nodelist.c | 167 ++++++++++++++++++++++++++++++++ 6 files changed, 198 insertions(+), 8 deletions(-) diff --cc src/feature/nodelist/networkstatus.c index 58fad49a8,042852eb8..0d2ff96a6 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@@ -1578,10 -1578,42 +1579,16 @@@ networkstatus_consensus_is_already_down return answer; } - /** Given two router status entries for the same router identity, return 1 if - * if the contents have changed between them. Otherwise, return 0. */ - static int - routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b) -/* Does the current, reasonably live consensus have IPv6 addresses? - * Returns 1 if there is a reasonably live consensus and its consensus method - * includes IPv6 addresses in the consensus. - * Otherwise, if there is no consensus, or the method does not include IPv6 - * addresses, returns 0. */ -int -networkstatus_consensus_has_ipv6(const or_options_t* options) -{ - const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus( - approx_time(), - usable_consensus_flavor()); - - /* If we have no consensus, we have no IPv6 in it */ - if (!cons) { - return 0; - } - - /* Different flavours of consensus gained IPv6 at different times */ - if (we_use_microdescriptors_for_circuits(options)) { - return - cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS; - } else { - return 1; - } -} - + /** Given two router status entries for the same router identity, return 1 + * if the contents have changed between them. Otherwise, return 0. + * It only checks for fields that are output by control port. + * This should be kept in sync with the struct routerstatus_t + * and the printing function routerstatus_format_entry in + * NS_CONTROL_PORT mode. + **/ + STATIC int + routerstatus_has_visibly_changed(const routerstatus_t *a, + const routerstatus_t *b) { tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));

1 0

[tor/master] test/nodelist: Remove ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit af30be9515931d3eafbd6efc8e2caf1122ad6bec Author: teor <teor(a)torproject.org> Date: Mon Jan 20 13:42:13 2020 +1000 test/nodelist: Remove ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD When we removed obsolete consensus methods in 32695, we also removed ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD, so we need to remove it from the routerstatus format tests. Part of 20218. --- src/test/test_nodelist.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 388cd74f3..fbbbf0a99 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -1272,10 +1272,8 @@ test_nodelist_routerstatus_has_visibly_changed(void *arg) tor_free(fmt); \ fmt_orig = routerstatus_format_entry(&rs_orig, NULL, NULL, \ NS_CONTROL_PORT, \ - ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD, \ NULL); \ fmt = routerstatus_format_entry(&rs, NULL, NULL, NS_CONTROL_PORT, \ - ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD, \ NULL); \ tt_assert(fmt_orig); \ tt_assert(fmt); \

1 0

[tor/master] split a wide line
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit e2637214f2340e46c9e9042ca357e48c65df4f76 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 17 09:23:32 2020 -0500 split a wide line --- src/feature/nodelist/networkstatus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 03edfe62b..042852eb8 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1612,7 +1612,8 @@ networkstatus_consensus_has_ipv6(const or_options_t* options) * NS_CONTROL_PORT mode. **/ STATIC int -routerstatus_has_visibly_changed(const routerstatus_t *a, const routerstatus_t *b) +routerstatus_has_visibly_changed(const routerstatus_t *a, + const routerstatus_t *b) { tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));

1 0

[fallback-scripts/master] update: py3: Use py3 exception syntax
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 86247cdedd318ba34f0707f62e5606e5359eafe3 Author: teor <teor(a)torproject.org> Date: Mon Dec 2 15:01:57 2019 +1000 update: py3: Use py3 exception syntax Part of 28863. --- updateFallbackDirs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index 77e9bbb..fe78fbd 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -514,7 +514,7 @@ def write_to_file(str, file_name, max_len): try: with open(file_name, 'w') as f: f.write(str[0:max_len]) - except EnvironmentError, error: + except EnvironmentError as error: logging.error('Writing file %s failed: %d: %s'% (file_name, error.errno, @@ -526,7 +526,7 @@ def read_from_file(file_name, max_len): if os.path.isfile(file_name): with open(file_name, 'r') as f: return f.read(max_len) - except EnvironmentError, error: + except EnvironmentError as error: logging.info('Loading file %s failed: %d: %s'% (file_name, error.errno, @@ -557,7 +557,7 @@ def load_json_from_file(json_file_name): try: with open(json_file_name, 'r') as f: return json.load(f) - except EnvironmentError, error: + except EnvironmentError as error: raise Exception('Reading not-modified json file %s failed: %d: %s'% (json_file_name, error.errno, @@ -635,7 +635,7 @@ def onionoo_fetch(what, **kwargs): try: response = urllib2.urlopen(request) response_code = response.getcode() - except urllib2.HTTPError, error: + except urllib2.HTTPError as error: response_code = error.code if response_code == 304: # not modified pass @@ -1415,7 +1415,7 @@ class Candidate(object): end = datetime.datetime.utcnow() time_since_expiry = (end - consensus.valid_until).total_seconds() time_until_valid = (consensus.valid_after - end).total_seconds() - except Exception, stem_error: + except Exception as stem_error: end = datetime.datetime.utcnow() log_excluded('Unable to retrieve a consensus from %s: %s', nickname, stem_error)

1 0

[fallback-scripts/master] update: py3: Use the py3 print function
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 081415b46f39fdb88006aa414f62f7814c0d2bed Author: teor <teor(a)torproject.org> Date: Mon Dec 2 15:08:03 2019 +1000 update: py3: Use the py3 print function Part of 28863. --- updateFallbackDirs.py | 58 +++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index fe78fbd..eb7dd62 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -688,7 +688,7 @@ def onionoo_fetch(what, **kwargs): def fetch(what, **kwargs): #x = onionoo_fetch(what, **kwargs) # don't use sort_keys, as the order of or_addresses is significant - #print json.dumps(x, indent=4, separators=(',', ': ')) + #print(json.dumps(x, indent=4, separators=(',', ': '))) #sys.exit(0) return onionoo_fetch(what, **kwargs) @@ -978,8 +978,8 @@ class Candidate(object): logging.warning('Inconsistent time information in %s document for %s' %(p, which)) - #print json.dumps(generic_history, sort_keys=True, - # indent=4, separators=(',', ': ')) + #print(json.dumps(generic_history, sort_keys=True, + # indent=4, separators=(',', ': '))) return generic_history @staticmethod @@ -1007,7 +1007,7 @@ class Candidate(object): periods = r['read_history'].keys() periods.sort(key = lambda x: r['read_history'][x]['interval'] ) - print periods + print(periods) def add_running_history(self, history): pass @@ -1723,8 +1723,8 @@ class CandidateList(dict): key_value_split = item.split('=') kvl = len(key_value_split) if kvl < 1 or kvl > 2: - print '#error Bad %s item: %s, format is key=value.'%( - file_name, item) + print('#error Bad %s item: %s, format is key=value.'%( + file_name, item)) if kvl == 1: # assume that entries without a key are the ipv4 address, # perhaps with a dirport @@ -1732,8 +1732,8 @@ class CandidateList(dict): ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':') dirl = len(ipv4_maybe_dirport_split) if dirl < 1 or dirl > 2: - print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%( - file_name, item) + print('#error Bad %s IPv4 item: %s, format is ipv4:port.'%( + file_name, item)) if dirl >= 1: relay_entry['ipv4'] = ipv4_maybe_dirport_split[0] if dirl == 2: @@ -1745,8 +1745,8 @@ class CandidateList(dict): ipv6_orport_split = key_value_split[1].rsplit(':', 1) ipv6l = len(ipv6_orport_split) if ipv6l != 2: - print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( - file_name, item) + print('#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( + file_name, item)) relay_entry['ipv6_addr'] = ipv6_orport_split[0] relay_entry['ipv6_orport'] = ipv6_orport_split[1] relaylist.append(relay_entry) @@ -2419,19 +2419,19 @@ def list_fallbacks(whitelist, exact=False): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays, passing exact to apply_filter_lists(). """ - print "/* type=fallback */" - print ("/* version={} */" - .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) + print("/* type=fallback */") + print("/* version={} */" + .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) now = datetime.datetime.utcnow() timestamp = now.strftime('%Y%m%d%H%M%S') - print ("/* timestamp={} */" - .format(cleanse_c_multiline_comment(timestamp))) + print("/* timestamp={} */" + .format(cleanse_c_multiline_comment(timestamp))) if whitelist['check_existing']: - print "/* source=fallback */" + print("/* source=fallback */") else: - print "/* source=whitelist */" + print("/* source=whitelist */") # end the header with a separator, to make it easier for parsers - print SECTION_SEPARATOR_COMMENT + print(SECTION_SEPARATOR_COMMENT) logging.warning('Downloading and parsing Onionoo data. ' + 'This may take some time.') @@ -2463,8 +2463,8 @@ def list_fallbacks(whitelist, exact=False): # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) - print candidates.summarise_filters(initial_count, excluded_count, - whitelist['check_existing']) + print(candidates.summarise_filters(initial_count, excluded_count, + whitelist['check_existing'])) eligible_count = len(candidates.fallbacks) # calculate the measured bandwidth of each relay, @@ -2474,9 +2474,9 @@ def list_fallbacks(whitelist, exact=False): # print the raw fallback list #for x in candidates.fallbacks: - # print x.fallbackdir_line(True) - # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) + # print(x.fallbackdir_line(True)) + # print(json.dumps(candidates[x]._data, sort_keys=True, indent=4, + # separators=(',', ': '), default=json_util.default)) # impose mandatory conditions here, like one per contact, family, IP # in measured bandwidth order @@ -2515,19 +2515,19 @@ def list_fallbacks(whitelist, exact=False): # output C comments summarising the fallback selection process if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, operator_count, + print(candidates.summarise_fallbacks(eligible_count, operator_count, failed_count, guard_count, target_count, - whitelist['check_existing']) + whitelist['check_existing'])) else: - print '/* No Fallbacks met criteria */' + print('/* No Fallbacks met criteria */') # output C comments specifying the Onionoo data used to create the list for s in fetch_source_list(): - print describe_fetch_source(s) + print(describe_fetch_source(s)) # start the list with a separator, to make it easy for parsers - print SECTION_SEPARATOR_COMMENT + print(SECTION_SEPARATOR_COMMENT) # sort the list differently depending on why we've created it: # if we're outputting the final fallback list, sort by fingerprint @@ -2539,7 +2539,7 @@ def list_fallbacks(whitelist, exact=False): candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD) for x in candidates.fallbacks: - print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks) + print(x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)) if __name__ == "__main__": main()

1 0

[fallback-scripts/master] update: py3: Fix module imports for py3
by teor＠torproject.org 20 Jan '20

20 Jan '20

commit 0f4b9ec4b1bef0effe21da30bb698abc7494d86a Author: teor <teor(a)torproject.org> Date: Mon Dec 2 15:29:20 2019 +1000 update: py3: Fix module imports for py3 Part of 28863. --- updateFallbackDirs.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py index eb7dd62..9a8e400 100755 --- a/updateFallbackDirs.py +++ b/updateFallbackDirs.py @@ -35,28 +35,36 @@ # https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.… # Modifications by teor, 2015 +## Imports: version-independent + import copy import datetime import dateutil.parser import gzip import hashlib import json +import logging import math import os import os.path import re import string -import StringIO import sys -import urllib -import urllib2 from stem.descriptor import DocumentHandler from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS -import logging +## Imports: python 2/3 compatibility + +import six +from six.moves import urllib + +## Logging Configuration + logging.root.name = '' +## Imports: optional + HAVE_IPADDRESS = False try: # python 3 builtin, or install package py2-ipaddress

1 0