May 2016 - tor-commits - lists.torproject.org

[tor/release-0.2.8] Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8
by nickm＠torproject.org 16 May '16

16 May '16

commit 6cc3397e26ff37d6f01471b83e0e5bb1b5aa8eee Merge: e8cc9f3 ab06b79 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon May 16 08:33:17 2016 -0400 Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8 scripts/maint/updateFallbackDirs.py | 57 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.2.8'
by nickm＠torproject.org 16 May '16

16 May '16

commit 5dbb0bdae934f56c34e48b52cecc973ea400805b Merge: 060e0d0 6cc3397 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon May 16 08:33:53 2016 -0400 Merge branch 'maint-0.2.8' scripts/maint/updateFallbackDirs.py | 57 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-)

1 0

[tor/master] Improve logging in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit d41f92b00655197463ed26c0854113ef5c1b6707 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:45:13 2016 +1000 Improve logging in fallback update script Log a notice just before the script is about to perform a potentially time-consuming operation Clarify the warning when py2-ipaddress isn't found Make log levels more consistent No behavioural change (just logging) --- scripts/maint/updateFallbackDirs.py | 45 +++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 7e882a8..c003a1b 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -42,7 +42,7 @@ from stem.descriptor.remote import DescriptorDownloader import logging # INFO tells you why each relay was included or excluded -# WARN tells you about potential misconfigurations +# WARN tells you about potential misconfigurations and relay detail changes logging.basicConfig(level=logging.WARNING) logging.root.name = '' # INFO tells you about each consensus download attempt @@ -58,7 +58,9 @@ try: HAVE_IPADDRESS = True except ImportError: # if this happens, we avoid doing netblock analysis - logging.warning('Unable to import ipaddress, please install py2-ipaddress') + logging.warning('Unable to import ipaddress, please install py2-ipaddress.' + + ' A fallback list will be created, but optional netblock' + + ' analysis will not be performed.') ## Top-Level Configuration @@ -309,11 +311,11 @@ def write_to_file(str, file_name, max_len): with open(file_name, 'w') as f: f.write(str[0:max_len]) except EnvironmentError, error: - logging.warning('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.error('Writing file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) def read_from_file(file_name, max_len): try: @@ -679,8 +681,8 @@ class Candidate(object): this_ts = parse_ts(h['last']) if (len(h['values']) != h['count']): - logging.warn('Inconsistent value count in %s document for %s' - %(p, which)) + logging.warning('Inconsistent value count in %s document for %s' + %(p, which)) for v in reversed(h['values']): if (this_ts <= newest): agt1 = now - this_ts @@ -698,8 +700,8 @@ class Candidate(object): this_ts -= interval if (this_ts + interval != parse_ts(h['first'])): - logging.warn('Inconsistent time information in %s document for %s' - %(p, which)) + logging.warning('Inconsistent time information in %s document for %s' + %(p, which)) #print json.dumps(generic_history, sort_keys=True, # indent=4, separators=(',', ': ')) @@ -999,10 +1001,10 @@ class Candidate(object): if a.version != b.version: raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) if mask_bits > a.max_prefixlen: - logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) mask_bits = a.max_prefixlen if mask_bits < 0: - logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) mask_bits = 0 a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) return b in a_net @@ -1081,7 +1083,7 @@ class Candidate(object): retries = 0, fall_back_to_authority = False).run() except Exception, stem_error: - logging.debug('Unable to retrieve a consensus from %s: %s', nickname, + logging.info('Unable to retrieve a consensus from %s: %s', nickname, stem_error) status = 'error: "%s"' % (stem_error) level = logging.WARNING @@ -1384,7 +1386,7 @@ class CandidateList(dict): elif in_blacklist: # exclude excluded_count += 1 - logging.debug('Excluding %s: in blacklist.', f._fpr) + logging.info('Excluding %s: in blacklist.', f._fpr) else: if INCLUDE_UNLISTED_ENTRIES: # include @@ -1498,10 +1500,10 @@ class CandidateList(dict): if f.has_ipv6(): ip_list.append(f.ipv6addr) elif not CandidateList.allow(f.dirip, ip_list): - logging.debug('Eliminated %s: already have fallback on IPv4 %s'%( + logging.info('Eliminated %s: already have fallback on IPv4 %s'%( f._fpr, f.dirip)) elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list): - logging.debug('Eliminated %s: already have fallback on IPv6 %s'%( + logging.info('Eliminated %s: already have fallback on IPv6 %s'%( f._fpr, f.ipv6addr)) original_count = len(self.fallbacks) self.fallbacks = ip_limit_fallbacks @@ -1521,7 +1523,7 @@ class CandidateList(dict): contact_limit_fallbacks.append(f) contact_list.append(f._data['contact']) else: - logging.debug(('Eliminated %s: already have fallback on ' + + logging.info(('Eliminated %s: already have fallback on ' + 'ContactInfo %s')%(f._fpr, f._data['contact'])) original_count = len(self.fallbacks) self.fallbacks = contact_limit_fallbacks @@ -1544,7 +1546,7 @@ class CandidateList(dict): else: # technically, we already have a fallback with this fallback in its # effective family - logging.debug('Eliminated %s: already have fallback in effective ' + + logging.info('Eliminated %s: already have fallback in effective ' + 'family'%(f._fpr)) original_count = len(self.fallbacks) self.fallbacks = family_limit_fallbacks @@ -1884,6 +1886,8 @@ def list_fallbacks(): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """ + logging.warning('Downloading and parsing Onionoo data. ' + + 'This may take some time.') # find relays that could be fallbacks candidates = CandidateList() candidates.add_relays() @@ -1938,6 +1942,9 @@ def list_fallbacks(): # can serve a consensus, in favour of one that can't # but given it takes up to 15 seconds to check each consensus download, # the risk is worth it + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + logging.warning('Checking consensus download speeds. ' + + 'This may take some time.') failed_count = candidates.perform_download_consensus_checks(max_count) # analyse and log interesting diversity metrics

1 0

[tor/master] Improve comments in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit 46d813922ab891d934fca02d5511b7fbe18ebda8 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:43:55 2016 +1000 Improve comments in fallback update script --- scripts/maint/updateFallbackDirs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index d27c144..7e882a8 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,6 +1,13 @@ #!/usr/bin/python # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc +# +# This script should be run from a stable, reliable network connection, +# with no other network activity (and not over tor). +# If this is not possible, please disable: +# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS +# +# Needs dateutil (and potentially other python packages) # Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis, in PYTHONPATH, or just @@ -1061,7 +1068,7 @@ class Candidate(object): downloader = DescriptorDownloader() start = datetime.datetime.utcnow() # some directory mirrors respond to requests in ways that hang python - # sockets, which is why we long this line here + # sockets, which is why we log this line here logging.info('Initiating consensus download from %s (%s:%d).', nickname, dirip, dirport) # there appears to be about 1 second of overhead when comparing stem's

1 0

[tor/master] Simplify string cleansing in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit ab06b79cba4b8eebec7afee680bc0f72a72aaf4a Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:47:13 2016 +1000 Simplify string cleansing in fallback update script No behavioural change --- scripts/maint/updateFallbackDirs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index c003a1b..464d847 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -219,8 +219,7 @@ def cleanse_unprintable(raw_string): # Remove all unprintable characters cleansed_string = '' for c in raw_string: - if (c in string.ascii_letters or c in string.digits - or c in string.punctuation or c in string.whitespace): + if c in string.printable: cleansed_string += c return cleansed_string

1 0

[tor/master] Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8
by nickm＠torproject.org 16 May '16

16 May '16

commit 6cc3397e26ff37d6f01471b83e0e5bb1b5aa8eee Merge: e8cc9f3 ab06b79 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon May 16 08:33:17 2016 -0400 Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8 scripts/maint/updateFallbackDirs.py | 57 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-)

1 0

[tor/maint-0.2.8] Improve logging in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit d41f92b00655197463ed26c0854113ef5c1b6707 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:45:13 2016 +1000 Improve logging in fallback update script Log a notice just before the script is about to perform a potentially time-consuming operation Clarify the warning when py2-ipaddress isn't found Make log levels more consistent No behavioural change (just logging) --- scripts/maint/updateFallbackDirs.py | 45 +++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 7e882a8..c003a1b 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -42,7 +42,7 @@ from stem.descriptor.remote import DescriptorDownloader import logging # INFO tells you why each relay was included or excluded -# WARN tells you about potential misconfigurations +# WARN tells you about potential misconfigurations and relay detail changes logging.basicConfig(level=logging.WARNING) logging.root.name = '' # INFO tells you about each consensus download attempt @@ -58,7 +58,9 @@ try: HAVE_IPADDRESS = True except ImportError: # if this happens, we avoid doing netblock analysis - logging.warning('Unable to import ipaddress, please install py2-ipaddress') + logging.warning('Unable to import ipaddress, please install py2-ipaddress.' + + ' A fallback list will be created, but optional netblock' + + ' analysis will not be performed.') ## Top-Level Configuration @@ -309,11 +311,11 @@ def write_to_file(str, file_name, max_len): with open(file_name, 'w') as f: f.write(str[0:max_len]) except EnvironmentError, error: - logging.warning('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.error('Writing file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) def read_from_file(file_name, max_len): try: @@ -679,8 +681,8 @@ class Candidate(object): this_ts = parse_ts(h['last']) if (len(h['values']) != h['count']): - logging.warn('Inconsistent value count in %s document for %s' - %(p, which)) + logging.warning('Inconsistent value count in %s document for %s' + %(p, which)) for v in reversed(h['values']): if (this_ts <= newest): agt1 = now - this_ts @@ -698,8 +700,8 @@ class Candidate(object): this_ts -= interval if (this_ts + interval != parse_ts(h['first'])): - logging.warn('Inconsistent time information in %s document for %s' - %(p, which)) + logging.warning('Inconsistent time information in %s document for %s' + %(p, which)) #print json.dumps(generic_history, sort_keys=True, # indent=4, separators=(',', ': ')) @@ -999,10 +1001,10 @@ class Candidate(object): if a.version != b.version: raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) if mask_bits > a.max_prefixlen: - logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) mask_bits = a.max_prefixlen if mask_bits < 0: - logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) mask_bits = 0 a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) return b in a_net @@ -1081,7 +1083,7 @@ class Candidate(object): retries = 0, fall_back_to_authority = False).run() except Exception, stem_error: - logging.debug('Unable to retrieve a consensus from %s: %s', nickname, + logging.info('Unable to retrieve a consensus from %s: %s', nickname, stem_error) status = 'error: "%s"' % (stem_error) level = logging.WARNING @@ -1384,7 +1386,7 @@ class CandidateList(dict): elif in_blacklist: # exclude excluded_count += 1 - logging.debug('Excluding %s: in blacklist.', f._fpr) + logging.info('Excluding %s: in blacklist.', f._fpr) else: if INCLUDE_UNLISTED_ENTRIES: # include @@ -1498,10 +1500,10 @@ class CandidateList(dict): if f.has_ipv6(): ip_list.append(f.ipv6addr) elif not CandidateList.allow(f.dirip, ip_list): - logging.debug('Eliminated %s: already have fallback on IPv4 %s'%( + logging.info('Eliminated %s: already have fallback on IPv4 %s'%( f._fpr, f.dirip)) elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list): - logging.debug('Eliminated %s: already have fallback on IPv6 %s'%( + logging.info('Eliminated %s: already have fallback on IPv6 %s'%( f._fpr, f.ipv6addr)) original_count = len(self.fallbacks) self.fallbacks = ip_limit_fallbacks @@ -1521,7 +1523,7 @@ class CandidateList(dict): contact_limit_fallbacks.append(f) contact_list.append(f._data['contact']) else: - logging.debug(('Eliminated %s: already have fallback on ' + + logging.info(('Eliminated %s: already have fallback on ' + 'ContactInfo %s')%(f._fpr, f._data['contact'])) original_count = len(self.fallbacks) self.fallbacks = contact_limit_fallbacks @@ -1544,7 +1546,7 @@ class CandidateList(dict): else: # technically, we already have a fallback with this fallback in its # effective family - logging.debug('Eliminated %s: already have fallback in effective ' + + logging.info('Eliminated %s: already have fallback in effective ' + 'family'%(f._fpr)) original_count = len(self.fallbacks) self.fallbacks = family_limit_fallbacks @@ -1884,6 +1886,8 @@ def list_fallbacks(): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """ + logging.warning('Downloading and parsing Onionoo data. ' + + 'This may take some time.') # find relays that could be fallbacks candidates = CandidateList() candidates.add_relays() @@ -1938,6 +1942,9 @@ def list_fallbacks(): # can serve a consensus, in favour of one that can't # but given it takes up to 15 seconds to check each consensus download, # the risk is worth it + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + logging.warning('Checking consensus download speeds. ' + + 'This may take some time.') failed_count = candidates.perform_download_consensus_checks(max_count) # analyse and log interesting diversity metrics

1 0

[tor/maint-0.2.8] Simplify string cleansing in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit ab06b79cba4b8eebec7afee680bc0f72a72aaf4a Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:47:13 2016 +1000 Simplify string cleansing in fallback update script No behavioural change --- scripts/maint/updateFallbackDirs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index c003a1b..464d847 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -219,8 +219,7 @@ def cleanse_unprintable(raw_string): # Remove all unprintable characters cleansed_string = '' for c in raw_string: - if (c in string.ascii_letters or c in string.digits - or c in string.punctuation or c in string.whitespace): + if c in string.printable: cleansed_string += c return cleansed_string

1 0

[tor/maint-0.2.8] Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8
by nickm＠torproject.org 16 May '16

16 May '16

commit 6cc3397e26ff37d6f01471b83e0e5bb1b5aa8eee Merge: e8cc9f3 ab06b79 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon May 16 08:33:17 2016 -0400 Merge remote-tracking branch 'teor/fallback-script' into maint-0.2.8 scripts/maint/updateFallbackDirs.py | 57 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-)

1 0

[tor/maint-0.2.8] Improve comments in fallback update script
by nickm＠torproject.org 16 May '16

16 May '16

commit 46d813922ab891d934fca02d5511b7fbe18ebda8 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 29 11:43:55 2016 +1000 Improve comments in fallback update script --- scripts/maint/updateFallbackDirs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index d27c144..7e882a8 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,6 +1,13 @@ #!/usr/bin/python # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc +# +# This script should be run from a stable, reliable network connection, +# with no other network activity (and not over tor). +# If this is not possible, please disable: +# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS +# +# Needs dateutil (and potentially other python packages) # Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis, in PYTHONPATH, or just @@ -1061,7 +1068,7 @@ class Candidate(object): downloader = DescriptorDownloader() start = datetime.datetime.utcnow() # some directory mirrors respond to requests in ways that hang python - # sockets, which is why we long this line here + # sockets, which is why we log this line here logging.info('Initiating consensus download from %s (%s:%d).', nickname, dirip, dirport) # there appears to be about 1 second of overhead when comparing stem's

1 0