commit da968e558e89b3ee9201d6c9cdd339ef48979071 Author: teor (Tim Wilson-Brown) <teor2345@gmail.com> Date: Tue Jan 12 14:15:22 2016 +1100 Fix various issues with fallback directory data handling * support maximum history age in _avg_generic_history() * fix division-by-zero trap in _avg_generic_history() * skip missing (i.e. null/None) intervals in _avg_generic_history() * Python timedelta.total_seconds() function not available in 2.6; replace with equivalent expression * set DEBUG logging level to make relay exclusion reasons visible * move CUTOFF_GUARD test to end in order to expose more exclusion reasons Patch by "starlight", merge modifications by "teor". --- changes/bug18035 | 6 ++++++ scripts/maint/updateFallbackDirs.py | 28 ++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/changes/bug18035 b/changes/bug18035 new file mode 100644 index 0000000..31889f5 --- /dev/null +++ b/changes/bug18035 @@ -0,0 +1,6 @@ + o Minor fixes (fallback directories): + - Work around an issue where OnionOO supplies the entire history, + rather than the requested 120 days. Activate debug logging by + default. Fix other minor calculation and compatibility issues. + Closes ticket #18035. Patch by "starlight", merge fixes by + "teor". Not in any released version of tor. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 8fe234f..29be433 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -27,7 +27,7 @@ import dateutil.parser #from bson import json_util import logging -logging.basicConfig(level=logging.INFO) +logging.basicConfig(level=logging.DEBUG) ## Top-Level Configuration @@ -579,9 +579,15 @@ class Candidate(object): %(p, which)) for v in reversed(h['values']): if (this_ts <= newest): + agt1 = now - this_ts + agt2 = interval + agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600) + * 10**6) / 10**6 + agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600) + * 10**6) / 10**6 generic_history.append( - { 'age': (now - this_ts).total_seconds(), - 'length': interval.total_seconds(), + { 'age': agetmp1, + 'length': agetmp2, 'value': v }) newest = this_ts @@ -599,6 +605,8 @@ class Candidate(object): def _avg_generic_history(generic_history): a = [] for i in generic_history: + if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600): + continue if (i['length'] is not None and i['age'] is not None and i['value'] is not None): @@ -608,7 +616,11 @@ class Candidate(object): sv = math.fsum(map(lambda x: x[0], a)) sw = math.fsum(map(lambda x: x[1], a)) - return sv/sw + if sw == 0.0: + svw = 0.0 + else: + svw = sv/sw + return svw def _add_generic_history(self, history): periods = r['read_history'].keys() @@ -659,10 +671,6 @@ class Candidate(object): logging.debug('%s not a candidate: running avg too low (%lf)', self._fpr, self._running) return False - if self._guard < CUTOFF_GUARD: - logging.debug('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) - return False if self._v2dir < CUTOFF_V2DIR: logging.debug('%s not a candidate: v2dir avg too low (%lf)', self._fpr, self._v2dir) @@ -675,6 +683,10 @@ class Candidate(object): if (not self._data.has_key('recommended_version') or not self._data['recommended_version']): return False + if self._guard < CUTOFF_GUARD: + logging.debug('%s not a candidate: guard avg too low (%lf)', + self._fpr, self._guard) + return False return True def is_in_whitelist(self, relaylist):