- tor-commits - lists.torproject.org

[tor/maint-0.2.8] Select fallbacks by bandwidth rather than consensus weight
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 78ec782f76d4e228761128f24589316276e80473 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 8 23:53:24 2016 +1000 Select fallbacks by bandwidth rather than consensus weight But as advertised bandwidth is controlled by relays, use consensus weight and median weight to bandwidth ratio to approximate measured bandwidth. Includes minor comment changes and parameter reordering. --- scripts/maint/updateFallbackDirs.py | 208 +++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 52 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 3120496..44a7318 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -112,17 +112,13 @@ CUTOFF_GUARD = .95 # .00 means no bad exits PERMITTED_BADEXIT = .00 -# Clients will time out after 30 seconds trying to download a consensus -# So allow fallback directories half that to deliver a consensus -# The exact download times might change based on the network connection -# running this script, but only by a few seconds -# There is also about a second of python overhead -CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 -# If the relay fails a consensus check, retry the download -# This avoids delisting a relay due to transient network conditions -CONSENSUS_DOWNLOAD_RETRY = True +# older entries' weights are adjusted with ALPHA^(age in days) +AGE_ALPHA = 0.99 + +# this factor is used to scale OnionOO entries to [0,1] +ONIONOO_SCALE_ONE = 999. -## List Length Limits +## Fallback Count Limits # The target for these parameters is 20% of the guards in the network # This is around 200 as of October 2015 @@ -130,37 +126,53 @@ _FB_POG = 0.2 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG # We want exactly 100 fallbacks for the initial release -# Limit the number of fallbacks (eliminating lowest by weight) +# This gives us scope to add extra fallbacks to the list as needed +# Limit the number of fallbacks (eliminating lowest by advertised bandwidth) MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100 # Emit a C #error if the number of fallbacks is below MIN_FALLBACK_COUNT = 100 -## Fallback Weight Settings +## Fallback Bandwidth Requirements -# Any fallback with the Exit flag has its consensus weight multipled by this -EXIT_WEIGHT_FRACTION = 1.0 +# Any fallback with the Exit flag has its bandwidth multipled by this fraction +# to make sure we aren't further overloading exits +# (Set to 1.0, because we asked that only lightly loaded exits opt-in, +# and the extra load really isn't that much for large relays.) +EXIT_BANDWIDTH_FRACTION = 1.0 -# If a single fallback's consensus weight is too low, it's pointless adding it +# If a single fallback's bandwidth is too low, it's pointless adding it # We expect fallbacks to handle an extra 30 kilobytes per second of traffic -# Make sure they support a hundred times that -MIN_CONSENSUS_WEIGHT = 30.0 * 100.0 +# Make sure they can support a hundred times the expected extra load +# (Use 102.4 to make it come out nicely in MB/s) +# We convert this to a consensus weight before applying the filter, +# because all the bandwidth amounts are specified by the relay +MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0 + +# Clients will time out after 30 seconds trying to download a consensus +# So allow fallback directories half that to deliver a consensus +# The exact download times might change based on the network connection +# running this script, but only by a few seconds +# There is also about a second of python overhead +CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 +# If the relay fails a consensus check, retry the download +# This avoids delisting a relay due to transient network conditions +CONSENSUS_DOWNLOAD_RETRY = True + +## Fallback Weights for Client Selection # All fallback weights are equal, and set to the value below # Authorities are weighted 1.0 by default # Clients use these weights to select fallbacks and authorities at random # If there are 100 fallbacks and 9 authorities: -# - each fallback is chosen with probability 10/(1000 + 9) ~= 0.99% -# - each authority is chosen with probability 1/(1000 + 9) ~= 0.09% +# - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99% +# - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09% +# A client choosing a bootstrap directory server will choose a fallback for +# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for +# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts. +# (This disregards the bootstrap schedules, where clients start by choosing +# from fallbacks & authoritites, then later choose from only authorities.) FALLBACK_OUTPUT_WEIGHT = 10.0 -## Other Configuration Parameters - -# older entries' weights are adjusted with ALPHA^(age in days) -AGE_ALPHA = 0.99 - -# this factor is used to scale OnionOO entries to [0,1] -ONIONOO_SCALE_ONE = 999. - ## Parsing Functions def parse_ts(t): @@ -448,6 +460,11 @@ class Candidate(object): details['contact'] = None if not 'flags' in details or details['flags'] is None: details['flags'] = [] + if (not 'advertised_bandwidth' in details + or details['advertised_bandwidth'] is None): + # relays without advertised bandwdith have it calculated from their + # consensus weight + details['advertised_bandwidth'] = 0 details['last_changed_address_or_port'] = parse_ts( details['last_changed_address_or_port']) self._data = details @@ -462,10 +479,6 @@ class Candidate(object): self._compute_ipv6addr() if self.ipv6addr is None: logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) - # Reduce the weight of exits to EXIT_WEIGHT_FRACTION * consensus_weight - if self.is_exit(): - exit_weight = self._data['consensus_weight'] * EXIT_WEIGHT_FRACTION - self._data['consensus_weight'] = exit_weight def _stable_sort_or_addresses(self): # replace self._data['or_addresses'] with a stable ordering, @@ -754,11 +767,9 @@ class Candidate(object): logging.info('%s not a candidate: guard avg too low (%lf)', self._fpr, self._guard) return False - if (MIN_CONSENSUS_WEIGHT is not None - and self._data['consensus_weight'] < MIN_CONSENSUS_WEIGHT): - logging.info('%s not a candidate: consensus weight %.0f too low, must ' + - 'be at least %.0f', self._fpr, - self._data['consensus_weight'], MIN_CONSENSUS_WEIGHT) + if (not self._data.has_key('consensus_weight') + or self._data['consensus_weight'] < 1): + logging.info('%s not a candidate: consensus weight invalid', self._fpr) return False return True @@ -889,6 +900,30 @@ class Candidate(object): ipv6 if has_ipv6 else value) return False + def cw_to_bw_factor(self): + # any relays with a missing or zero consensus weight are not candidates + # any relays with a missing advertised bandwidth have it set to zero + return self._data['advertised_bandwidth'] / self._data['consensus_weight'] + + # since advertised_bandwidth is reported by the relay, it can be gamed + # to avoid this, use the median consensus weight to bandwidth factor to + # estimate this relay's measured bandwidth, and make that the upper limit + def measured_bandwidth(self, median_cw_to_bw_factor): + cw_to_bw= median_cw_to_bw_factor + # Reduce exit bandwidth to make sure we're not overloading them + if self.is_exit(): + cw_to_bw *= EXIT_BANDWIDTH_FRACTION + measured_bandwidth = self._data['consensus_weight'] * cw_to_bw + if self._data['advertised_bandwidth'] != 0: + # limit advertised bandwidth (if available) to measured bandwidth + return min(measured_bandwidth, self._data['advertised_bandwidth']) + else: + return measured_bandwidth + + def set_measured_bandwidth(self, median_cw_to_bw_factor): + self._data['measured_bandwidth'] = self.measured_bandwidth( + median_cw_to_bw_factor) + def is_exit(self): return 'Exit' in self._data['flags'] @@ -1056,8 +1091,8 @@ class CandidateList(dict): logging.debug('Loading details document.') d = fetch('details', fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + - 'consensus_weight,or_addresses,dir_address,' + - 'recommended_version,flags')) + 'consensus_weight,advertised_bandwidth,or_addresses,' + + 'dir_address,recommended_version,flags')) logging.debug('Loading details document done.') if not 'relays' in d: raise Exception("No relays found in document.") @@ -1083,15 +1118,24 @@ class CandidateList(dict): guard_count += 1 return guard_count - # Find fallbacks that fit the uptime, stability, and flags criteria + # Find fallbacks that fit the uptime, stability, and flags criteria, + # and make an array of them in self.fallbacks def compute_fallbacks(self): self.fallbacks = map(lambda x: self[x], - sorted( - filter(lambda x: self[x].is_candidate(), - self.keys()), - key=lambda x: self[x]._data['consensus_weight'], + filter(lambda x: self[x].is_candidate(), + self.keys())) + + # sort fallbacks by their consensus weight to advertised bandwidth factor, + # lowest to highest + # used to find the median cw_to_bw_factor() + def sort_fallbacks_by_cw_to_bw_factor(self): + self.fallbacks.sort(key=lambda x: self[x].cw_to_bw_factor()) + + # sort fallbacks by their measured bandwidth, highest to lowest + # calculate_measured_bandwidth before calling this + def sort_fallbacks_by_measured_bandwidth(self): + self.fallbacks.sort(key=lambda x: self[x].self._data['measured_bandwidth'], reverse=True) - ) @staticmethod def load_relaylist(file_name): @@ -1194,13 +1238,64 @@ class CandidateList(dict): return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( excluded_count, initial_count) - def fallback_min_weight(self): + # calculate each fallback's measured bandwidth based on the median + # consensus weight to advertised bandwdith ratio + def calculate_measured_bandwidth(self): + self.sort_fallbacks_by_cw_to_bw_factor() + median_fallback = self.fallback_median(True) + median_cw_to_bw_factor = median_fallback.cw_to_bw_factor() + for f in self.fallbacks: + f.set_measured_bandwidth(median_cw_to_bw_factor) + + # remove relays with low measured bandwidth from the fallback list + # calculate_measured_bandwidth for each relay before calling this + def remove_low_bandwidth_relays(self): + if MIN_BANDWIDTH is None: + return + above_min_bw_fallbacks = [] + for f in self.fallbacks: + if f._data['measured_bandwidth'] >= MIN_BANDWIDTH: + above_min_bw_fallbacks.append(f) + else: + # the bandwidth we log here is limited by the relay's consensus weight + # as well as its adverttised bandwidth. See set_measured_bandwidth + # for details + logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' + + 'be at least %.1fMB/s', f._fpr, + f._data['measured_bandwidth']/(1024.0*1024.0), + MIN_BANDWIDTH/(1024.0*1024.0)) + self.fallbacks = above_min_bw_fallbacks + + # the minimum fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_min(self): if len(self.fallbacks) > 0: return self.fallbacks[-1] else: return None - def fallback_max_weight(self): + # the median fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_median(self, require_advertised_bandwidth): + # use the low-median when there are an evan number of fallbacks, + # for consistency with the bandwidth authorities + if len(self.fallbacks) > 0: + median_position = (len(self.fallbacks) - 1) / 2 + if not require_advertised_bandwidth: + return self.fallbacks[median_position] + # if we need advertised_bandwidth but this relay doesn't have it, + # move to a fallback with greater consensus weight until we find one + while not self.fallbacks[median_position]._data['advertised_bandwidth']: + median_position += 1 + if median_position >= len(self.fallbacks): + return None + return self.fallbacks[median_position] + else: + return None + + # the maximum fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_max(self): if len(self.fallbacks) > 0: return self.fallbacks[0] else: @@ -1211,7 +1306,7 @@ class CandidateList(dict): # Report: # whether we checked consensus download times # the number of fallback directories (and limits/exclusions, if relevant) - # min & max fallback weights + # min & max fallback bandwidths # #error if below minimum count if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: s = '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( @@ -1243,11 +1338,12 @@ class CandidateList(dict): s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( eligible_count - fallback_count) s += '\n' - min_fb = self.fallback_min_weight() - min_weight = min_fb._data['consensus_weight'] - max_fb = self.fallback_max_weight() - max_weight = max_fb._data['consensus_weight'] - s += 'Consensus Weight Range: %d - %d'%(min_weight, max_weight) + min_fb = self.fallback_min() + min_bw = min_fb._data['measured_bandwidth'] + max_fb = self.fallback_max() + max_bw = max_fb._data['measured_bandwidth'] + s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0), + max_bw/(1024.0*1024.0)) s += '\n' s += '*/' if fallback_count < MIN_FALLBACK_COUNT: @@ -1293,6 +1389,14 @@ def list_fallbacks(): print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) + # calculate the measured bandwidth of each relay, + # then remove low-bandwidth relays + candidates.calculate_measured_bandwidth() + candidates.remove_low_bandwidth_relays() + # make sure the list is sorted by bandwidth when we output it + # so that we include the active fallbacks with the greatest bandwidth + candidates.sort_fallbacks_by_measured_bandwidth() + # print the raw fallback list #for x in candidates.fallbacks: # print x.fallbackdir_line(True)

1 0

[tor/maint-0.2.8] Remove fallback directory weights
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 205a641e1dad6db967825dcad01bc59c6909b395 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Mar 31 12:47:42 2016 +1100 Remove fallback directory weights Give each fallback a set weight of 10.0 for client selection. Fallbacks must have at least 3000 consensus weight. This is (nominally) 100 times the expected extra load of 20 kilobytes per second (50 GB per month). Fixes issue #17905. --- scripts/maint/updateFallbackDirs.py | 221 +++++++----------------------------- 1 file changed, 40 insertions(+), 181 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 4860700..124f85f 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -120,39 +120,31 @@ CONSENSUS_DOWNLOAD_RETRY = True # The target for these parameters is 20% of the guards in the network # This is around 200 as of October 2015 -FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else 0.2 +_FB_POG = 0.2 +FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG # Limit the number of fallbacks (eliminating lowest by weight) MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 500 # Emit a C #error if the number of fallbacks is below -MIN_FALLBACK_COUNT = 50 +MIN_FALLBACK_COUNT = 100 ## Fallback Weight Settings -# Any fallback with the Exit flag has its weight multipled by this fraction +# Any fallback with the Exit flag has its consensus weight multipled by this EXIT_WEIGHT_FRACTION = 1.0 -# If True, emit a C #error if we can't satisfy various constraints -# If False, emit a C comment instead -STRICT_FALLBACK_WEIGHTS = False - -# Limit the proportional weight -# If a single fallback's weight is too high, it will see too many clients -# We reweight using a lower threshold to provide some leeway for: -# * elimination of low weight relays -# * consensus weight changes -# * fallback directory losses over time -# A relay weighted at 1 in 10 fallbacks will see about 10% of clients that -# use the fallback directories. (The 9 directory authorities see a similar -# proportion of clients.) -TARGET_MAX_WEIGHT_FRACTION = 1/10.0 -REWEIGHTING_FUDGE_FACTOR = 0.8 -MAX_WEIGHT_FRACTION = TARGET_MAX_WEIGHT_FRACTION * REWEIGHTING_FUDGE_FACTOR -# If a single fallback's weight is too low, it's pointless adding it. -# (Final weights may be slightly higher than this, due to low weight relays -# being excluded.) -# A relay weighted at 1 in 1000 fallbacks will see about 0.1% of clients. -MIN_WEIGHT_FRACTION = 0.0 if OUTPUT_CANDIDATES else 1/1000.0 +# If a single fallback's consensus weight is too low, it's pointless adding it +# We expect fallbacks to handle an extra 30 kilobytes per second of traffic +# Make sure they support a hundred times that +MIN_CONSENSUS_WEIGHT = 30.0 * 100.0 + +# All fallback weights are equal, and set to the value below +# Authorities are weighted 1.0 by default +# Clients use these weights to select fallbacks and authorities at random +# If there are 100 fallbacks and 9 authorities: +# - each fallback is chosen with probability 10/(1000 + 9) ~= 0.99% +# - each authority is chosen with probability 1/(1000 + 9) ~= 0.09% +FALLBACK_OUTPUT_WEIGHT = 10.0 ## Other Configuration Parameters @@ -465,9 +457,7 @@ class Candidate(object): logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) # Reduce the weight of exits to EXIT_WEIGHT_FRACTION * consensus_weight if self.is_exit(): - current_weight = self._data['consensus_weight'] - exit_weight = current_weight * EXIT_WEIGHT_FRACTION - self._data['original_consensus_weight'] = current_weight + exit_weight = self._data['consensus_weight'] * EXIT_WEIGHT_FRACTION self._data['consensus_weight'] = exit_weight def _stable_sort_or_addresses(self): @@ -757,6 +747,12 @@ class Candidate(object): logging.info('%s not a candidate: guard avg too low (%lf)', self._fpr, self._guard) return False + if (MIN_CONSENSUS_WEIGHT is not None + and self._data['consensus_weight'] < MIN_CONSENSUS_WEIGHT): + logging.info('%s not a candidate: consensus weight %.0f too low, must ' + + 'be at least %.0f', self._fpr, + self._data['consensus_weight'], MIN_CONSENSUS_WEIGHT) + return False return True def is_in_whitelist(self, relaylist): @@ -895,20 +891,6 @@ class Candidate(object): def is_running(self): return 'Running' in self._data['flags'] - def fallback_weight_fraction(self, total_weight): - return float(self._data['consensus_weight']) / total_weight - - # return the original consensus weight, if it exists, - # or, if not, return the consensus weight - def original_consensus_weight(self): - if self._data.has_key('original_consensus_weight'): - return self._data['original_consensus_weight'] - else: - return self._data['consensus_weight'] - - def original_fallback_weight_fraction(self, total_weight): - return float(self.original_consensus_weight()) / total_weight - @staticmethod def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time): download_failed = False @@ -976,17 +958,15 @@ class Candidate(object): CONSENSUS_DOWNLOAD_SPEED_MAX) return ((not ipv4_failed) and (not ipv6_failed)) - def fallbackdir_line(self, total_weight, original_total_weight, dl_speed_ok): + def fallbackdir_line(self, dl_speed_ok): # /* # nickname # flags - # weight / total (percentage) - # [original weight / original total (original percentage)] # [contact] # */ # "address:dirport orport=port id=fingerprint" # "[ipv6=addr:orport]" - # "weight=num", + # "weight=FALLBACK_OUTPUT_WEIGHT", # # Multiline C comment s = '/*' @@ -996,19 +976,6 @@ class Candidate(object): s += 'Flags: ' s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags']))) s += '\n' - weight = self._data['consensus_weight'] - percent_weight = self.fallback_weight_fraction(total_weight)*100 - s += 'Fallback Weight: %d / %d (%.3f%%)'%(weight, total_weight, - percent_weight) - s += '\n' - o_weight = self.original_consensus_weight() - if o_weight != weight: - o_percent_weight = self.original_fallback_weight_fraction( - original_total_weight)*100 - s += 'Consensus Weight: %d / %d (%.3f%%)'%(o_weight, - original_total_weight, - o_percent_weight) - s += '\n' if self._data['contact'] is not None: s += cleanse_c_multiline_comment(self._data['contact']) s += '\n' @@ -1030,7 +997,7 @@ class Candidate(object): s += '" ipv6=%s:%s"'%( cleanse_c_string(self.ipv6addr), cleanse_c_string(self.ipv6orport)) s += '\n' - s += '" weight=%d",'%(weight) + s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT) if not dl_speed_ok: s += '\n' s += '*/' @@ -1205,48 +1172,11 @@ class CandidateList(dict): # Remove any fallbacks in excess of MAX_FALLBACK_COUNT, # starting with the lowest-weighted fallbacks - # total_weight should be recalculated after calling this + # this changes total weight def exclude_excess_fallbacks(self): if MAX_FALLBACK_COUNT is not None: self.fallbacks = self.fallbacks[:MAX_FALLBACK_COUNT] - # Clamp the weight of all fallbacks to MAX_WEIGHT_FRACTION * total_weight - # fallbacks are kept sorted, but since excessive weights are reduced to - # the maximum acceptable weight, these relays end up with equal weights - def clamp_high_weight_fallbacks(self, total_weight): - if MAX_WEIGHT_FRACTION * len(self.fallbacks) < 1.0: - error_str = 'Max Fallback Weight %.3f%% is unachievable'%( - MAX_WEIGHT_FRACTION) - error_str += ' with Current Fallback Count %d.'%(len(self.fallbacks)) - if STRICT_FALLBACK_WEIGHTS: - print '#error ' + error_str - else: - print '/* ' + error_str + ' */' - relays_clamped = 0 - max_acceptable_weight = total_weight * MAX_WEIGHT_FRACTION - for f in self.fallbacks: - frac_weight = f.fallback_weight_fraction(total_weight) - if frac_weight > MAX_WEIGHT_FRACTION: - relays_clamped += 1 - current_weight = f._data['consensus_weight'] - # if we already have an original weight, keep it - if (not f._data.has_key('original_consensus_weight') - or f._data['original_consensus_weight'] == current_weight): - f._data['original_consensus_weight'] = current_weight - f._data['consensus_weight'] = max_acceptable_weight - return relays_clamped - - # Remove any fallbacks with weights lower than MIN_WEIGHT_FRACTION - # total_weight should be recalculated after calling this - def exclude_low_weight_fallbacks(self, total_weight): - self.fallbacks = filter( - lambda x: - x.fallback_weight_fraction(total_weight) >= MIN_WEIGHT_FRACTION, - self.fallbacks) - - def fallback_weight_total(self): - return sum(f._data['consensus_weight'] for f in self.fallbacks) - def fallback_min_weight(self): if len(self.fallbacks) > 0: return self.fallbacks[-1] @@ -1259,14 +1189,12 @@ class CandidateList(dict): else: return None - def summarise_fallbacks(self, eligible_count, eligible_weight, - relays_clamped, clamped_weight, - guard_count, target_count, max_count): + def summarise_fallbacks(self, eligible_count, guard_count, target_count, + max_count): # Report: # the number of fallback directories (with min & max limits); # #error if below minimum count - # the total weight, min & max fallback proportions - # #error if outside max weight proportion + # min & max fallback weights # Multiline C comment with #error if things go bad s = '/*' s += '\n' @@ -1279,7 +1207,7 @@ class CandidateList(dict): else: fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d, Usable %d, Target %d%s'%( + s += 'Final Count: %d (Eligible %d, Target %d%s'%( min(max_count, fallback_count), eligible_count, fallback_count, @@ -1299,51 +1227,19 @@ class CandidateList(dict): s += '\n' s += '/*' s += '\n' - total_weight = self.fallback_weight_total() min_fb = self.fallback_min_weight() min_weight = min_fb._data['consensus_weight'] - min_percent = min_fb.fallback_weight_fraction(total_weight)*100.0 max_fb = self.fallback_max_weight() max_weight = max_fb._data['consensus_weight'] - max_frac = max_fb.fallback_weight_fraction(total_weight) - max_percent = max_frac*100.0 - s += 'Final Weight: %d (Eligible %d)'%(total_weight, eligible_weight) - s += '\n' - s += 'Max Weight: %d (%.3f%%) (Clamped to %.3f%%)'%( - max_weight, - max_percent, - TARGET_MAX_WEIGHT_FRACTION*100) + s += 'Max Weight: %d'%(max_weight) s += '\n' - s += 'Min Weight: %d (%.3f%%) (Clamped to %.3f%%)'%( - min_weight, - min_percent, - MIN_WEIGHT_FRACTION*100) + s += 'Min Weight: %d'%(min_weight) s += '\n' if eligible_count != fallback_count: - s += 'Excluded: %d (Clamped, Below Target, or Low Weight)'%( + s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( eligible_count - fallback_count) s += '\n' - if relays_clamped > 0: - s += 'Clamped: %d (%.3f%%) Excess Weight, '%( - clamped_weight, - (100.0 * clamped_weight) / total_weight) - s += '%d High Weight Fallbacks (%.1f%%)'%( - relays_clamped, - (100.0 * relays_clamped) / fallback_count) - s += '\n' s += '*/' - if max_frac > TARGET_MAX_WEIGHT_FRACTION: - s += '\n' - # We must restrict the maximum fallback weight, so an adversary - # at or near the fallback doesn't see too many clients - error_str = 'Max Fallback Weight %.3f%% is too high. '%(max_frac*100) - error_str += 'Must be at most %.3f%% for client anonymity.'%( - TARGET_MAX_WEIGHT_FRACTION*100) - if STRICT_FALLBACK_WEIGHTS: - s += '#error ' + error_str - else: - s += '/* ' + error_str + ' */' - s += '\n' if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', @@ -1386,54 +1282,17 @@ def list_fallbacks(): excluded_count = candidates.apply_filter_lists() print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) - eligible_weight = candidates.fallback_weight_total() # print the raw fallback list - #total_weight = candidates.fallback_weight_total() #for x in candidates.fallbacks: - # print x.fallbackdir_line(total_weight, total_weight) + # print x.fallbackdir_line(True) - # When candidates are excluded, total_weight decreases, and - # the proportional weight of other candidates increases. + # exclude low-weight fallbacks if we have more than we want candidates.exclude_excess_fallbacks() - total_weight = candidates.fallback_weight_total() - - # When candidates are reweighted, total_weight decreases, and - # the proportional weight of other candidates increases. - # Previously low-weight candidates might obtain sufficient proportional - # weights to be included. - # Save the weight at which we reweighted fallbacks for the summary. - pre_clamp_total_weight = total_weight - relays_clamped = candidates.clamp_high_weight_fallbacks(total_weight) - - # When candidates are excluded, total_weight decreases, and - # the proportional weight of other candidates increases. - # No new low weight candidates will be created during exclusions. - # However, high weight candidates may increase over the maximum proportion. - # This should not be an issue, except in pathological cases. - candidates.exclude_low_weight_fallbacks(total_weight) - total_weight = candidates.fallback_weight_total() - - # check we haven't exceeded TARGET_MAX_WEIGHT_FRACTION - # since reweighting preserves the orginal sort order, - # the maximum weights will be at the head of the list - if len(candidates.fallbacks) > 0: - max_weight_fb = candidates.fallback_max_weight() - max_weight = max_weight_fb.fallback_weight_fraction(total_weight) - if max_weight > TARGET_MAX_WEIGHT_FRACTION: - error_str = 'Maximum fallback weight: %.3f%% exceeds target %.3f%%. '%( - max_weight*100.0, - TARGET_MAX_WEIGHT_FRACTION*100.0) - error_str += 'Try decreasing REWEIGHTING_FUDGE_FACTOR.' - if STRICT_FALLBACK_WEIGHTS: - print '#error ' + error_str - else: - print '/* ' + error_str + ' */' - print candidates.summarise_fallbacks(eligible_count, eligible_weight, - relays_clamped, - pre_clamp_total_weight - total_weight, - guard_count, target_count, max_count) + if len(candidates.fallbacks) > 0: + print candidates.summarise_fallbacks(eligible_count, guard_count, + target_count, max_count) else: print '/* No Fallbacks met criteria */' @@ -1442,7 +1301,7 @@ def list_fallbacks(): for x in candidates.fallbacks[:max_count]: dl_speed_ok = x.fallback_consensus_dl_check() - print x.fallbackdir_line(total_weight, pre_clamp_total_weight, dl_speed_ok) + print x.fallbackdir_line(dl_speed_ok) #print json.dumps(candidates[x]._data, sort_keys=True, indent=4, # separators=(',', ': '), default=json_util.default)

1 0

[tor/maint-0.2.8] Only count active fallbacks in updateFallbackDirs.py
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit e115523c98ce56973330c45c92939328f92642f7 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Apr 7 12:55:39 2016 +1000 Only count active fallbacks in updateFallbackDirs.py Previously, we would cut the list down to 100 fallbacks, then check if they could serve a consensus, and comment them out if they couldn't. This would leave us with fewer than 100 active fallbacks. Now, we stop when there are 100 active fallbacks. Also count fallbacks with identical contact info. Also fix minor logging issues. --- scripts/maint/updateFallbackDirs.py | 129 ++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 124f85f..3120496 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -26,6 +26,7 @@ import hashlib import dateutil.parser # bson_lazy provides bson #from bson import json_util +import copy from stem.descriptor.remote import DescriptorDownloader @@ -52,6 +53,11 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False +# Output matching ContactInfo in fallbacks list or the blacklist? +# Useful if you're trying to contact operators +CONTACT_COUNT = True if OUTPUT_CANDIDATES else False +CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False + ## OnionOO Settings ONIONOO = 'https://onionoo.torproject.org/' @@ -123,8 +129,9 @@ CONSENSUS_DOWNLOAD_RETRY = True _FB_POG = 0.2 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG +# We want exactly 100 fallbacks for the initial release # Limit the number of fallbacks (eliminating lowest by weight) -MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 500 +MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100 # Emit a C #error if the number of fallbacks is below MIN_FALLBACK_COUNT = 100 @@ -958,11 +965,12 @@ class Candidate(object): CONSENSUS_DOWNLOAD_SPEED_MAX) return ((not ipv4_failed) and (not ipv6_failed)) - def fallbackdir_line(self, dl_speed_ok): + def fallbackdir_line(self, dl_speed_ok, fallbacks, prefilter_fallbacks): # /* # nickname # flags # [contact] + # [identical contact counts] # */ # "address:dirport orport=port id=fingerprint" # "[ipv6=addr:orport]" @@ -978,6 +986,22 @@ class Candidate(object): s += '\n' if self._data['contact'] is not None: s += cleanse_c_multiline_comment(self._data['contact']) + if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT: + fallback_count = len([f for f in fallbacks + if f._data['contact'] == self._data['contact']]) + if fallback_count > 1: + s += '\n' + s += '%d identical contacts listed' % (fallback_count) + if CONTACT_BLACKLIST_COUNT: + prefilter_count = len([f for f in prefilter_fallbacks + if f._data['contact'] == self._data['contact']]) + filter_count = prefilter_count - fallback_count + if filter_count > 0: + if fallback_count > 1: + s += ' ' + else: + s += '\n' + s += '%d blacklisted' % (filter_count) s += '\n' s += '*/' s += '\n' @@ -1170,13 +1194,6 @@ class CandidateList(dict): return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( excluded_count, initial_count) - # Remove any fallbacks in excess of MAX_FALLBACK_COUNT, - # starting with the lowest-weighted fallbacks - # this changes total weight - def exclude_excess_fallbacks(self): - if MAX_FALLBACK_COUNT is not None: - self.fallbacks = self.fallbacks[:MAX_FALLBACK_COUNT] - def fallback_min_weight(self): if len(self.fallbacks) > 0: return self.fallbacks[-1] @@ -1192,63 +1209,55 @@ class CandidateList(dict): def summarise_fallbacks(self, eligible_count, guard_count, target_count, max_count): # Report: - # the number of fallback directories (with min & max limits); - # #error if below minimum count + # whether we checked consensus download times + # the number of fallback directories (and limits/exclusions, if relevant) # min & max fallback weights - # Multiline C comment with #error if things go bad - s = '/*' + # #error if below minimum count + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + s = '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( + 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', + ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS + and PERFORM_IPV6_DIRPORT_CHECKS) else '', + 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', + CONSENSUS_DOWNLOAD_SPEED_MAX) + else: + s = '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' s += '\n' - s += 'Fallback Directory Summary' + # Multiline C comment with #error if things go bad + s += '/*' s += '\n' # Integers don't need escaping in C comments fallback_count = len(self.fallbacks) if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = ' (none)' + fallback_proportion = '' else: - fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d, Target %d%s'%( - min(max_count, fallback_count), - eligible_count, - fallback_count, - fallback_proportion) + fallback_proportion = ', Target %d (%d * %f)'%(target_count, guard_count, + FALLBACK_PROPORTION_OF_GUARDS) + s += 'Final Count: %d (Eligible %d%s'%(fallback_count, + eligible_count, + fallback_proportion) if MAX_FALLBACK_COUNT is not None: s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) s += ')\n' - if fallback_count < MIN_FALLBACK_COUNT: - s += '*/' - s += '\n' - # We must have a minimum number of fallbacks so they are always - # reachable, and are in diverse locations - s += '#error Fallback Count %d is too low. '%(fallback_count) - s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to the whitelist, ' - s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' - s += '\n' - s += '/*' + if eligible_count != fallback_count: + s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( + eligible_count - fallback_count) s += '\n' min_fb = self.fallback_min_weight() min_weight = min_fb._data['consensus_weight'] max_fb = self.fallback_max_weight() max_weight = max_fb._data['consensus_weight'] - s += 'Max Weight: %d'%(max_weight) + s += 'Consensus Weight Range: %d - %d'%(min_weight, max_weight) s += '\n' - s += 'Min Weight: %d'%(min_weight) - s += '\n' - if eligible_count != fallback_count: - s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( - eligible_count - fallback_count) - s += '\n' s += '*/' - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( - 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', - ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS - and PERFORM_IPV6_DIRPORT_CHECKS) else '', - 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', - CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' + if fallback_count < MIN_FALLBACK_COUNT: + # We must have a minimum number of fallbacks so they are always + # reachable, and are in diverse locations + s += '\n' + s += '#error Fallback Count %d is too low. '%(fallback_count) + s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) + s += 'Try adding entries to the whitelist, ' + s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' return s ## Main Function @@ -1271,11 +1280,12 @@ def list_fallbacks(): # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count) # - the maximum fallback count (MAX_FALLBACK_COUNT) if MAX_FALLBACK_COUNT is None: - max_count = guard_count + max_count = target_count else: max_count = min(target_count, MAX_FALLBACK_COUNT) candidates.compute_fallbacks() + prefilter_fallbacks = copy.copy(candidates.fallbacks) # filter with the whitelist and blacklist initial_count = len(candidates.fallbacks) @@ -1286,9 +1296,8 @@ def list_fallbacks(): # print the raw fallback list #for x in candidates.fallbacks: # print x.fallbackdir_line(True) - - # exclude low-weight fallbacks if we have more than we want - candidates.exclude_excess_fallbacks() + # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, + # separators=(',', ': '), default=json_util.default) if len(candidates.fallbacks) > 0: print candidates.summarise_fallbacks(eligible_count, guard_count, @@ -1299,11 +1308,17 @@ def list_fallbacks(): for s in fetch_source_list(): print describe_fetch_source(s) - for x in candidates.fallbacks[:max_count]: + active_count = 0 + for x in candidates.fallbacks: dl_speed_ok = x.fallback_consensus_dl_check() - print x.fallbackdir_line(dl_speed_ok) - #print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) + print x.fallbackdir_line(dl_speed_ok, candidates.fallbacks, + prefilter_fallbacks) + if dl_speed_ok: + # this fallback is included in the list + active_count += 1 + if active_count >= max_count: + # we have enough fallbacks + break if __name__ == "__main__": list_fallbacks()

1 0

[tor/maint-0.2.8] Improve fallback selection and output
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 7e1b8ae79cc7da3b4a0989d6f9ba95aa5cd9b73b Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Wed Mar 30 12:41:57 2016 +1100 Improve fallback selection and output Improve the download test: * Allow IPv4 DirPort checks to be turned off. * Add a timeout to stem's consensus download. * Actually check for download errors, rather than ignoring them. * Simplify the timeout and download error checking logic. Tweak whitelist/blacklist checks to be more robust. Improve logging, make it warn by default. Cleanse fallback comments more thoroughly: * non-printables (yes, ContactInfo can have these) * // comments (don't rely on newlines to prevent // */ escapes) --- scripts/maint/updateFallbackDirs.py | 289 +++++++++++++++++++++++++----------- 1 file changed, 201 insertions(+), 88 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index d110335..4860700 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,6 +1,7 @@ #!/usr/bin/python # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc +# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . # # Then read the generated list to ensure no-one slipped anything funny into # their name or contactinfo @@ -29,17 +30,28 @@ import dateutil.parser from stem.descriptor.remote import DescriptorDownloader import logging -logging.basicConfig(level=logging.DEBUG) +# INFO tells you why each relay was included or excluded +# WARN tells you about potential misconfigurations +logging.basicConfig(level=logging.WARNING) ## Top-Level Configuration -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -PERFORM_IPV6_DIRPORT_CHECKS = False - # Output all candidate fallbacks, or only output selected fallbacks? OUTPUT_CANDIDATES = False +# Perform DirPort checks over IPv4? +# Change this to False if IPv4 doesn't work for you, or if you don't want to +# download a consensus for each fallback +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True + +# Perform DirPort checks over IPv6? +# If you know IPv6 works for you, set this to True +# This will exclude IPv6 relays without an IPv6 DirPort configured +# So it's best left at False until #18394 is implemented +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False + ## OnionOO Settings ONIONOO = 'https://onionoo.torproject.org/' @@ -81,7 +93,7 @@ MAX_LIST_FILE_SIZE = 1024 * 1024 # Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted # This causes OnionOO to (correctly) reset its stability timer -# This issue is fixed in 0.2.7.7 and master. +# This issue will be fixed in 0.2.7.7 and 0.2.8.2 # Until then, the CUTOFFs below ensure a decent level of stability. ADDRESS_AND_PORT_STABLE_DAYS = 7 # What time-weighted-fraction of these flags must FallbackDirs @@ -157,36 +169,52 @@ def parse_ts(t): def remove_bad_chars(raw_string, bad_char_list): # Remove each character in the bad_char_list - escaped_string = raw_string + cleansed_string = raw_string for c in bad_char_list: - escaped_string = escaped_string.replace(c, '') - return escaped_string + cleansed_string = cleansed_string.replace(c, '') + return cleansed_string + +def cleanse_unprintable(raw_string): + # Remove all unprintable characters + cleansed_string = '' + for c in raw_string: + if (c in string.ascii_letters or c in string.digits + or c in string.punctuation or c in string.whitespace): + cleansed_string += c + return cleansed_string def cleanse_whitespace(raw_string): # Replace all whitespace characters with a space - escaped_string = raw_string + cleansed_string = raw_string for c in string.whitespace: - escaped_string = escaped_string.replace(c, ' ') - return escaped_string + cleansed_string = cleansed_string.replace(c, ' ') + return cleansed_string def cleanse_c_multiline_comment(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious / unanticipated string from breaking out # of a C-style multiline comment - # This removes '/*' and '*/' - # To deal with '//', the end comment must be on its own line - bad_char_list = '*' + # This removes '/*' and '*/' and '//' + bad_char_list = '*/' # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of comments # There isn't much we can do to cover every possible case # But comment-based directives are typically only advisory - return escaped_string + return cleansed_string def cleanse_c_string(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious address/fingerprint string from breaking out # of a C-style string bad_char_list = '"' @@ -195,13 +223,11 @@ def cleanse_c_string(raw_string): # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of strings # There isn't much we can do to cover every possible case # But this typically only results in changes to the string data - return escaped_string + return cleansed_string ## OnionOO Source Functions @@ -244,11 +270,11 @@ def write_to_file(str, file_name, max_len): with open(file_name, 'w') as f: f.write(str[0:max_len]) except EnvironmentError, error: - logging.debug('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.warning('Writing file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) def read_from_file(file_name, max_len): try: @@ -256,11 +282,11 @@ def read_from_file(file_name, max_len): with open(file_name, 'r') as f: return f.read(max_len) except EnvironmentError, error: - logging.debug('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.info('Loading file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) return None def load_possibly_compressed_response_json(response): @@ -699,30 +725,37 @@ class Candidate(object): self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE def is_candidate(self): + must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS + or PERFORM_IPV6_DIRPORT_CHECKS) + if (must_be_running_now and not self.is_running()): + logging.info('%s not a candidate: not running now, unable to check ' + + 'DirPort consensus download', self._fpr) + return False if (self._data['last_changed_address_or_port'] > self.CUTOFF_ADDRESS_AND_PORT_STABLE): - logging.debug('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) + logging.info('%s not a candidate: changed address/port recently (%s)', + self._fpr, self._data['last_changed_address_or_port']) return False if self._running < CUTOFF_RUNNING: - logging.debug('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) + logging.info('%s not a candidate: running avg too low (%lf)', + self._fpr, self._running) return False if self._v2dir < CUTOFF_V2DIR: - logging.debug('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) + logging.info('%s not a candidate: v2dir avg too low (%lf)', + self._fpr, self._v2dir) return False if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - logging.debug('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) + logging.info('%s not a candidate: badexit avg too high (%lf)', + self._fpr, self._badexit) return False # if the relay doesn't report a version, also exclude the relay if (not self._data.has_key('recommended_version') or not self._data['recommended_version']): + logging.info('%s not a candidate: version not recommended', self._fpr) return False if self._guard < CUTOFF_GUARD: - logging.debug('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) + logging.info('%s not a candidate: guard avg too low (%lf)', + self._fpr, self._guard) return False return True @@ -736,24 +769,48 @@ class Candidate(object): If the fallback has an ipv6 key, the whitelist line must also have it, and vice versa, otherwise they don't match. """ for entry in relaylist: + if entry['id'] != self._fpr: + # can't log here, every relay's fingerprint is compared to the entry + continue if entry['ipv4'] != self.dirip: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'IPv4 (%s) does not match entry IPv4 (%s)', + self._fpr, self.dirip, entry['ipv4']) continue if int(entry['dirport']) != self.dirport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'DirPort (%d) does not match entry DirPort (%d)', + self._fpr, self.dirport, int(entry['dirport'])) continue if int(entry['orport']) != self.orport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'ORPort (%d) does not match entry ORPort (%d)', + self._fpr, self.orport, int(entry['orport'])) continue - if entry['id'] != self._fpr: - continue - if (entry.has_key('ipv6') - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + if (entry.has_key('ipv6') and has_ipv6): + ipv6 = self.ipv6addr + ':' + self.ipv6orport # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != self.ipv6addr + ':' + self.ipv6orport: + if entry['ipv6'] != ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, ' + + 'but IPv6 (%s) does not match entry IPv6 (%s)', + self._fpr, ipv6, entry['ipv6']) continue # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and self.ipv6addr is None: + elif entry.has_key('ipv6') and not has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has no IPv6, and entry has IPv6 (%s)', self._fpr, + entry['ipv6']) + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) continue - elif not entry.has_key('ipv6') and self.ipv6addr is not None: + elif not entry.has_key('ipv6') and has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has IPv6 (%s), and entry has no IPv6', self._fpr, + ipv6) + logging.warning('%s excluded: has it gained an IPv6 address %s?', + self._fpr, ipv6) continue return True return False @@ -773,34 +830,60 @@ class Candidate(object): for entry in relaylist: for key in entry: value = entry[key] + if key == 'id' and value == self._fpr: + logging.info('%s is in the blacklist: fingerprint matches', + self._fpr) + return True if key == 'ipv4' and value == self.dirip: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'DirPort (%d) match', self._fpr, self.dirip, + self.dirport) return True # if the orport is present, check it too elif entry.has_key('orport'): if int(entry['orport']) == self.orport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'ORPort (%d) match', self._fpr, self.dirip, + self.orport) return True else: + logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' + + 'entry has no DirPort or ORPort', self._fpr, + self.dirip) return True - if key == 'id' and value == self._fpr: - return True - if (key == 'ipv6' - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + ipv6 = (self.ipv6addr + ':' + self.ipv6orport) if has_ipv6 else None + if (key == 'ipv6' and has_ipv6): # if both entry and fallback have an ipv6 address, compare them, # otherwise, disregard ipv6 addresses - if value == self.ipv6addr + ':' + self.ipv6orport: + if value == ipv6: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv6 (%s) and ' + + 'DirPort (%d) match', self._fpr, ipv6, + self.dirport) return True - # if the orport is present, check it too - elif entry.has_key('orport'): - if int(entry['orport']) == self.orport: - return True + # we've already checked the ORPort, it's part of entry['ipv6'] else: + logging.info('%s is in the blacklist: IPv6 (%s) matches, and' + + 'entry has no DirPort', self._fpr, ipv6) return True + elif (key == 'ipv6' or has_ipv6): + # only log if the fingerprint matches but the IPv6 doesn't + if entry.has_key('id') and entry['id'] == self._fpr: + logging.info('%s skipping IPv6 blacklist comparison: relay ' + + 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, + '' if has_ipv6 else ' no', + (' (' + ipv6 + ')') if has_ipv6 else '', + '' if key == 'ipv6' else ' no', + (' (' + value + ')') if key == 'ipv6' else '') + logging.warning('Has %s %s IPv6 address %s?', self._fpr, + 'gained an' if has_ipv6 else 'lost its former', + ipv6 if has_ipv6 else value) return False def is_exit(self): @@ -809,6 +892,9 @@ class Candidate(object): def is_guard(self): return 'Guard' in self._data['flags'] + def is_running(self): + return 'Running' in self._data['flags'] + def fallback_weight_fraction(self, total_weight): return float(self._data['consensus_weight']) / total_weight @@ -825,53 +911,70 @@ class Candidate(object): @staticmethod def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time): + download_failed = False downloader = DescriptorDownloader() start = datetime.datetime.utcnow() + # some directory mirrors respond to requests in ways that hang python + # sockets, which is why we long this line here + logging.info('Initiating consensus download from %s (%s:%d).', nickname, + dirip, dirport) # there appears to be about 1 second of overhead when comparing stem's # internal trace time and the elapsed time calculated here - downloader.get_consensus(endpoints = [(dirip, dirport)]).run() + TIMEOUT_SLOP = 1.0 + try: + downloader.get_consensus(endpoints = [(dirip, dirport)], + timeout = (max_time + TIMEOUT_SLOP), + validate = True, + retries = 0, + fall_back_to_authority = False).run() + except Exception, stem_error: + logging.debug('Unable to retrieve a consensus from %s: %s', nickname, + stem_error) + status = 'error: "%s"' % (stem_error) + level = logging.WARNING + download_failed = True elapsed = (datetime.datetime.utcnow() - start).total_seconds() if elapsed > max_time: status = 'too slow' + level = logging.WARNING + download_failed = True else: status = 'ok' - logging.debug(('Consensus download: %0.2fs %s from %s (%s:%d), ' - + 'max download time %0.2fs.') % (elapsed, status, - nickname, dirip, dirport, - max_time)) - return elapsed + level = logging.DEBUG + logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' + + 'max download time %0.1fs.', elapsed, status, nickname, + dirip, dirport, max_time) + return download_failed def fallback_consensus_dl_check(self): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + # include the relay if we're not doing a check, or we can't check (IPv6) + ipv4_failed = False + ipv6_failed = False + if PERFORM_IPV4_DIRPORT_CHECKS: + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS: # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - ipv6_speed = None # Now retry the relay if it took too long the first time - if (ipv4_speed > CONSENSUS_DOWNLOAD_SPEED_MAX + if (PERFORM_IPV4_DIRPORT_CHECKS and ipv4_failed and CONSENSUS_DOWNLOAD_RETRY): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS - and ipv6_speed > CONSENSUS_DOWNLOAD_SPEED_MAX - and CONSENSUS_DOWNLOAD_RETRY): - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + and ipv6_failed and CONSENSUS_DOWNLOAD_RETRY): + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - - return (ipv4_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX - and (not PERFORM_IPV6_DIRPORT_CHECKS - or ipv6_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX)) + return ((not ipv4_failed) and (not ipv6_failed)) def fallbackdir_line(self, total_weight, original_total_weight, dl_speed_ok): # /* @@ -1071,8 +1174,8 @@ class CandidateList(dict): if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES: # exclude excluded_count += 1 - logging.debug('Excluding %s: in both blacklist and whitelist.' % - f._fpr) + logging.warning('Excluding %s: in both blacklist and whitelist.', + f._fpr) else: # include filtered_fallbacks.append(f) @@ -1082,8 +1185,7 @@ class CandidateList(dict): elif in_blacklist: # exclude excluded_count += 1 - logging.debug('Excluding %s: in blacklist.' % - f._fpr) + logging.debug('Excluding %s: in blacklist.', f._fpr) else: if INCLUDE_UNLISTED_ENTRIES: # include @@ -1091,8 +1193,8 @@ class CandidateList(dict): else: # exclude excluded_count += 1 - logging.debug('Excluding %s: in neither blacklist nor whitelist.' % - f._fpr) + logging.info('Excluding %s: in neither blacklist nor whitelist.', + f._fpr) self.fallbacks = filtered_fallbacks return excluded_count @@ -1173,15 +1275,14 @@ class CandidateList(dict): # Integers don't need escaping in C comments fallback_count = len(self.fallbacks) if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' + fallback_proportion = ' (none)' else: - fallback_proportion = ' (%d * %f)'%(guard_count, + fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, FALLBACK_PROPORTION_OF_GUARDS) s += 'Final Count: %d (Eligible %d, Usable %d, Target %d%s'%( min(max_count, fallback_count), eligible_count, fallback_count, - target_count, fallback_proportion) if MAX_FALLBACK_COUNT is not None: s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) @@ -1242,6 +1343,16 @@ class CandidateList(dict): s += '#error ' + error_str else: s += '/* ' + error_str + ' */' + s += '\n' + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( + 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', + ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS + and PERFORM_IPV6_DIRPORT_CHECKS) else '', + 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', + CONSENSUS_DOWNLOAD_SPEED_MAX) + else: + s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' return s ## Main Function @@ -1250,9 +1361,11 @@ def list_fallbacks(): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """ + # find relays that could be fallbacks candidates = CandidateList() candidates.add_relays() + # work out how many fallbacks we want guard_count = candidates.count_guards() if FALLBACK_PROPORTION_OF_GUARDS is None: target_count = guard_count @@ -1268,10 +1381,10 @@ def list_fallbacks(): candidates.compute_fallbacks() + # filter with the whitelist and blacklist initial_count = len(candidates.fallbacks) excluded_count = candidates.apply_filter_lists() print candidates.summarise_filters(initial_count, excluded_count) - eligible_count = len(candidates.fallbacks) eligible_weight = candidates.fallback_weight_total()

1 0

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 26 Apr '16

26 Apr '16

commit 428c6e6b8f45cdb9fa6b71c63395e217b1afa56b Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 26 23:15:45 2016 +0000 Update translations for tails-misc_completed --- pt_BR.po | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pt_BR.po b/pt_BR.po index b16b114..2a7a4c6 100644 --- a/pt_BR.po +++ b/pt_BR.po @@ -15,15 +15,16 @@ # Lucas Possatti, 2014 # Luciana Dark Blue <lucianadarkblue(a)gmail.com>, 2015 # Matheus Boni Vicari <matheus_boni_vicari(a)hotmail.com>, 2014 +# Matheus Gaboardi Tralli <matheusgtralli(a)live.com>, 2016 # Matheus Martins, 2013 # Wagner Marques Oliveira <wagnermarques00(a)hotmail.com>, 2016 msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-03-17 15:03+0100\n" -"PO-Revision-Date: 2016-03-30 20:26+0000\n" -"Last-Translator: Communia <ameaneantie(a)riseup.net>\n" +"POT-Creation-Date: 2016-04-25 14:02+0200\n" +"PO-Revision-Date: 2016-04-26 23:06+0000\n" +"Last-Translator: Matheus Gaboardi Tralli <matheusgtralli(a)live.com>\n" "Language-Team: Portuguese (Brazil) (http://www.transifex.com/otf/torproject/language/pt_BR/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -72,13 +73,13 @@ msgid "Do you want to start Electrum anyway?" msgstr "Você deseja iniciar o Electrum assim mesmo?" #: config/chroot_local-includes/usr/local/bin/electrum:23 -#: config/chroot_local-includes/usr/local/bin/icedove:23 +#: config/chroot_local-includes/usr/local/bin/icedove:30 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:41 msgid "_Launch" msgstr "_Lançar" #: config/chroot_local-includes/usr/local/bin/electrum:24 -#: config/chroot_local-includes/usr/local/bin/icedove:24 +#: config/chroot_local-includes/usr/local/bin/icedove:31 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:42 msgid "_Exit" msgstr "_Saída" @@ -240,17 +241,24 @@ msgstr "Saída do GnuPG:" msgid "Other messages provided by GnuPG:" msgstr "Outras mensagens fornecidas pelo GnuPG:" -#: config/chroot_local-includes/usr/local/bin/icedove:19 +#: config/chroot_local-includes/usr/local/bin/icedove:20 msgid "The Claws Mail persistence feature is activated." msgstr "O recurso persistente Claws Mail foi ativado." -#: config/chroot_local-includes/usr/local/bin/icedove:21 +#: config/chroot_local-includes/usr/local/bin/icedove:22 msgid "" "If you have emails saved in Claws Mail, you should <a " "href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove'>migrate" " your data</a> before starting Icedove." msgstr "Se você tem emails salvos no Claws Mail, você deve <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove'>migrar seus dados</a> antes de iniciar o Icedove." +#: config/chroot_local-includes/usr/local/bin/icedove:27 +msgid "" +"If you already migrated your emails to Icedove, you should <a " +"href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete" +" all your Claws Mail data</a> to remove this warning." +msgstr "Se você já migrou seus emails para o Icedove, você deveria <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>apagar todos seus dados do Claws Mail</a> para remover esse aviso." + #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:71 msgid "Restart" msgstr "Reiniciar"

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 26 Apr '16

26 Apr '16

commit 77540278a2d829a87f61abdcb9d3526d0d878d39 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 26 23:15:42 2016 +0000 Update translations for tails-misc --- pt_BR.po | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pt_BR.po b/pt_BR.po index 6bd2e1b..2a7a4c6 100644 --- a/pt_BR.po +++ b/pt_BR.po @@ -15,6 +15,7 @@ # Lucas Possatti, 2014 # Luciana Dark Blue <lucianadarkblue(a)gmail.com>, 2015 # Matheus Boni Vicari <matheus_boni_vicari(a)hotmail.com>, 2014 +# Matheus Gaboardi Tralli <matheusgtralli(a)live.com>, 2016 # Matheus Martins, 2013 # Wagner Marques Oliveira <wagnermarques00(a)hotmail.com>, 2016 msgid "" @@ -22,8 +23,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-04-25 14:02+0200\n" -"PO-Revision-Date: 2016-04-26 09:06+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"PO-Revision-Date: 2016-04-26 23:06+0000\n" +"Last-Translator: Matheus Gaboardi Tralli <matheusgtralli(a)live.com>\n" "Language-Team: Portuguese (Brazil) (http://www.transifex.com/otf/torproject/language/pt_BR/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -256,7 +257,7 @@ msgid "" "If you already migrated your emails to Icedove, you should <a " "href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete" " all your Claws Mail data</a> to remove this warning." -msgstr "" +msgstr "Se você já migrou seus emails para o Icedove, você deveria <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>apagar todos seus dados do Claws Mail</a> para remover esse aviso." #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:71 msgid "Restart"

1 0

[tor-browser-bundle/hardened-builds] Preparations for -build4
by gk＠torproject.org 26 Apr '16

26 Apr '16

commit 52d518cc8c89b2934190838fbe3075e686e05b97 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Apr 26 21:28:04 2016 +0000 Preparations for -build4 --- Bundle-Data/Docs/ChangeLog.txt | 6 +++++- gitian/versions.alpha | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 49a49e5..35d0e9d 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,4 +1,4 @@ -Tor Browser 6.0a5-hardened -- April 27 2016 +Tor Browser 6.0a5-hardened -- April 28 2016 * All Platforms * Update Firefox to 45.1.0esr * Update Tor to 0.2.8.2-alpha @@ -14,6 +14,7 @@ Tor Browser 6.0a5-hardened -- April 27 2016 * Update meek to 0.22 (tag 0.22-18371-2) * Bug 18371: Symlinks are incompatible with Gatekeeper signing * Bug 15197 and child tickets: Rebase Tor Browser patches to ESR 45 + * Bug 18900: Fix broken updater on Linux * Bug 18042: Disable SHA1 certificate support * Bug 18821: Disable libmdns support for desktop and mobile * Bug 18848: Disable additional welcome URL shown on first start @@ -33,6 +34,9 @@ Tor Browser 6.0a5-hardened -- April 27 2016 * Bug 18741: Fix OCSP and favicon isolation for ESR 45 * Bug 16998: Disable <link rel="preconnect"> for now * Bug 17506: Reenable building hardened Tor Browser with startup cache + * Bug 18898: Exempt the meek extension from the signing requirement as well + * Bug 18899: Don't copy Torbutton, TorLauncher, etc. into meek profile + * Bug 18890: Test importScripts() for cache and network isolation * Bug 18726: Add new default obfs4 bridge (GreenBelt) * Build System * Bug 16224: Don't use BUILD_HOSTNAME anymore in Firefox builds diff --git a/gitian/versions.alpha b/gitian/versions.alpha index 2a8e10f..b9239dc 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -18,7 +18,7 @@ FIREFOX_VERSION=45.1.0esr TORBROWSER_UPDATE_CHANNEL=hardened -TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build2 +TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build3 TOR_TAG=tor-0.2.8.2-alpha TORLAUNCHER_TAG=0.2.8.4 TORBUTTON_TAG=1.9.5.3

1 0

[tor-browser-bundle/master] Using tor-browser-45.1.0esr-6.0-1-build3
by gk＠torproject.org 26 Apr '16

26 Apr '16

commit 127c4835c3ff00db224b4e38e51d0bf3ef322a9b Author: Georg Koppen <gk(a)torproject.org> Date: Tue Apr 26 21:15:07 2016 +0000 Using tor-browser-45.1.0esr-6.0-1-build3 --- gitian/versions.alpha | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitian/versions.alpha b/gitian/versions.alpha index 8752e76..f2023f0 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -14,7 +14,7 @@ FIREFOX_VERSION=45.1.0esr TORBROWSER_UPDATE_CHANNEL=alpha -TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build2 +TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-6.0-1-build3 TOR_TAG=tor-0.2.8.2-alpha TORLAUNCHER_TAG=0.2.9.1 TORBUTTON_TAG=1.9.5.3

1 0

[tor-browser-bundle/master] Update changelog for -build4
by gk＠torproject.org 26 Apr '16

26 Apr '16

commit 91ee3ab05a1a2775acd680bb3dd75b8eb2797890 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Apr 26 20:41:06 2016 +0000 Update changelog for -build4 --- Bundle-Data/Docs/ChangeLog.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 4d27e9e..5ea56f1 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,4 +1,4 @@ -Tor Browser 6.0a5 -- April 27 2016 +Tor Browser 6.0a5 -- April 28 2016 * All Platforms * Update Firefox to 45.1.0esr * Update Tor to 0.2.8.2-alpha @@ -14,6 +14,7 @@ Tor Browser 6.0a5 -- April 27 2016 * Update meek to 0.22 (tag 0.22-18371-2) * Bug 18371: Symlinks are incompatible with Gatekeeper signing * Bug 15197 and child tickets: Rebase Tor Browser patches to ESR 45 + * Bug 18900: Fix broken updater on Linux * Bug 18042: Disable SHA1 certificate support * Bug 18821: Disable libmdns support for desktop and mobile * Bug 18848: Disable additional welcome URL shown on first start @@ -32,6 +33,9 @@ Tor Browser 6.0a5 -- April 27 2016 * Bug 16326: Verify cache isolation for Request and Fetch APIs * Bug 18741: Fix OCSP and favicon isolation for ESR 45 * Bug 16998: Disable <link rel="preconnect"> for now + * Bug 18898: Exempt the meek extension from the signing requirement as well + * Bug 18899: Don't copy Torbutton, TorLauncher, etc. into meek profile + * Bug 18890: Test importScripts() for cache and network isolation * Bug 18726: Add new default obfs4 bridge (GreenBelt) * Windows * Bug 13419: Support ICU in Windows builds

1 0

[tor-browser/tor-browser-45.1.0esr-6.0-1] fixup! Bug 14970: Don't block our unsigned extensions
by gk＠torproject.org 26 Apr '16

26 Apr '16

commit f46c63ff6856251faf39e20a040ee104fa113176 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Apr 26 19:55:31 2016 +0000 fixup! Bug 14970: Don't block our unsigned extensions --- toolkit/mozapps/extensions/internal/XPIProvider.jsm | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/toolkit/mozapps/extensions/internal/XPIProvider.jsm b/toolkit/mozapps/extensions/internal/XPIProvider.jsm index 3056201..94c33ea 100644 --- a/toolkit/mozapps/extensions/internal/XPIProvider.jsm +++ b/toolkit/mozapps/extensions/internal/XPIProvider.jsm @@ -683,12 +683,13 @@ function isUsableAddon(aAddon) { return false; } // Temporary and system add-ons do not require signing. Neither do Torbutton - // nor TorLauncher nor EFF's HTTPS-Everywhere. + // nor TorLauncher nor EFF's HTTPS-Everywhere nor meek. if ((aAddon._installLocation.name != KEY_APP_SYSTEM_DEFAULTS && aAddon._installLocation.name != KEY_APP_TEMPORARY) && aAddon.id != "torbutton(a)torproject.org" && aAddon.id != "tor-launcher(a)torproject.org" && aAddon.id != "https-everywhere-eff(a)eff.org" && + aAddon.id != "meek-http-helper(a)bamsoftware.com" && mustSign(aAddon.type)) { if (aAddon.signedState <= AddonManager.SIGNEDSTATE_MISSING) return false; @@ -3289,12 +3290,13 @@ this.XPIProvider = { continue; } - // Make sure Torbutton, TorLauncher and EFF's HTTPS-Everywhere are still - // working after an update. + // Make sure Torbutton, TorLauncher, EFF's HTTPS-Everywhere and meek + // are still working after an update. if (mustSign(addon.type) && addon.id != "torbutton(a)torproject.org" && addon.id != "tor-launcher(a)torproject.org" && addon.id != "https-everywhere-eff(a)eff.org" && + addon.id != "meek-http-helper(a)bamsoftware.com" && addon.signedState <= AddonManager.SIGNEDSTATE_MISSING) { logger.warn("Refusing to install staged add-on " + id + " with signed state " + addon.signedState); seenFiles.push(stageDirEntry.leafName);

1 0