[tor-commits] [tor/master] Fallbacks: refactor is_in_whitelist() to take an exact match boolean
nickm at torproject.org
nickm at torproject.org
Tue Dec 11 15:04:27 UTC 2018
commit 75b5cc047d3e257701de6dfab9f80d358025fb95
Author: teor <teor at torproject.org>
Date: Fri Dec 7 11:16:15 2018 +1000
Fallbacks: refactor is_in_whitelist() to take an exact match boolean
No behaviour change.
Preparation for 24838.
---
scripts/maint/fallback.whitelist | 23 +---
scripts/maint/updateFallbackDirs.py | 220 ++++++++++++++++++++++++++----------
2 files changed, 167 insertions(+), 76 deletions(-)
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index 79551948c..97291d73b 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -1,34 +1,21 @@
# updateFallbackDirs.py directory mirror whitelist
-#
-# Format:
-# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-# or use:
-# scripts/maint/generateFallbackDirLine.py fingerprint ...
-#
+
# All attributes must match for the directory mirror to be included.
# If the fallback has an ipv6 key, the whitelist line must also have
# it, and vice versa, otherwise they don't match.
-# (The blacklist overrides the whitelist.)
# To replace this list with the hard-coded fallback list (for testing), use
-# a command similar to:
+# "updateFallbackDirs.py check_existing", or a command similar to:
# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \
# tr -d '\n' | \
# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \
# > scripts/maint/fallback.whitelist
-#
-# When testing before a release, exclusions due to changed details will result
-# in a warning, unless the IPv4 address or port change happened recently.
-# Then it is only logged at info level, as part of the eligibility check.
-# Exclusions due to stability also are only shown at info level.
-#
-# Add the number of selected, slow, and excluded relays, and compare that to
-# the number of hard-coded relays. If it's less, use info-level logs to find
-# out why each of the missing relays was excluded.
# If a relay operator wants their relay to be a FallbackDir,
# enter the following information here:
-# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
+# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ( ipv6=[<IPv6>]:<IPv6 ORPort> )?
+# or use:
+# scripts/maint/generateFallbackDirLine.py fingerprint ...
# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index 914d12134..b33435215 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -920,61 +920,155 @@ class Candidate(object):
return False
return True
- def is_in_whitelist(self, relaylist):
- """ A fallback matches if each key in the whitelist line matches:
+ def id_matches(self, id, exact=False):
+ """ Does this fallback's id match id?
+ exact is ignored. """
+ return self._fpr == id
+
+ def ipv4_addr_matches(self, ipv4_addr, exact=False):
+ """ Does this fallback's IPv4 address match ipv4_addr?
+ exact is ignored. """
+ return self.dirip == ipv4_addr
+
+ def ipv4_dirport_matches(self, ipv4_dirport, exact=False):
+ """ Does this fallback's IPv4 dirport match ipv4_dirport?
+ If exact is False, always return True. """
+ if exact:
+ return self.dirport == int(ipv4_dirport)
+ else:
+ return True
+
+ def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False):
+ """ Does this fallback's IPv4 address match ipv4_addr?
+ If exact is True, also check ipv4_dirport. """
+ ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact)
+ if exact:
+ return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport,
+ exact=exact)
+ else:
+ return ipv4_match
+
+ def ipv4_orport_matches(self, ipv4_orport, exact=False):
+ """ Does this fallback's IPv4 orport match ipv4_orport?
+ If exact is False, always return True. """
+ if exact:
+ return self.orport == int(ipv4_orport)
+ else:
+ return True
+
+ def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False):
+ """ Does this fallback's IPv4 address match ipv4_addr?
+ If exact is True, also check ipv4_orport. """
+ ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact)
+ if exact:
+ return ipv4_match and self.ipv4_orport_matches(ipv4_orport,
+ exact=exact)
+ else:
+ return ipv4_match
+
+ def ipv6_addr_matches(self, ipv6_addr, exact=False):
+ """ Does this fallback's IPv6 address match ipv6_addr?
+ Both addresses must be present to match.
+ exact is ignored. """
+ if self.has_ipv6() and ipv6_addr is not None:
+ # Check that we have a bracketed IPv6 address without a port
+ assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']'))
+ return self.ipv6addr == ipv6_addr
+ else:
+ return False
+
+ def ipv6_orport_matches(self, ipv6_orport, exact=False):
+ """ Does this fallback's IPv6 orport match ipv6_orport?
+ Both ports must be present to match.
+ If exact is False, always return True. """
+ if exact:
+ return (self.has_ipv6() and ipv6_orport is not None and
+ self.ipv6orport == int(ipv6_orport))
+ else:
+ return True
+
+ def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False):
+ """ Does this fallback's IPv6 address match ipv6_addr?
+ If exact is True, also check ipv6_orport. """
+ ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact)
+ if exact:
+ return ipv6_match and self.ipv6_orport_matches(ipv6_orport,
+ exact=exact)
+ else:
+ return ipv6_match
+
+ def entry_matches_exact(self, entry):
+ """ Is entry an exact match for this fallback?
+ A fallback is an exact match for entry if each key in entry matches:
ipv4
dirport
orport
id
- ipv6 address and port (if present)
+ ipv6 address and port (if present in the fallback or the whitelist)
If the fallback has an ipv6 key, the whitelist line must also have
- it, and vice versa, otherwise they don't match. """
- ipv6 = None
- if self.has_ipv6():
- ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
+ it, otherwise they don't match.
+
+ Logs a warning-level message if the fallback would be an exact match,
+ but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport
+ have changed. """
+ if not self.id_matches(entry['id'], exact=True):
+ # can't log here unless we match an IP and port, because every relay's
+ # fingerprint is compared to every entry's fingerprint
+ if self.ipv4_and_orport_matches(entry['ipv4'],
+ entry['orport'],
+ exact=True):
+ logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
+ '%s?', entry['id'], self.dirip, self.orport,
+ self._fpr)
+ if self.ipv6_and_orport_matches(entry.get('ipv6_addr'),
+ entry.get('ipv6_orport'),
+ exact=True):
+ logging.warning('%s excluded: has OR %s changed fingerprint to ' +
+ '%s?', entry['id'], entry['ipv6'], self._fpr)
+ return False
+ if not self.ipv4_addr_matches(entry['ipv4'], exact=True):
+ logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
+ self._fpr, entry['ipv4'], self.dirip)
+ return False
+ if not self.ipv4_dirport_matches(entry['dirport'], exact=True):
+ logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
+ '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
+ self.dirip, self.dirport)
+ return False
+ if not self.ipv4_orport_matches(entry['orport'], exact=True):
+ logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
+ '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
+ self.dirip, self.orport)
+ return False
+ if entry.has_key('ipv6') and self.has_ipv6():
+ # if both entry and fallback have an ipv6 address, compare them
+ if not self.ipv6_and_orport_matches(entry['ipv6_addr'],
+ entry['ipv6_orport'],
+ exact=True):
+ logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
+ 'to %s:%d?', self._fpr, entry['ipv6'],
+ self.ipv6addr, self.ipv6orport)
+ return False
+ # if the fallback has an IPv6 address but the whitelist entry
+ # doesn't, or vice versa, the whitelist entry doesn't match
+ elif entry.has_key('ipv6') and not self.has_ipv6():
+ logging.warning('%s excluded: has it lost its former IPv6 address %s?',
+ self._fpr, entry['ipv6'])
+ return False
+ elif not entry.has_key('ipv6') and self.has_ipv6():
+ logging.warning('%s excluded: has it gained an IPv6 address %s:%d?',
+ self._fpr, self.ipv6addr, self.ipv6orport)
+ return False
+ return True
+
+ def is_in_whitelist(self, relaylist, exact=False):
+ """ If exact is True (existing fallback list), check if this fallback is
+ an exact match for any whitelist entry, using entry_matches_exact().
+ """
for entry in relaylist:
- if entry['id'] != self._fpr:
- # can't log here unless we match an IP and port, because every relay's
- # fingerprint is compared to every entry's fingerprint
- if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport:
- logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
- '%s?', entry['id'], self.dirip, self.orport,
- self._fpr)
- if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6:
- logging.warning('%s excluded: has OR %s changed fingerprint to ' +
- '%s?', entry['id'], ipv6, self._fpr)
- continue
- if entry['ipv4'] != self.dirip:
- logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
- self._fpr, entry['ipv4'], self.dirip)
- continue
- if int(entry['dirport']) != self.dirport:
- logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
- '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
- self.dirip, self.dirport)
- continue
- if int(entry['orport']) != self.orport:
- logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
- '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
- self.dirip, self.orport)
- continue
- if entry.has_key('ipv6') and self.has_ipv6():
- # if both entry and fallback have an ipv6 address, compare them
- if entry['ipv6'] != ipv6:
- logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
- 'to %s?', self._fpr, entry['ipv6'], ipv6)
- continue
- # if the fallback has an IPv6 address but the whitelist entry
- # doesn't, or vice versa, the whitelist entry doesn't match
- elif entry.has_key('ipv6') and not self.has_ipv6():
- logging.warning('%s excluded: has it lost its former IPv6 address %s?',
- self._fpr, entry['ipv6'])
- continue
- elif not entry.has_key('ipv6') and self.has_ipv6():
- logging.warning('%s excluded: has it gained an IPv6 address %s?',
- self._fpr, ipv6)
- continue
- return True
+ if exact:
+ if self.entry_matches_exact(entry):
+ return True
return False
def cw_to_bw_factor(self):
@@ -1458,18 +1552,28 @@ class CandidateList(dict):
relay_entry['dirport'] = ipv4_maybe_dirport_split[1]
elif kvl == 2:
relay_entry[key_value_split[0]] = key_value_split[1]
+ # split ipv6 addresses and orports
+ if key_value_split[0] == 'ipv6':
+ ipv6_orport_split = key_value_split[1].rsplit(':', 1)
+ ipv6l = len(ipv6_orport_split)
+ if ipv6l != 2:
+ print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%(
+ file_name, item)
+ relay_entry['ipv6_addr'] = ipv6_orport_split[0]
+ relay_entry['ipv6_orport'] = ipv6_orport_split[1]
relaylist.append(relay_entry)
return relaylist
- # apply the fallback whitelist
- def apply_filter_lists(self, whitelist_obj):
+ def apply_filter_lists(self, whitelist_obj, exact=False):
+ """ Apply the fallback whitelist_obj to this fallback list,
+ passing exact to is_in_whitelist(). """
excluded_count = 0
logging.debug('Applying whitelist')
# parse the whitelist
whitelist = self.load_relaylist(whitelist_obj)
filtered_fallbacks = []
for f in self.fallbacks:
- in_whitelist = f.is_in_whitelist(whitelist)
+ in_whitelist = f.is_in_whitelist(whitelist, exact=exact)
if in_whitelist:
# include
filtered_fallbacks.append(f)
@@ -2082,14 +2186,14 @@ def process_existing():
logging.getLogger('stem').setLevel(logging.INFO)
whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
'name': FALLBACK_FILE_NAME}
- list_fallbacks(whitelist)
+ list_fallbacks(whitelist, exact=True)
def process_default():
logging.basicConfig(level=logging.WARNING)
logging.getLogger('stem').setLevel(logging.WARNING)
whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
'name': WHITELIST_FILE_NAME}
- list_fallbacks(whitelist)
+ list_fallbacks(whitelist, exact=True)
## Main Function
def main():
@@ -2110,10 +2214,10 @@ def log_excluded(msg, *args):
else:
logging.info(msg, *args)
-def list_fallbacks(whitelist):
+def list_fallbacks(whitelist, exact=False):
""" Fetches required onionoo documents and evaluates the
- fallback directory criteria for each of the relays """
-
+ fallback directory criteria for each of the relays,
+ passing exact to apply_filter_lists(). """
print "/* type=fallback */"
print ("/* version={} */"
.format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
@@ -2153,7 +2257,7 @@ def list_fallbacks(whitelist):
# warning that the details have changed from those in the whitelist.
# instead, there will be an info-level log during the eligibility check.
initial_count = len(candidates.fallbacks)
- excluded_count = candidates.apply_filter_lists(whitelist)
+ excluded_count = candidates.apply_filter_lists(whitelist, exact=exact)
print candidates.summarise_filters(initial_count, excluded_count)
eligible_count = len(candidates.fallbacks)
More information about the tor-commits
mailing list