commit 5f097080460d54fb34ba116e84c1ab4dfc8dc744 Author: Kaidan fishstacks@protonmail.com Date: Sun Jul 8 06:54:14 2018 +1000
#26502: Stop using fallback blacklist --- changes/bug26502 | 3 + scripts/maint/fallback.blacklist | 232 ------------------------------------ scripts/maint/updateFallbackDirs.py | 154 +++--------------------- 3 files changed, 21 insertions(+), 368 deletions(-)
diff --git a/changes/bug26502 b/changes/bug26502 new file mode 100644 index 000000000..b3fabda6b --- /dev/null +++ b/changes/bug26502 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - 'updateFallbackDirs.py' now ignores the blacklist file as it's not longer needed + Fixes bug 26502; bugfix on 0.3.5.0-alpha. diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist deleted file mode 100644 index a118cb591..000000000 --- a/scripts/maint/fallback.blacklist +++ /dev/null @@ -1,232 +0,0 @@ -# updateFallbackDirs.py directory mirror blacklist -# -# Format: -# [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ... -# [ ipv6=<IPv6>[:<IPv6 ORPort>] ] -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... -# -# If a sufficiently specific group of attributes matches, the directory mirror -# will be excluded: (each group is listed on its own line) -# <IPv4>, <DirPort> -# <IPv4>, <ORPort> -# <ID> -# <IPv6>, <DirPort> -# <IPv6>, <IPv6 ORPort> -# If DirPort and ORPort are not present, the entire IP address is blacklisted. -# (The blacklist overrides the whitelist.) - -# If a relay operator doesn't want their relay to be a FallbackDir, -# enter the following information here: -# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ipv6=<IPv6>:<IPv6 ORPort> - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008364.html -87.181.248.227:9030 orport=443 id=8827944C4BDCBDAC9079803F47823403C11A9B7A - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008368.html -149.18.2.82:9030 orport=9001 id=953DB709F2A2DECC8D7560661F934E64411444F7 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html -80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443 - -# Email sent directly to teor, verified using relay contact info -5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070 -217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001 -62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101 - -# Email sent directly to teor, verified using relay contact info -216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7 -216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6 -65.19.167.130:80 orport=443 id=890E2EA65455FBF0FAAB4159FAC4412BDCB24295 -65.19.167.131:80 orport=443 id=0DA9BD201766EDB19F57F49F1A013A8A5432C008 -65.19.167.132:80 orport=443 id=12B80ABF019354A9D25EE8BE85EB3C0AD8F7DFC1 -65.19.167.133:80 orport=443 id=C170AE5A886C5A09D6D1CF5CF284653632EEF25D - -# Email sent directly to teor, verified using relay contact info -213.136.83.225:80 orport=443 id=B411027C926A9BFFCF7DA91E3CAF1856A321EFFD -195.154.126.78:80 orport=443 id=F6556156E2B3837248E03FDB770441CF64DBBFBE - -# Email sent directly to teor, verified using relay contact info -178.63.198.113:80 orport=443 id=872B18761953254914F77C71846E8A2623C52591 - -# Email sent directly to teor, verified using relay contact info -63.141.226.34:80 orport=9001 id=5EF131C0C82270F40B756987FDB5D54D9C966238 -185.75.56.103:80 orport=9001 id=3763CE5C3F574670D4296573744F821C0FFFB98E - -# Email sent directly to teor, verified using relay contact info -81.7.14.227:9030 orport=9001 id=BCA197C43A44B7B9D14509637F96A45B13C233D0 - -# Email sent directly to teor, verified using relay contact info -84.245.32.195:9030 orport=9001 id=4CD4DFFEF3971C902A22100D911CAC639BE2EF5C - -# Email sent directly to teor, verified using relay contact info -185.21.217.10:9030 orport=9001 id=41537E1D3DD3CAE86F5A3F0882F1C647FE8FC0A0 - -# Email sent directly to teor, verified using relay contact info -185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869 - -# Email sent directly to teor, verified using relay contact info -46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443 - -# Email sent directly to teor, verified using relay contact info -195.154.107.23:80 orport=443 id=A1F89F26E82209169E4037B035AE7B6C94A49AEB ipv6=[2001:bc8:3829:300::1]:443 -195.154.92.70:80 orport=443 id=E7FF4ECEEFCFE3A40A6D3594898A4A3DE018BBF5 ipv6=[2001:bc8:3829:500::1]:443 -195.154.113.200:80 orport=443 id=D1A4763FA0BD71978901B1951FEE1DC29777F95A ipv6=[2001:bc8:3829:600::1]:443 -195.154.92.155:110 orport=993 id=4477D3466FE136B7FE6F7FF8EBD0D6E2FFE3288B ipv6=[2001:bc8:3829:100::1]:993 -195.154.117.182:110 orport=993 id=B1A0F1143789466AADD5FAE5948C8138548EECEC ipv6=[2001:bc8:3829:400::1]:993 -195.154.97.163:80 orport=443 id=8A2994A63B20813B7724817A8FB8C444D10BA2E2 - -# Email sent directly to teor, verified using relay contact info -5.135.154.206:9030 orport=9001 id=7D67B342DC1158F4CFFEE8BC530A2448848026E3 - -# Email sent directly to teor, verified using relay contact info -85.24.215.117:9030 orport=9001 id=5989521A85C94EE101E88B8DB2E68321673F9405 ipv6=[2001:9b0:20:2106:21a:4aff:fea5:ad05]:9001 - -# Email sent directly to teor, verified using relay contact info -62.210.137.230:8888 orport=8843 id=CD6B850159CFF4C068A8D0F1BA5296AE4EDCAB39 ipv6=[2001:bc8:31d3:100::1]:3443 -62.210.137.230:8080 orport=8443 id=F596E1B1EF98E1DDBBDC934DB722AF54069868F6 ipv6=[2001:bc8:31d3:100::1]:8443 - -# Email sent directly to teor, verified using relay contact info -195.154.99.80:80 orport=443 id=6E7CB6E783C1B67B79D0EBBE7D48BC09BD441201 -195.154.127.60:80 orport=443 id=D74ABE34845190E242EC74BA28B8C89B0A480D4B - -# Email sent directly to teor, verified using relay contact info -212.51.143.20:80 orport=443 id=62DA0256BBC28992D41CBAFB549FFD7C9B846A99 - -# Email sent directly to teor, verified using relay contact info -195.154.90.122:80 orport=443 id=3A0D88024A30152E6F6372CFDF8F9B725F984362 - -# Email sent directly to teor, verified using relay contact info -188.166.118.215:9030 orport=443 id=FB5FF60F5EBA010F8A45AC6ED31A4393718A2C31 ipv6=[2a03:b0c0:2:d0::72:9001]:443 - -# Email sent directly to teor, verified using relay contact info -185.87.185.245:40001 orport=40000 id=2A499AEEA95FB10F07544383D562368E49BE32CA - -# Email sent directly to teor, verified using relay contact info -82.161.109.71:9030 orport=9001 id=BD9CE352648B940E788A8E45393C5400CC3E87E7 - -# Email sent directly to teor, verified using relay contact info -212.83.40.239:9030 orport=9001 id=6DC5616BD3FC463329DCE87DD7AAAEA112C264B5 - -# Email sent directly to teor, verified using relay contact info -178.32.53.53:80 orport=443 id=10582C360E972EE76B0DB1C246F4E892A6BF5465 - -# Email sent directly to teor, verified using relay contact info -85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001 -148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001 -62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001 - -# Email sent directly to teor, verified using relay contact info -74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0 - -# Email sent directly to teor, verified using relay contact info -89.163.140.168:9030 orport=9001 id=839C1212DB15723263BE96C83DA7E1B24FA395E8 - -# Email sent directly to teor, verified using relay contact info -212.47.246.211:9030 orport=9001 id=AA34219475E41282095DD3C088009EE562AF14E5 - -# Email sent directly to teor, verified using relay contact info -85.195.235.148:9030 orport=9001 id=103336165A0D2EFCAD3605339843A0A7710B8B92 -85.195.235.148:19030 orport=19001 id=713235638AB6C64715EAFD1B4772685E38AFD52A - -# Email sent directly to teor, verified using relay contact info -163.172.7.30:9030 orport=9001 id=E2EACD4752B2583202F374A34CACC844A3AECAC4 - -# Email sent directly to teor, verified using relay contact info -178.62.90.111:22 orport=25 id=3254D1DC1F1531D9C07C535E4991F38EE99B99E1 - -# Email sent directly to teor, verified using relay contact info -213.200.106.131:9030 orport=4443 id=B07CE79FD215129C381F6645B16E76DCA0845CAB - -# Email sent directly to teor, verified using relay contact info -198.51.75.165:80 orport=9001 id=DABCB84A524A22FDDD3AFCB090E3090CC12D9770 - -# Email sent directly to teor, verified using relay contact info -204.194.29.4:80 orport=9001 id=78C7C299DB4C4BD119A22B87B57D5AF5F3741A79 - -# Email sent directly to teor, verified using relay contact info -104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001 - -# Email sent directly to teor, verified using relay contact info -46.252.25.249:9030 orport=443 id=80DCBB6EF4E86A7CD4FBCBDEE64979645509A610 - -# Email sent directly to teor, verified using relay contact info -176.10.99.200:8080 orport=443 id=2B44FD1742D26E4F28D4CACF1F0CF8A686270E45 -176.10.99.200:8000 orport=22 id=EB79F07792A065D3C534063773E83268E069F5EB -176.10.99.201:667 orport=666 id=3EAAAB35932610411E24FA4317603CB5780B80BC -176.10.99.201:990 orport=989 id=7C3A4CFF09C1981D41173CDE2A2ADD4A5CA109FD -176.10.99.202:992 orport=991 id=615EBC4B48F03858FA50A3E23E5AF569D0D2308A -176.10.99.202:994 orport=993 id=E34E25D958D46DDE5092385B14117C9B301DC0E9 -176.10.99.203:1194 orport=995 id=AD368442E9FF33C08C7407DF2DA7DB958F406CE2 -176.10.99.203:43 orport=53 id=79CF377F0ACEC5F0002D85335E4192B34202A269 -176.10.99.204:1755 orport=1723 id=69DF3CDA1CDA460C17ECAD9D6F0C117A42384FA0 -176.10.99.204:1293 orport=4321 id=3F061400B6FB1F55E7F19BB3C713884D677E55B7 -176.10.99.205:426 orport=425 id=C30B284784BF11D0D58C6A250240EE58D2084AD0 -176.10.99.205:109 orport=110 id=12D17D9F9E30FA901DE68806950A0EA278716CED -176.10.99.206:24 orport=23 id=2C804AAB0C02F971A4386B3A1F2AC00F9E080679 -176.10.99.206:20 orport=21 id=237588726AB6BEA37FF23CA00F5BD178586CA68E -176.10.99.207:3390 orport=3389 id=A838D5B8890B10172429ECE92EB5677DF93DC4DD -176.10.99.207:1415 orport=1414 id=377E5E817A84FAE0F4DC3427805DB2E8A6CBBFC0 -176.10.99.208:390 orport=389 id=7C288587BA0D99CC6B8537CDC2C4639FA827B907 -176.10.99.208:3307 orport=3306 id=1F0D2A44C56F42816DED2022EFD631878C29905B -176.10.99.209:1434 orport=1433 id=BDA7A91FF3806DE5109FDAE74CFEFB3BABB9E10F -176.10.99.209:220 orport=219 id=B8C2030001D832066A648269CFBA94171951D34B - -# Email sent directly to teor, verified using relay contact info -78.193.40.205:8080 orport=8443 id=C91450840E75AC1B654A3096744338A573A239C6 - -# Email sent directly to teor, verified using relay contact info -37.187.22.172:9030 orport=9035 id=335E4117BD9A4966403C2AFA31CFDD1BC13BD46A - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008367.html -# Email sent directly to teor to opt-out -88.198.38.226:22 orport=443 id=4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB ipv6=[2a01:4f8:a0:1351::2]:80 -213.239.210.204:22 orport=443 id=5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9 ipv6=[2a01:4f8:a0:9474::2]:80 -213.239.220.25:22 orport=443 id=BEE2317AE127EB681C5AE1551C1EA0630580638A ipv6=[2a01:4f8:a0:710c::2]:80 -85.10.201.38:22 orport=443 id=F6279A203C1950ACF592322A235647A05BFBCF91 ipv6=[2a01:4f8:a0:43cc::2]:80 - -# Email sent directly to teor, verified using relay contact info -88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E - -# Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6 -37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001 - -# Email sent directly to teor, verified using relay contact info -195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796 - -# Fallback was on 0.2.8.6 list, but opted-out before 0.2.9 -144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907 - -# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html -# no dirport -86.107.110.34:0 orport=9001 id=A0E3D30A660DB70CA0B6D081BA54D094DED6F28D -94.242.59.147:80 orport=9001 id=674DCBB0D9C1C4C4DBFB4A9AE024AF59FE4E7F46 ipv6=[2a00:1838:35:42::b648]:9001 - -# Email sent directly to teor, verified using relay contact info -167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443 - -# Email sent directly to teor, verified using relay contact info -163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 - -# Email sent directly to teor, verified using relay contact info -104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD - -# Emails sent directly to teor, verified using relay contact info -217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443 - -# Emails sent directly to teor, verified using relay contact info -195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D -195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C - -# Email sent directly to teor, verified using relay contact info -5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE - -#https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html -62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001 -62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101 - -# Email sent directly to teor -212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110 diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 355dc027d..a78be1836 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -117,10 +117,9 @@ CONSENSUS_EXPIRY_TOLERANCE = 0 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
-# Output matching ContactInfo in fallbacks list or the blacklist? +# Output matching ContactInfo in fallbacks list? # Useful if you're trying to contact operators CONTACT_COUNT = True if OUTPUT_CANDIDATES else False -CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
# How the list should be sorted: # fingerprint: is useful for stable diffs of fallback lists @@ -141,26 +140,15 @@ LOCAL_FILES_ONLY = False
# The whitelist contains entries that are included if all attributes match # (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport) -# The blacklist contains (partial) entries that are excluded if any -# sufficiently specific group of attributes matches: -# IPv4 & DirPort -# IPv4 & ORPort -# ID -# IPv6 & DirPort -# IPv6 & IPv6 ORPort -# If neither port is included in the blacklist, the entire IP address is -# blacklisted. - -# What happens to entries in neither list? + +# What happens to entries not in whitelist? # When True, they are included, when False, they are excluded INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
# If an entry is in both lists, what happens? # When True, it is excluded, when False, it is included -BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True
WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist' -BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist' FALLBACK_FILE_NAME = 'src/app/config/fallback_dirs.inc'
# The number of bytes we'll read from a filter file before giving up @@ -984,78 +972,6 @@ class Candidate(object): return True return False
- def is_in_blacklist(self, relaylist): - """ A fallback matches a blacklist line if a sufficiently specific group - of attributes matches: - ipv4 & dirport - ipv4 & orport - id - ipv6 & dirport - ipv6 & ipv6 orport - If the fallback and the blacklist line both have an ipv6 key, - their values will be compared, otherwise, they will be ignored. - If there is no dirport and no orport, the entry matches all relays on - that ip. """ - for entry in relaylist: - for key in entry: - value = entry[key] - if key == 'id' and value == self._fpr: - log_excluded('%s is in the blacklist: fingerprint matches', - self._fpr) - return True - if key == 'ipv4' and value == self.dirip: - # if the dirport is present, check it too - if entry.has_key('dirport'): - if int(entry['dirport']) == self.dirport: - log_excluded('%s is in the blacklist: IPv4 (%s) and ' + - 'DirPort (%d) match', self._fpr, self.dirip, - self.dirport) - return True - # if the orport is present, check it too - elif entry.has_key('orport'): - if int(entry['orport']) == self.orport: - log_excluded('%s is in the blacklist: IPv4 (%s) and ' + - 'ORPort (%d) match', self._fpr, self.dirip, - self.orport) - return True - else: - log_excluded('%s is in the blacklist: IPv4 (%s) matches, and ' + - 'entry has no DirPort or ORPort', self._fpr, - self.dirip) - return True - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) - if (key == 'ipv6' and self.has_ipv6()): - # if both entry and fallback have an ipv6 address, compare them, - # otherwise, disregard ipv6 addresses - if value == ipv6: - # if the dirport is present, check it too - if entry.has_key('dirport'): - if int(entry['dirport']) == self.dirport: - log_excluded('%s is in the blacklist: IPv6 (%s) and ' + - 'DirPort (%d) match', self._fpr, ipv6, - self.dirport) - return True - # we've already checked the ORPort, it's part of entry['ipv6'] - else: - log_excluded('%s is in the blacklist: IPv6 (%s) matches, and' + - 'entry has no DirPort', self._fpr, ipv6) - return True - elif (key == 'ipv6' or self.has_ipv6()): - # only log if the fingerprint matches but the IPv6 doesn't - if entry.has_key('id') and entry['id'] == self._fpr: - log_excluded('%s skipping IPv6 blacklist comparison: relay ' + - 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, - '' if self.has_ipv6() else ' no', - (' (' + ipv6 + ')') if self.has_ipv6() else '', - '' if key == 'ipv6' else ' no', - (' (' + value + ')') if key == 'ipv6' else '') - logging.warning('Has %s %s IPv6 address %s?', self._fpr, - 'gained an' if self.has_ipv6() else 'lost its former', - ipv6 if self.has_ipv6() else value) - return False - def cw_to_bw_factor(self): # any relays with a missing or zero consensus weight are not candidates # any relays with a missing advertised bandwidth have it set to zero @@ -1317,26 +1233,12 @@ class Candidate(object): s += '\n' if self._data['contact'] is not None: s += cleanse_c_multiline_comment(self._data['contact']) - if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT: + if CONTACT_COUNT: fallback_count = len([f for f in fallbacks if f._data['contact'] == self._data['contact']]) if fallback_count > 1: s += '\n' s += '%d identical contacts listed' % (fallback_count) - if CONTACT_BLACKLIST_COUNT: - prefilter_count = len([f for f in prefilter_fallbacks - if f._data['contact'] == self._data['contact']]) - filter_count = prefilter_count - fallback_count - if filter_count > 0: - if fallback_count > 1: - s += ' ' - else: - s += '\n' - s += '%d blacklisted' % (filter_count) - s += '\n' - s += '*/' - s += '\n' - return s
# output the fallback info C string for this fallback # this is the text that would go after FallbackDir in a torrc @@ -1544,48 +1446,32 @@ class CandidateList(dict): relaylist.append(relay_entry) return relaylist
- # apply the fallback whitelist and blacklist - def apply_filter_lists(self, whitelist_obj, blacklist_obj): + # apply the fallback whitelist + def apply_filter_lists(self, whitelist_obj): excluded_count = 0 - logging.debug('Applying whitelist and blacklist.') - # parse the whitelist and blacklist + logging.debug('Applying whitelist') + # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) - blacklist = self.load_relaylist(blacklist_obj) filtered_fallbacks = [] for f in self.fallbacks: in_whitelist = f.is_in_whitelist(whitelist) - in_blacklist = f.is_in_blacklist(blacklist) - if in_whitelist and in_blacklist: - if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES: - # exclude - excluded_count += 1 - logging.warning('Excluding %s: in both blacklist and whitelist.', - f._fpr) - else: - # include - filtered_fallbacks.append(f) - elif in_whitelist: + if in_whitelist: # include filtered_fallbacks.append(f) - elif in_blacklist: - # exclude - excluded_count += 1 - log_excluded('Excluding %s: in blacklist.', f._fpr) - else: - if INCLUDE_UNLISTED_ENTRIES: + elif INCLUDE_UNLISTED_ENTRIES: # include filtered_fallbacks.append(f) - else: + else: # exclude excluded_count += 1 - log_excluded('Excluding %s: in neither blacklist nor whitelist.', + log_excluded('Excluding %s: not in whitelist.', f._fpr) self.fallbacks = filtered_fallbacks return excluded_count
@staticmethod def summarise_filters(initial_count, excluded_count): - return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( + return '/* Whitelist excluded %d of %d candidates. */'%( excluded_count, initial_count)
# calculate each fallback's measured bandwidth based on the median @@ -2181,18 +2067,14 @@ def process_existing(): logging.getLogger('stem').setLevel(logging.INFO) whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), 'name': FALLBACK_FILE_NAME} - blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE), - 'name': BLACKLIST_FILE_NAME} - list_fallbacks(whitelist, blacklist) + list_fallbacks(whitelist)
def process_default(): logging.basicConfig(level=logging.WARNING) logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE), - 'name': BLACKLIST_FILE_NAME} - list_fallbacks(whitelist, blacklist) + list_fallbacks(whitelist)
## Main Function def main(): @@ -2213,7 +2095,7 @@ def log_excluded(msg, *args): else: logging.info(msg, *args)
-def list_fallbacks(whitelist, blacklist): +def list_fallbacks(whitelist): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """
@@ -2250,13 +2132,13 @@ def list_fallbacks(whitelist, blacklist): candidates.compute_fallbacks() prefilter_fallbacks = copy.copy(candidates.fallbacks)
- # filter with the whitelist and blacklist + # filter with the whitelist # if a relay has changed IPv4 address or ports recently, it will be excluded # as ineligible before we call apply_filter_lists, and so there will be no # warning that the details have changed from those in the whitelist. # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist, blacklist) + excluded_count = candidates.apply_filter_lists(whitelist) print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks)