April 2016 - tor-commits - lists.torproject.org

[tor/master] Add fallbacks to white/blacklist from operator responses
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 8b90d45f78fe3d4b27667839a229dd299fcb9d29 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Mar 31 09:19:50 2016 +1100 Add fallbacks to white/blacklist from operator responses Also add misbehaving relays to updateFallbackDirs.py blacklist, but leave them commented out in case it's a transient issue, or it's been resolved by the download check fixes. (These relays hang stem's downloader. It's unlikely they'll ever help clients.) --- scripts/maint/fallback.blacklist | 131 ++++++++ scripts/maint/fallback.whitelist | 638 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 765 insertions(+), 4 deletions(-) diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist index 3c0f8fc..f34986a 100644 --- a/scripts/maint/fallback.blacklist +++ b/scripts/maint/fallback.blacklist @@ -39,3 +39,134 @@ # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html 62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001 62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101 + +# these fallback candidates fail the consensus download test in a way that +# causes stem to hang (and not respond to ^C, at least on OS X) +# (Is something sending weird responses to DirPort traffic?) +#217.23.14.190:1194 +#151.80.164.147:80 +#148.251.255.92:80 +#78.142.19.59:80 + +# Email sent directly to teor, verified using relay contact info +216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7 +216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6 +65.19.167.130:80 orport=443 id=890E2EA65455FBF0FAAB4159FAC4412BDCB24295 +65.19.167.131:80 orport=443 id=0DA9BD201766EDB19F57F49F1A013A8A5432C008 +65.19.167.132:80 orport=443 id=12B80ABF019354A9D25EE8BE85EB3C0AD8F7DFC1 +65.19.167.133:80 orport=443 id=C170AE5A886C5A09D6D1CF5CF284653632EEF25D + +# Email sent directly to teor, verified using relay contact info +195.154.165.227:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A +62.210.69.34:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 +163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 + +# Email sent directly to teor, verified using relay contact info +213.136.83.225:80 orport=443 id=B411027C926A9BFFCF7DA91E3CAF1856A321EFFD +195.154.126.78:80 orport=443 id=F6556156E2B3837248E03FDB770441CF64DBBFBE + +# Email sent directly to teor, verified using relay contact info +178.63.198.113:80 orport=443 id=872B18761953254914F77C71846E8A2623C52591 + +# Email sent directly to teor, verified using relay contact info +63.141.226.34:80 orport=9001 id=5EF131C0C82270F40B756987FDB5D54D9C966238 +185.75.56.103:80 orport=9001 id=3763CE5C3F574670D4296573744F821C0FFFB98E + +# Email sent directly to teor, verified using relay contact info +108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699 +81.7.14.227:9030 orport=9001 id=BCA197C43A44B7B9D14509637F96A45B13C233D0 + +# Email sent directly to teor, verified using relay contact info +84.245.32.195:9030 orport=9001 id=4CD4DFFEF3971C902A22100D911CAC639BE2EF5C + +# Email sent directly to teor, verified using relay contact info +185.21.217.10:9030 orport=9001 id=41537E1D3DD3CAE86F5A3F0882F1C647FE8FC0A0 + +# Email sent directly to teor, verified using relay contact info +185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869 + +# Email sent directly to teor, verified using relay contact info +62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21 + +# Email sent directly to teor, verified using relay contact info +46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443 + +# Email sent directly to teor, verified using relay contact info +195.154.107.23:80 orport=443 id=A1F89F26E82209169E4037B035AE7B6C94A49AEB ipv6=[2001:bc8:3829:300::1]:443 +195.154.92.70:80 orport=443 id=E7FF4ECEEFCFE3A40A6D3594898A4A3DE018BBF5 ipv6=[2001:bc8:3829:500::1]:443 +195.154.113.200:80 orport=443 id=D1A4763FA0BD71978901B1951FEE1DC29777F95A ipv6=[2001:bc8:3829:600::1]:443 +195.154.92.155:110 orport=993 id=4477D3466FE136B7FE6F7FF8EBD0D6E2FFE3288B ipv6=[2001:bc8:3829:100::1]:993 +195.154.117.182:110 orport=993 id=B1A0F1143789466AADD5FAE5948C8138548EECEC ipv6=[2001:bc8:3829:400::1]:993 +195.154.97.163:80 orport=443 id=8A2994A63B20813B7724817A8FB8C444D10BA2E2 + +# Email sent directly to teor, verified using relay contact info +5.135.154.206:9030 orport=9001 id=7D67B342DC1158F4CFFEE8BC530A2448848026E3 + +# Email sent directly to teor, verified using relay contact info +85.24.215.117:9030 orport=9001 id=5989521A85C94EE101E88B8DB2E68321673F9405 ipv6=[2001:9b0:20:2106:21a:4aff:fea5:ad05]:9001 + +# Email sent directly to teor, verified using relay contact info +62.210.137.230:8888 orport=8843 id=CD6B850159CFF4C068A8D0F1BA5296AE4EDCAB39 ipv6=[2001:bc8:31d3:100::1]:3443 +62.210.137.230:8080 orport=8443 id=F596E1B1EF98E1DDBBDC934DB722AF54069868F6 ipv6=[2001:bc8:31d3:100::1]:8443 + +# Email sent directly to teor, verified using relay contact info +195.154.99.80:80 orport=443 id=6E7CB6E783C1B67B79D0EBBE7D48BC09BD441201 +195.154.127.60:80 orport=443 id=D74ABE34845190E242EC74BA28B8C89B0A480D4B + +# Email sent directly to teor, verified using relay contact info +212.51.143.20:80 orport=443 id=62DA0256BBC28992D41CBAFB549FFD7C9B846A99 + +# Email sent directly to teor, verified using relay contact info +195.154.90.122:80 orport=443 id=3A0D88024A30152E6F6372CFDF8F9B725F984362 + +# Email sent directly to teor, verified using relay contact info +188.166.118.215:9030 orport=443 id=FB5FF60F5EBA010F8A45AC6ED31A4393718A2C31 ipv6=[2a03:b0c0:2:d0::72:9001]:443 + +# Email sent directly to teor, verified using relay contact info +185.87.185.245:40001 orport=40000 id=2A499AEEA95FB10F07544383D562368E49BE32CA + +# Email sent directly to teor, verified using relay contact info +82.161.109.71:9030 orport=9001 id=BD9CE352648B940E788A8E45393C5400CC3E87E7 + +# Email sent directly to teor, verified using relay contact info +212.83.40.239:9030 orport=9001 id=6DC5616BD3FC463329DCE87DD7AAAEA112C264B5 + +# Email sent directly to teor, verified using relay contact info +178.32.53.53:80 orport=443 id=10582C360E972EE76B0DB1C246F4E892A6BF5465 + +# Email sent directly to teor, verified using relay contact info +85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001 +148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001 +62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001 +212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB + +# Email sent directly to teor, verified using relay contact info +74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0 + +# Email sent directly to teor, verified using relay contact info +89.163.140.168:9030 orport=9001 id=839C1212DB15723263BE96C83DA7E1B24FA395E8 + +# Email sent directly to teor, verified using relay contact info +212.47.246.211:9030 orport=9001 id=AA34219475E41282095DD3C088009EE562AF14E5 + +# Email sent directly to teor, verified using relay contact info +85.195.235.148:9030 orport=9001 id=103336165A0D2EFCAD3605339843A0A7710B8B92 +85.195.235.148:19030 orport=19001 id=713235638AB6C64715EAFD1B4772685E38AFD52A + +# Email sent directly to teor, verified using relay contact info +163.172.7.30:9030 orport=9001 id=E2EACD4752B2583202F374A34CACC844A3AECAC4 + +# Email sent directly to teor, verified using relay contact info +178.62.90.111:22 orport=25 id=3254D1DC1F1531D9C07C535E4991F38EE99B99E1 + +# Email sent directly to teor, verified using relay contact info +213.200.106.131:9030 orport=4443 id=B07CE79FD215129C381F6645B16E76DCA0845CAB + +# Email sent directly to teor, verified using relay contact info +198.51.75.165:80 orport=9001 id=DABCB84A524A22FDDD3AFCB090E3090CC12D9770 + +# Email sent directly to teor, verified using relay contact info +204.194.29.4:80 orport=9001 id=78C7C299DB4C4BD119A22B87B57D5AF5F3741A79 + +# Email sent directly to teor, verified using relay contact info +104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001 diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 6edb368..61cb97a 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -20,7 +20,8 @@ 84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008365.html -81.7.17.171:80 orport=443 id=00C4B4731658D3B4987132A3F77100CFCB190D97 ipv6=[2a02:180:1:1::517:11ab]:443 +# Email sent directly to teor with fingerprint update +81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html 5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050 @@ -54,7 +55,8 @@ 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html -85.25.138.93:9030 orport=4029 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F +# changed to 91.121.84.137? Awaiting operator confirmation +#85.25.138.93:9030 orport=4029 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F # https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html 5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33 @@ -95,6 +97,634 @@ # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443 -# Email sent directly to teor, verified using relay contact info +# Emails sent directly to teor, verified using relay contact info 217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 -217.12.210.214:80 orport=443 id=943C0C6841C1E914B9FCA796C6846620A5AF9BC7 + +# Email sent directly to teor, verified using relay contact info +94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69 + +# https://twitter.com/binarytenshi/status/717952514327453697 +94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073 + +# Email sent directly to teor, verified using relay contact info +171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443 +171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443 +171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576 +171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443 +# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576 +#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8 +# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02 +#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443 + +# Email sent directly to teor, verified using relay contact info +212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 +93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D +46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001 + +# Email sent directly to teor, verified using relay contact info +85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE +185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE +185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 + +# Email sent directly to teor, verified using relay contact info +185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778 + +# Email sent directly to teor, verified using relay contact info +178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443 +46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 +178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 +178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 +178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 +81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 +209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 +81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 +46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 +178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443" +46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 +178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 +178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 +178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 +81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 +209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 +81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 +46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 +178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 +178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443 +178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 +178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443 + +# Email sent directly to teor, verified using relay contact info +195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64 +51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132 +5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651 + +# Email sent directly to teor, verified using relay contact info +109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B +109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722 +109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA +77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D +109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6 +77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107 +109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8 +77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69 +109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A +62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF +109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B +109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722 +109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA +77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D +109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6 +77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107 +109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8 +77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69 +109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A +62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF + +# https://twitter.com/biotimylated/status/718994247500718080 +212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0 + +# Email sent directly to teor, verified using relay contact info +46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E + +# Email sent directly to teor, verified using relay contact info +94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 +176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 +176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 +94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 +85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 +85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 + +# Email sent directly to teor, verified using relay contact info +148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010 + +# Email sent directly to teor, verified using relay contact info +5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443 +5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001 + +# Email sent directly to teor, verified using relay contact info +62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001 +62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101 + +# Email sent directly to teor, verified using relay contact info +188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF + +# Message sent directly to teor, verified using relay contact info +95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5 +95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2 +95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F + +# Email sent directly to teor, verified using relay contact info +86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0 + +# Email sent directly to teor, verified using relay contact info +144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907 + +# Email sent directly to teor, verified using relay contact info +193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001 +109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001 + +# Email sent directly to teor, verified using relay contact info +146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E + +# Email sent directly to teor, verified using relay contact info +46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D + +# Email sent directly to teor, verified using relay contact info +89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C + +# Email sent directly to teor, verified using relay contact info +212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001 + +# Email sent directly to teor, verified using relay contact info +46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB + +# Email sent directly to teor, verified using relay contact info +217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B + +# Email sent directly to teor, verified using relay contact info +193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524 + +# Email sent directly to teor, verified using relay contact info +197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 + +# Email sent directly to teor, verified using relay contact info +185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C + +# Email sent directly to teor, verified using relay contact info +193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB +193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050 +193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E + +# Email sent directly to teor, verified using relay contact info +144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510 + +# Email sent directly to teor, verified using relay contact info +37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811 + +# Email sent directly to teor, verified using relay contact info +91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5 + +# Email sent directly to teor, verified using relay contact info +185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443 + +# Email sent directly to teor, verified using relay contact info +193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4 + +# Email sent directly to teor, verified using relay contact info +134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE + +# Email sent directly to teor, verified using relay contact info +81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7 + +# Email sent directly to teor, verified using relay contact info +178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050 +46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050 +188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050 +198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050 +159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050 + +# Email sent directly to teor, verified using relay contact info +5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900 + +# Email sent directly to teor, verified using relay contact info +188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788 + +# Email sent directly to teor, verified using relay contact info +213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22 +213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4 +213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2 +213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690 + +# Email sent directly to teor, verified using relay contact info +136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E + +# Email sent directly to teor, verified using relay contact info +212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97 + +# Email sent directly to teor, verified using relay contact info +176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 + +# Email sent directly to teor, verified using relay contact info +192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023 +51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D +192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244 +51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A +51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79 +192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B +51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA +51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF +192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4 +192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE + +# Email sent directly to teor, verified using relay contact info +151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 + +# Email sent directly to teor, verified using relay contact info +5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 + +# Email sent directly to teor, verified using relay contact info +176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 +176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 +195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F +195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685 +46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7 + +# Email sent directly to teor, verified using relay contact info +178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C + +# Email sent directly to teor, verified using relay contact info +37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 +212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C + +# Email sent directly to teor, verified using relay contact info +195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197 + +# Email sent directly to teor, verified using relay contact info +37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001 + +# Email sent directly to teor, verified using relay contact info +62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4 + +# Email sent directly to teor, verified using relay contact info +37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 + +# Email sent directly to teor, verified using relay contact info +37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443 + +# Email sent directly to teor, verified using relay contact info +185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 + +# Email sent directly to teor, verified using relay contact info +185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 + +# Email sent directly to teor, verified using relay contact info +37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 + +# Email sent directly to teor, verified using relay contact info +5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE + +# Email sent directly to teor, verified using relay contact info +198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E + +# Email sent directly to teor, verified using relay contact info +212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF +212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B + +# Email sent directly to teor, verified using relay contact info +51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A + +# Email sent directly to teor, verified using relay contact info +78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74 + +# Email sent directly to teor, verified using relay contact info +195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714 + +# Email sent directly to teor, verified using relay contact info +62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8 + +# Email sent directly to teor, verified using relay contact info +5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6 + +# Email sent directly to teor, verified using relay contact info +212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20 + +# Email sent directly to teor, verified using relay contact info +85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0 + +# Email sent directly to teor, verified using relay contact info +46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205 +46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6 + +# Email sent directly to teor, verified using relay contact info +5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725 + +# Email sent directly to teor, verified using relay contact info +46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E +185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D +95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF + +# Email sent directly to teor, verified using relay contact info +212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7 + +# Email sent directly to teor, verified using relay contact info +91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 + +# Email sent directly to teor, verified using relay contact info +# Suitable, check with operator before adding +#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB +212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60 +163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443 +# Suitable, check with operator before adding +#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 + +# Email sent directly to teor, verified using relay contact info +46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 +178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 + +# Email sent directly to teor, verified using relay contact info +178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 + +# Email sent directly to teor, verified using relay contact info +82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050 + +# Email sent directly to teor, verified using relay contact info +146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D + +# Email sent directly to teor, verified using relay contact info +37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001 + +# Email sent directly to teor, verified using relay contact info +37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7 + +# Email sent directly to teor, verified using relay contact info +212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477 +# Confirm with operator before adding these +#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9 +#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB + +# Email sent directly to teor, verified using relay contact info +5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 + +# Email sent directly to teor, verified using relay contact info +188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001 + +# Email sent directly to teor, verified using relay contact info +5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 + +# Email sent directly to teor, verified using relay contact info +46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D + +# Email sent directly to teor, verified using relay contact info +144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC + +# Email sent directly to teor, verified using relay contact info +46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED +46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4 +46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A +163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 + +# Email sent directly to teor, verified using relay contact info +176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600 +195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D +195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C + +# Email sent directly to teor, verified using relay contact info +46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321 +46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB +46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76 +95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA + +# Email sent directly to teor, verified using relay contact info +93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F + +# Email sent directly to teor, verified using relay contact info +37.187.22.172:9030 orport=9035 id=335E4117BD9A4966403C2AFA31CFDD1BC13BD46A + +# Email sent directly to teor, verified using relay contact info +37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A + +# Email sent directly to teor, verified using relay contact info +5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8 + +# Email sent directly to teor, verified using relay contact info +51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5 + +# Email sent directly to teor, verified using relay contact info +5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 + +# Email sent directly to teor, verified using relay contact info +167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443 + +# Email sent directly to teor, verified using relay contact info +192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207 +167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F + +# Email sent directly to teor, verified using relay contact info +204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70 + +# Email sent directly to teor, verified using relay contact info +151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54 + +# Email sent directly to teor, verified using relay contact info +194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C + +# Email sent directly to teor, verified using relay contact info +185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001 + +# Email sent directly to teor, verified using relay contact info +5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A + +# Email sent directly to teor, verified using relay contact info +89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7 + +# Email sent directly to teor, verified using relay contact info +92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C + +# Email sent directly to teor, verified using relay contact info +80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001 + +# Email sent directly to teor, verified using relay contact info +83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443 + +# Email sent directly to teor, verified using relay contact info +51.254.101.242:32991 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF + +# Email sent directly to teor, verified using relay contact info +95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A + +# Email sent directly to teor, verified using relay contact info +176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C + +# Email sent directly to teor, verified using relay contact info +128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001 + +# Email sent directly to teor, verified using relay contact info +88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E + +# Email sent directly to teor, verified using relay contact info +178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2 + +# Email sent directly to teor, verified using relay contact info +78.193.40.205:8080 orport=8443 id=C91450840E75AC1B654A3096744338A573A239C6 + +# Email sent directly to teor, verified using relay contact info +212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887 + +# Email sent directly to teor, verified using relay contact info +204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28 + +# Email sent directly to teor, verified using relay contact info +80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B + +# Email sent directly to teor, verified using relay contact info +185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A + +# Email sent directly to teor, verified using relay contact info +149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 + +# Email sent directly to teor, verified using relay contact info +81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 + +# Email sent directly to teor, verified using relay contact info +195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993 +138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F +81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993 +134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993 +46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993 +37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993 +46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485 +185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993 +195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993 +195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993 +37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993 +217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993 +81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993 +46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995 +91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993 +163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F +188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F +81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993 +213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993 +94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583 +217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307 +212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142 +217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993 +216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A +69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993 +89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D +107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993 + +# Email sent directly to teor, verified using relay contact info +212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001 + +# Email sent directly to teor, verified using relay contact info +176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 + +# Email sent directly to teor, verified using relay contact info +91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F + +# Email sent directly to teor, verified using relay contact info +# Awaiting confirmation of new ORPort from relay operator +80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443 + +# Email sent directly to teor, verified using relay contact info +163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D + +# Email sent directly to teor, verified using relay contact info +97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94 + +# Email sent directly to teor, verified using relay contact info +192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC + +# Email sent directly to teor, verified using relay contact info +178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A + +# Email sent directly to teor, verified using relay contact info +195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796 + +# Email sent directly to teor, verified using relay contact info +185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0 + +# Email sent directly to teor, verified using relay contact info +164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B +78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 + +# Email sent directly to teor, verified using relay contact info +46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050 +178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050 + +# Email sent directly to teor, verified using relay contact info +91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6 + +# Email sent directly to teor, verified using relay contact info +167.114.113.48:80 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0 + +# Email sent directly to teor, verified using relay contact info +79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 + +# Email sent directly to teor, verified using relay contact info +95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443 + +# Email sent directly to teor, verified using relay contact info +166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB + +# Email sent directly to teor, verified using relay contact info +91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC + +# Email sent directly to teor, verified using relay contact info +178.217.184.32:9030 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D + +# Email sent directly to teor, verified using relay contact info +85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001 + +# Email sent directly to teor, verified using relay contact info +120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62 + +# Email sent directly to teor, verified using relay contact info +37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438 + +# Email sent directly to teor, verified using relay contact info +92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C + +# Email sent directly to teor, verified using relay contact info +31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798 + +# Email sent directly to teor, verified using relay contact info +96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387 +192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002 +192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F + +# Email sent directly to teor, verified using relay contact info +136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664 + +# Email sent directly to teor, verified using relay contact info +192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 +192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 + +# Email sent directly to teor, verified using relay contact info +192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 +# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024 +#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 + +# https://twitter.com/kosjoli/status/719507270904758272 +85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A +176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911 +46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204 + +# Email sent directly to teor, verified using relay contact info +78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257 + +# Email sent directly to teor, verified using relay contact info +185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E + +# Email sent directly to teor, verified using relay contact info +108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699 + +# Email sent directly to teor, verified using relay contact info +91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99 + +# Email sent directly to teor, verified using relay contact info +185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260 +192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8 + +# Email sent directly to teor, verified using relay contact info +185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361 + +# Email sent directly to teor, verified using relay contact info +128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7 + +# Email sent directly to teor, verified using relay contact info +198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92 + +# Email sent directly to teor, verified using relay contact info +185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859 + +# Email sent directly to teor, verified using relay contact info +167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443 + +# Email sent directly to teor, verified using relay contact info +66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142 +66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A

1 0

[tor/release-0.2.8] Improve fallback selection and output
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 7e1b8ae79cc7da3b4a0989d6f9ba95aa5cd9b73b Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Wed Mar 30 12:41:57 2016 +1100 Improve fallback selection and output Improve the download test: * Allow IPv4 DirPort checks to be turned off. * Add a timeout to stem's consensus download. * Actually check for download errors, rather than ignoring them. * Simplify the timeout and download error checking logic. Tweak whitelist/blacklist checks to be more robust. Improve logging, make it warn by default. Cleanse fallback comments more thoroughly: * non-printables (yes, ContactInfo can have these) * // comments (don't rely on newlines to prevent // */ escapes) --- scripts/maint/updateFallbackDirs.py | 289 +++++++++++++++++++++++++----------- 1 file changed, 201 insertions(+), 88 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index d110335..4860700 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,6 +1,7 @@ #!/usr/bin/python # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc +# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . # # Then read the generated list to ensure no-one slipped anything funny into # their name or contactinfo @@ -29,17 +30,28 @@ import dateutil.parser from stem.descriptor.remote import DescriptorDownloader import logging -logging.basicConfig(level=logging.DEBUG) +# INFO tells you why each relay was included or excluded +# WARN tells you about potential misconfigurations +logging.basicConfig(level=logging.WARNING) ## Top-Level Configuration -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -PERFORM_IPV6_DIRPORT_CHECKS = False - # Output all candidate fallbacks, or only output selected fallbacks? OUTPUT_CANDIDATES = False +# Perform DirPort checks over IPv4? +# Change this to False if IPv4 doesn't work for you, or if you don't want to +# download a consensus for each fallback +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True + +# Perform DirPort checks over IPv6? +# If you know IPv6 works for you, set this to True +# This will exclude IPv6 relays without an IPv6 DirPort configured +# So it's best left at False until #18394 is implemented +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False + ## OnionOO Settings ONIONOO = 'https://onionoo.torproject.org/' @@ -81,7 +93,7 @@ MAX_LIST_FILE_SIZE = 1024 * 1024 # Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted # This causes OnionOO to (correctly) reset its stability timer -# This issue is fixed in 0.2.7.7 and master. +# This issue will be fixed in 0.2.7.7 and 0.2.8.2 # Until then, the CUTOFFs below ensure a decent level of stability. ADDRESS_AND_PORT_STABLE_DAYS = 7 # What time-weighted-fraction of these flags must FallbackDirs @@ -157,36 +169,52 @@ def parse_ts(t): def remove_bad_chars(raw_string, bad_char_list): # Remove each character in the bad_char_list - escaped_string = raw_string + cleansed_string = raw_string for c in bad_char_list: - escaped_string = escaped_string.replace(c, '') - return escaped_string + cleansed_string = cleansed_string.replace(c, '') + return cleansed_string + +def cleanse_unprintable(raw_string): + # Remove all unprintable characters + cleansed_string = '' + for c in raw_string: + if (c in string.ascii_letters or c in string.digits + or c in string.punctuation or c in string.whitespace): + cleansed_string += c + return cleansed_string def cleanse_whitespace(raw_string): # Replace all whitespace characters with a space - escaped_string = raw_string + cleansed_string = raw_string for c in string.whitespace: - escaped_string = escaped_string.replace(c, ' ') - return escaped_string + cleansed_string = cleansed_string.replace(c, ' ') + return cleansed_string def cleanse_c_multiline_comment(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious / unanticipated string from breaking out # of a C-style multiline comment - # This removes '/*' and '*/' - # To deal with '//', the end comment must be on its own line - bad_char_list = '*' + # This removes '/*' and '*/' and '//' + bad_char_list = '*/' # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of comments # There isn't much we can do to cover every possible case # But comment-based directives are typically only advisory - return escaped_string + return cleansed_string def cleanse_c_string(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious address/fingerprint string from breaking out # of a C-style string bad_char_list = '"' @@ -195,13 +223,11 @@ def cleanse_c_string(raw_string): # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of strings # There isn't much we can do to cover every possible case # But this typically only results in changes to the string data - return escaped_string + return cleansed_string ## OnionOO Source Functions @@ -244,11 +270,11 @@ def write_to_file(str, file_name, max_len): with open(file_name, 'w') as f: f.write(str[0:max_len]) except EnvironmentError, error: - logging.debug('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.warning('Writing file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) def read_from_file(file_name, max_len): try: @@ -256,11 +282,11 @@ def read_from_file(file_name, max_len): with open(file_name, 'r') as f: return f.read(max_len) except EnvironmentError, error: - logging.debug('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.info('Loading file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) return None def load_possibly_compressed_response_json(response): @@ -699,30 +725,37 @@ class Candidate(object): self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE def is_candidate(self): + must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS + or PERFORM_IPV6_DIRPORT_CHECKS) + if (must_be_running_now and not self.is_running()): + logging.info('%s not a candidate: not running now, unable to check ' + + 'DirPort consensus download', self._fpr) + return False if (self._data['last_changed_address_or_port'] > self.CUTOFF_ADDRESS_AND_PORT_STABLE): - logging.debug('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) + logging.info('%s not a candidate: changed address/port recently (%s)', + self._fpr, self._data['last_changed_address_or_port']) return False if self._running < CUTOFF_RUNNING: - logging.debug('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) + logging.info('%s not a candidate: running avg too low (%lf)', + self._fpr, self._running) return False if self._v2dir < CUTOFF_V2DIR: - logging.debug('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) + logging.info('%s not a candidate: v2dir avg too low (%lf)', + self._fpr, self._v2dir) return False if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - logging.debug('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) + logging.info('%s not a candidate: badexit avg too high (%lf)', + self._fpr, self._badexit) return False # if the relay doesn't report a version, also exclude the relay if (not self._data.has_key('recommended_version') or not self._data['recommended_version']): + logging.info('%s not a candidate: version not recommended', self._fpr) return False if self._guard < CUTOFF_GUARD: - logging.debug('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) + logging.info('%s not a candidate: guard avg too low (%lf)', + self._fpr, self._guard) return False return True @@ -736,24 +769,48 @@ class Candidate(object): If the fallback has an ipv6 key, the whitelist line must also have it, and vice versa, otherwise they don't match. """ for entry in relaylist: + if entry['id'] != self._fpr: + # can't log here, every relay's fingerprint is compared to the entry + continue if entry['ipv4'] != self.dirip: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'IPv4 (%s) does not match entry IPv4 (%s)', + self._fpr, self.dirip, entry['ipv4']) continue if int(entry['dirport']) != self.dirport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'DirPort (%d) does not match entry DirPort (%d)', + self._fpr, self.dirport, int(entry['dirport'])) continue if int(entry['orport']) != self.orport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'ORPort (%d) does not match entry ORPort (%d)', + self._fpr, self.orport, int(entry['orport'])) continue - if entry['id'] != self._fpr: - continue - if (entry.has_key('ipv6') - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + if (entry.has_key('ipv6') and has_ipv6): + ipv6 = self.ipv6addr + ':' + self.ipv6orport # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != self.ipv6addr + ':' + self.ipv6orport: + if entry['ipv6'] != ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, ' + + 'but IPv6 (%s) does not match entry IPv6 (%s)', + self._fpr, ipv6, entry['ipv6']) continue # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and self.ipv6addr is None: + elif entry.has_key('ipv6') and not has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has no IPv6, and entry has IPv6 (%s)', self._fpr, + entry['ipv6']) + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) continue - elif not entry.has_key('ipv6') and self.ipv6addr is not None: + elif not entry.has_key('ipv6') and has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has IPv6 (%s), and entry has no IPv6', self._fpr, + ipv6) + logging.warning('%s excluded: has it gained an IPv6 address %s?', + self._fpr, ipv6) continue return True return False @@ -773,34 +830,60 @@ class Candidate(object): for entry in relaylist: for key in entry: value = entry[key] + if key == 'id' and value == self._fpr: + logging.info('%s is in the blacklist: fingerprint matches', + self._fpr) + return True if key == 'ipv4' and value == self.dirip: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'DirPort (%d) match', self._fpr, self.dirip, + self.dirport) return True # if the orport is present, check it too elif entry.has_key('orport'): if int(entry['orport']) == self.orport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'ORPort (%d) match', self._fpr, self.dirip, + self.orport) return True else: + logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' + + 'entry has no DirPort or ORPort', self._fpr, + self.dirip) return True - if key == 'id' and value == self._fpr: - return True - if (key == 'ipv6' - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + ipv6 = (self.ipv6addr + ':' + self.ipv6orport) if has_ipv6 else None + if (key == 'ipv6' and has_ipv6): # if both entry and fallback have an ipv6 address, compare them, # otherwise, disregard ipv6 addresses - if value == self.ipv6addr + ':' + self.ipv6orport: + if value == ipv6: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv6 (%s) and ' + + 'DirPort (%d) match', self._fpr, ipv6, + self.dirport) return True - # if the orport is present, check it too - elif entry.has_key('orport'): - if int(entry['orport']) == self.orport: - return True + # we've already checked the ORPort, it's part of entry['ipv6'] else: + logging.info('%s is in the blacklist: IPv6 (%s) matches, and' + + 'entry has no DirPort', self._fpr, ipv6) return True + elif (key == 'ipv6' or has_ipv6): + # only log if the fingerprint matches but the IPv6 doesn't + if entry.has_key('id') and entry['id'] == self._fpr: + logging.info('%s skipping IPv6 blacklist comparison: relay ' + + 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, + '' if has_ipv6 else ' no', + (' (' + ipv6 + ')') if has_ipv6 else '', + '' if key == 'ipv6' else ' no', + (' (' + value + ')') if key == 'ipv6' else '') + logging.warning('Has %s %s IPv6 address %s?', self._fpr, + 'gained an' if has_ipv6 else 'lost its former', + ipv6 if has_ipv6 else value) return False def is_exit(self): @@ -809,6 +892,9 @@ class Candidate(object): def is_guard(self): return 'Guard' in self._data['flags'] + def is_running(self): + return 'Running' in self._data['flags'] + def fallback_weight_fraction(self, total_weight): return float(self._data['consensus_weight']) / total_weight @@ -825,53 +911,70 @@ class Candidate(object): @staticmethod def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time): + download_failed = False downloader = DescriptorDownloader() start = datetime.datetime.utcnow() + # some directory mirrors respond to requests in ways that hang python + # sockets, which is why we long this line here + logging.info('Initiating consensus download from %s (%s:%d).', nickname, + dirip, dirport) # there appears to be about 1 second of overhead when comparing stem's # internal trace time and the elapsed time calculated here - downloader.get_consensus(endpoints = [(dirip, dirport)]).run() + TIMEOUT_SLOP = 1.0 + try: + downloader.get_consensus(endpoints = [(dirip, dirport)], + timeout = (max_time + TIMEOUT_SLOP), + validate = True, + retries = 0, + fall_back_to_authority = False).run() + except Exception, stem_error: + logging.debug('Unable to retrieve a consensus from %s: %s', nickname, + stem_error) + status = 'error: "%s"' % (stem_error) + level = logging.WARNING + download_failed = True elapsed = (datetime.datetime.utcnow() - start).total_seconds() if elapsed > max_time: status = 'too slow' + level = logging.WARNING + download_failed = True else: status = 'ok' - logging.debug(('Consensus download: %0.2fs %s from %s (%s:%d), ' - + 'max download time %0.2fs.') % (elapsed, status, - nickname, dirip, dirport, - max_time)) - return elapsed + level = logging.DEBUG + logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' + + 'max download time %0.1fs.', elapsed, status, nickname, + dirip, dirport, max_time) + return download_failed def fallback_consensus_dl_check(self): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + # include the relay if we're not doing a check, or we can't check (IPv6) + ipv4_failed = False + ipv6_failed = False + if PERFORM_IPV4_DIRPORT_CHECKS: + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS: # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - ipv6_speed = None # Now retry the relay if it took too long the first time - if (ipv4_speed > CONSENSUS_DOWNLOAD_SPEED_MAX + if (PERFORM_IPV4_DIRPORT_CHECKS and ipv4_failed and CONSENSUS_DOWNLOAD_RETRY): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS - and ipv6_speed > CONSENSUS_DOWNLOAD_SPEED_MAX - and CONSENSUS_DOWNLOAD_RETRY): - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + and ipv6_failed and CONSENSUS_DOWNLOAD_RETRY): + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - - return (ipv4_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX - and (not PERFORM_IPV6_DIRPORT_CHECKS - or ipv6_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX)) + return ((not ipv4_failed) and (not ipv6_failed)) def fallbackdir_line(self, total_weight, original_total_weight, dl_speed_ok): # /* @@ -1071,8 +1174,8 @@ class CandidateList(dict): if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES: # exclude excluded_count += 1 - logging.debug('Excluding %s: in both blacklist and whitelist.' % - f._fpr) + logging.warning('Excluding %s: in both blacklist and whitelist.', + f._fpr) else: # include filtered_fallbacks.append(f) @@ -1082,8 +1185,7 @@ class CandidateList(dict): elif in_blacklist: # exclude excluded_count += 1 - logging.debug('Excluding %s: in blacklist.' % - f._fpr) + logging.debug('Excluding %s: in blacklist.', f._fpr) else: if INCLUDE_UNLISTED_ENTRIES: # include @@ -1091,8 +1193,8 @@ class CandidateList(dict): else: # exclude excluded_count += 1 - logging.debug('Excluding %s: in neither blacklist nor whitelist.' % - f._fpr) + logging.info('Excluding %s: in neither blacklist nor whitelist.', + f._fpr) self.fallbacks = filtered_fallbacks return excluded_count @@ -1173,15 +1275,14 @@ class CandidateList(dict): # Integers don't need escaping in C comments fallback_count = len(self.fallbacks) if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' + fallback_proportion = ' (none)' else: - fallback_proportion = ' (%d * %f)'%(guard_count, + fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, FALLBACK_PROPORTION_OF_GUARDS) s += 'Final Count: %d (Eligible %d, Usable %d, Target %d%s'%( min(max_count, fallback_count), eligible_count, fallback_count, - target_count, fallback_proportion) if MAX_FALLBACK_COUNT is not None: s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) @@ -1242,6 +1343,16 @@ class CandidateList(dict): s += '#error ' + error_str else: s += '/* ' + error_str + ' */' + s += '\n' + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( + 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', + ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS + and PERFORM_IPV6_DIRPORT_CHECKS) else '', + 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', + CONSENSUS_DOWNLOAD_SPEED_MAX) + else: + s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' return s ## Main Function @@ -1250,9 +1361,11 @@ def list_fallbacks(): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """ + # find relays that could be fallbacks candidates = CandidateList() candidates.add_relays() + # work out how many fallbacks we want guard_count = candidates.count_guards() if FALLBACK_PROPORTION_OF_GUARDS is None: target_count = guard_count @@ -1268,10 +1381,10 @@ def list_fallbacks(): candidates.compute_fallbacks() + # filter with the whitelist and blacklist initial_count = len(candidates.fallbacks) excluded_count = candidates.apply_filter_lists() print candidates.summarise_filters(initial_count, excluded_count) - eligible_count = len(candidates.fallbacks) eligible_weight = candidates.fallback_weight_total()

1 0

[tor/master] April 2016 fallbacks for 0.2.8-rc
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 1fd4340f827065485f8ce3fd03c5573f89880893 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 15 02:33:41 2016 +1000 April 2016 fallbacks for 0.2.8-rc --- src/or/fallback_dirs.inc | 516 +++++++++++++++++++++-------------------------- 1 file changed, 230 insertions(+), 286 deletions(-) diff --git a/src/or/fallback_dirs.inc b/src/or/fallback_dirs.inc index 45610fa..8e82a3f 100644 --- a/src/or/fallback_dirs.inc +++ b/src/or/fallback_dirs.inc @@ -1,298 +1,242 @@ -/* Trial fallbacks for 0.2.8.2-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 7 - * This works around an issue where relays post a descriptor without a DirPort - * when restarted. The flag CUTOFFs ensure sufficient relay stability. -- teor - */ -/* Whitelist & blacklist excluded 1380 of 1412 candidates. */ +/* Whitelist & blacklist excluded 1273 of 1553 candidates. */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ /* -Fallback Directory Summary -Final Count: 32 (Eligible 32, Usable 32, Target 335 (1679 * 0.200000), Clamped to 500) -*/ -/* Ignore low fallback numbers in alpha builds -- teor -#error Fallback Count 32 is too low. Must be at least 50 for diversity. Try adding entries to the whitelist, or setting INCLUDE_UNLISTED_ENTRIES = True. -*/ -/* -Final Weight: 704514 (Eligible 712270) -Max Weight: 56981 (8.088%) (Clamped to 10.000%) -Min Weight: 4450 (0.632%) (Clamped to 0.100%) -Clamped: 7755 (1.101%) Excess Weight, 3 High Weight Fallbacks (9.4%) +Final Count: 100 (Eligible 280, Target 356 (1781 * 0.20), Max 100) +Excluded: 180 (Same Operator 102, Failed/Skipped Download 40, Excess 38) +Bandwidth Range: 6.0 - 67.2 MB/s */ /* -Onionoo Source: details Date: 2016-02-27 07:00:00 Version: 3.1 -URL: https://onionoo.torproject.org/details?fields=fingerprint%2Cnickname%2Ccont… +Onionoo Source: details Date: 2016-04-18 01:00:00 Version: 3.1 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family&flag=V2Dir&type=relay&last_seen_days=-7&first_seen_days=7- */ /* -Onionoo Source: uptime Date: 2016-02-27 07:00:00 Version: 3.1 -URL: https://onionoo.torproject.org/uptime?first_seen_days=7-&flag=V2Dir&type=re… -*/ -/* -kitten1 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 56981 / 704514 (8.088%) -Consensus Weight: 61100 / 712270 (8.578%) -0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd -*/ -"62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C" -" ipv6=[2001:bc8:3f23:100::1]:9001" -" weight=56981", -/* -fluxe4 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 56981 / 704514 (8.088%) -Consensus Weight: 59800 / 712270 (8.396%) -Sebastian <tor(a)sebastianhahn.net> - 12NbRAjAG5U3LLWETSF7fSTcdaz32Mu5CN -*/ -"131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8" -" weight=56981", -/* -kitten2 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 56981 / 704514 (8.088%) -Consensus Weight: 57800 / 712270 (8.115%) -0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd +Onionoo Source: uptime Date: 2016-04-18 01:00:00 Version: 3.1 +URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay&last_seen_days=-7 */ +"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" +" weight=10", +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +" weight=10", +"37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D" +" ipv6=[2001:41d0:a:195::1]:9001" +" weight=10", +"5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6" +" weight=10", +"5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484" +" weight=10", +"178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163" +" weight=10", +"144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB" +" ipv6=[2a01:4f8:190:9490::dead]:443" +" weight=10", +"178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2" +" weight=10", +"46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8" +" weight=10", +"91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7" +" weight=10", +"212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68" +" weight=10", +"185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C" +" weight=10", +"51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D" +" weight=10", +"194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C" +" weight=10", +"144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510" +" weight=10", "62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" " ipv6=[2001:bc8:3f23:100::1]:9101" -" weight=56981", -/* -fluxe3 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 54800 / 704514 (7.778%) -Sebastian <tor(a)sebastianhahn.net> - 12NbRAjAG5U3LLWETSF7fSTcdaz32Mu5CN -*/ -"78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE" -" weight=54800", -/* -tornoderdednl -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 50500 / 704514 (7.168%) -0x4871E82F Thom Wiggers <thom @AT@ RDED POINT NL> BTC 1DLyDFV13zhCWJYHMh5bk5C58yYvpxqxfQ -*/ +" weight=10", +"213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22" +" weight=10", +"212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B" +" weight=10", +"109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F" +" ipv6=[2001:948:7:2::163]:5001" +" weight=10", +"146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E" +" weight=10", +"217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B" +" weight=10", +"62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF" +" ipv6=[2001:bc8:338c::1]:9101" +" weight=10", +"198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92" +" weight=10", +"164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B" +" weight=10", +"176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600" +" weight=10", +"37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053" +" ipv6=[2001:41d0:a:26ba::1]:9001" +" weight=10", +"188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF" +" weight=10", +"108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2" +" weight=10", +"212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2" +" ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001" +" weight=10", +"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" +" weight=10", +"94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69" +" weight=10", +"109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA" +" weight=10", +"185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0" +" ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443" +" weight=10", +"78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74" +" weight=10", +"185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859" +" weight=10", +"178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B" +" ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443" +" weight=10", +"95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90" +" ipv6=[2a02:ec0:209:10::4]:443" +" weight=10", +"89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C" +" weight=10", +"144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907" +" weight=10", +"94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073" +" weight=10", +"176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C" +" weight=10", +"46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C" +" ipv6=[2a03:b0c0:3:d0::208:5001]:9050" +" weight=10", +"185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778" +" weight=10", +"62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8" +" weight=10", +"82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98" +" ipv6=[2001:470:53e0::cafe]:9050" +" weight=10", +"192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F" +" weight=10", +"192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7" +" weight=10", +"84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34" +" weight=10", +"163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D" +" weight=10", +"185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE" +" weight=10", +"93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F" +" weight=10", +"178.217.184.32:9030 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D" +" weight=10", +"151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605" +" ipv6=[2001:41d0:e:f67::114]:9001" +" weight=10", +"5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E" +" ipv6=[2001:1af8:4700:a012:1::1]:443" +" weight=10", +"51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A" +" weight=10", +"91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5" +" weight=10", +"46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686" +" ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050" +" weight=10", +"85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0" +" weight=10", +"81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7" +" weight=10", +"46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E" +" weight=10", +"46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E" +" weight=10", +"171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A" +" ipv6=[2001:67c:289c:3::77]:443" +" weight=10", +"176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911" +" weight=10", +"192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8" +" weight=10", +"195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C" +" ipv6=[2001:bc8:399f:f000::1]:993" +" weight=10", +"86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0" +" weight=10", +"163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE" +" ipv6=[2001:bc8:4400:2100::2:1009]:443" +" weight=10", +"178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2" +" weight=10", +"37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05" +" ipv6=[2001:41d0:a:74a::1]:443" +" weight=10", +"212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7" +" weight=10", +"185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529" +" weight=10", +"193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E" +" weight=10", +"178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C" +" weight=10", +"197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626" +" weight=10", +"198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E" +" weight=10", +"148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8" +" ipv6=[2a01:4f8:211:c68::2]:9010" +" weight=10", +"188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788" +" weight=10", +"195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F" +" weight=10", +"37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7" +" weight=10", +"91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC" +" weight=10", "178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE" " ipv6=[2a03:b0c0:2:d0::b7:5001]:443" -" weight=50500", -/* -pixelminer -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 44800 / 704514 (6.359%) -Christian Sturm <reezer AT pixelminers dot net> - 1Q3PQJTELv33S1nruGcTUMQ7CuWxXmnjkZ -*/ -"81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304" -" ipv6=[2a02:180:a:51::dead]:443" -" weight=44800", -/* -bakunin -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 41800 / 704514 (5.933%) -GTor <contact _AT_ gtor _DOT_ org> -*/ -"178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A" -" ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443" -" weight=41800", -/* -kili -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 23800 / 704514 (3.378%) -0x49CBC553 Joost Rijneveld <joost AT joostrijneveld dot nl> -*/ +" weight=10", +"81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820" +" ipv6=[2a02:180:1:1::517:11ab]:443" +" weight=10", +"134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE" +" weight=10", +"185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361" +" weight=10", +"37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A" +" weight=10", +"185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260" +" weight=10", +"37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811" +" weight=10", +"193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4" +" weight=10", "178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10" " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" -" weight=23800", -/* -PedicaboMundi -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 23000 / 704514 (3.265%) -0x43DE8191 - 12LiRiasTEL346ZFjgCh5e3nBexQuvDBTg -*/ -"144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB" -" ipv6=[2a01:4f8:190:9490::dead]:443" -" weight=23000", -/* -Doedel26 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 22600 / 704514 (3.208%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377" -" weight=22600", -/* -Doedel24 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 20800 / 704514 (2.952%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.20.134:9030 orport=9001 id=2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC" -" weight=20800", -/* -Freebird31 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 20500 / 704514 (2.910%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE" -" weight=20500", -/* -rueckgrat -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 19200 / 704514 (2.725%) -Paul Staroch <paulchen AT rueckgr DOT at> - BTC 1G8pF66fnHc4n4oksY87pCN4TRXAV2Nqhh -*/ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -" weight=19200", -/* -coby -Flags: Fast Guard Running Stable V2Dir Valid -Fallback Weight: 16700 / 704514 (2.370%) -c0by <coby AT 127001 dot ovh> -*/ -"51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109" -" weight=16700", -/* -Logforme -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 16000 / 704514 (2.271%) -Logforme <m7527 AT abc dot se> -*/ -"84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34" -" weight=16000", -/* -12xBTME1 -Flags: Exit Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 15300 / 704514 (2.172%) -12xBTM(a)gmail.com - 12xBTMNArLvKXqvbsbyVhpPQfzUDuUaPGP -*/ -"81.7.17.171:80 orport=443 id=00C4B4731658D3B4987132A3F77100CFCB190D97" -" ipv6=[2a02:180:1:1::517:11ab]:443" -" weight=15300", -/* -Doedel21 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 14800 / 704514 (2.101%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2" -" weight=14800", -/* -Binnacle -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 13700 / 704514 (1.945%) -starlight dot YYYYqQ at binnacle dot cx -*/ -"108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2" -" weight=13700", -/* -Freebird32 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 13200 / 704514 (1.874%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.13.126:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5" -" weight=13200", -/* -eriador -Flags: Fast Guard Running Stable V2Dir Valid -Fallback Weight: 12400 / 704514 (1.760%) -hwertiout695(a)safe-mail.net -*/ -"85.25.138.93:9030 orport=4029 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F" -" weight=12400", -/* -Nurnberg04 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 11600 / 704514 (1.647%) -Please Donate <tor AT use.startmail dot com> - 1GuD8FxCnTqYGeRbx4MceYPhMLNTKDTsTT -*/ -"88.198.38.226:22 orport=443 id=4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB" -" ipv6=[2a01:4f8:a0:1351::2]:80" -" weight=11600", -/* -Nurnberg03 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 11000 / 704514 (1.561%) -Please Donate <tor AT use.startmail dot com> - 1GuD8FxCnTqYGeRbx4MceYPhMLNTKDTsTT -*/ -"85.10.201.38:22 orport=443 id=F6279A203C1950ACF592322A235647A05BFBCF91" -" ipv6=[2a01:4f8:a0:43cc::2]:80" -" weight=11000", -/* -Doedel22 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 10600 / 704514 (1.505%) -Felix <zwiebel ta quantentunnel tod de> -*/ -"178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422" -" weight=10600", -/* -Nurnberg01 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 10500 / 704514 (1.490%) -Please Donate <tor AT use.startmail dot com> - 1GuD8FxCnTqYGeRbx4MceYPhMLNTKDTsTT -*/ -"213.239.210.204:22 orport=443 id=5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9" -" ipv6=[2a01:4f8:a0:9474::2]:80" -" weight=10500", -/* -kitten4 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 10400 / 704514 (1.476%) -0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd -*/ -"212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88" -" weight=10400", -/* -Unnamed -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 9780 / 704514 (1.388%) -monitor0penmailbox0rg -*/ -"217.12.210.214:80 orport=443 id=943C0C6841C1E914B9FCA796C6846620A5AF9BC7" -" weight=9780", -/* -Nurnberg02 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 9490 / 704514 (1.347%) -Please Donate <tor AT use.startmail dot com> - 1GuD8FxCnTqYGeRbx4MceYPhMLNTKDTsTT -*/ -"213.239.220.25:22 orport=443 id=BEE2317AE127EB681C5AE1551C1EA0630580638A" -" ipv6=[2a01:4f8:a0:710c::2]:80" -" weight=9490", -/* -kitten3 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 8640 / 704514 (1.226%) -0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd -*/ -"212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6" -" weight=8640", -/* -horizons -Flags: Fast Guard Running Stable V2Dir Valid -Fallback Weight: 7860 / 704514 (1.116%) -kbesig(a)socal.rr.com -*/ -"167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48" -" weight=7860", -/* -wagner -Flags: Fast Guard Running Stable V2Dir Valid -Fallback Weight: 7700 / 704514 (1.093%) -Rarely used email <trff914 AT gmail DOT com> -*/ -"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" -" weight=7700", -/* -Unnamed -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 7650 / 704514 (1.086%) -monitor0penmailbox0rg -*/ -"217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05" -" weight=7650", -/* -ratchet -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 4450 / 704514 (0.632%) -0x1EB4B68F Sam Lanning <sam AT samlanning dot com> -*/ -"170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD" -" weight=4450", +" weight=10", +"5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8" +" weight=10", +"195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197" +" weight=10", +"46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D" +" weight=10", +"176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329" +" ipv6=[2001:41d0:8:eb9d::1]:22" +" weight=10", +"131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8" +" weight=10", +"91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99" +" weight=10", +"94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0" +" ipv6=[2a01:608:ffff:ff07::1:23]:9003" +" weight=10", +"46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB" +" weight=10", +"195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64" +" weight=10", +"192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC" +" weight=10", +"193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07" +" ipv6=[2001:6b0:7:125::243]:9001" +" weight=10",

1 0

[tor/master] Report fallback directory detail changes when rebuilding list
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit ba7691071e284623cc0c6e3e8ca8ff94d4be7ded Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Sat Apr 16 01:43:11 2016 +1000 Report fallback directory detail changes when rebuilding list As well as the existing reports of IPv6 address additions or removals, the script now warns when keys change but IPv4:ORPort or IPv6:IPv6ORPort remain the same. Existing checks for other whitelist detail changes have also been re-worded and upgraded to warnings. This makes it easier for changes to be identified so operators can be contacted to confirm whether the change is stable. --- changes/fallbacks-201604 | 5 ++-- scripts/maint/updateFallbackDirs.py | 46 ++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/changes/fallbacks-201604 b/changes/fallbacks-201604 index f88d5c6..d61615a 100644 --- a/changes/fallbacks-201604 +++ b/changes/fallbacks-201604 @@ -1,8 +1,9 @@ o Minor enhancements (fallback directory mirrors): - Give each fallback the same weight for client selection. Restrict fallbacks to one per operator. + Report fallback directory detail changes when rebuilding list. Add new fallback directory mirrors to the whitelist. Update fallback directories based on the latest OnionOO data. Many other minor simplifications and fixes. - Closes tasks 17158, 17905, 18749, and bug 18689 on - tor 0.2.8.1-alpha; patch by "teor". + Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of + bug 18812 on tor 0.2.8.1-alpha; patch by "teor". diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 4cfee5d..d27c144 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -806,48 +806,48 @@ class Candidate(object): ipv6 address and port (if present) If the fallback has an ipv6 key, the whitelist line must also have it, and vice versa, otherwise they don't match. """ + ipv6 = None + if self.has_ipv6(): + ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here, every relay's fingerprint is compared to the entry + if entry['id'] != self._fpr: + # can't log here unless we match an IP and port, because every relay's + # fingerprint is compared to every entry's fingerprint + if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: + logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + + '%s?', entry['id'], self.dirip, self.orport, + self._fpr) + if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: + logging.warning('%s excluded: has OR %s changed fingerprint to ' + + '%s?', entry['id'], ipv6, self._fpr) continue if entry['ipv4'] != self.dirip: - logging.info('%s is not in the whitelist: fingerprint matches, but ' + - 'IPv4 (%s) does not match entry IPv4 (%s)', - self._fpr, self.dirip, entry['ipv4']) + logging.warning('%s excluded: has it changed IPv4 from %s to %s?', + self._fpr, entry['ipv4'], self.dirip) continue if int(entry['dirport']) != self.dirport: - logging.info('%s is not in the whitelist: fingerprint matches, but ' + - 'DirPort (%d) does not match entry DirPort (%d)', - self._fpr, self.dirport, int(entry['dirport'])) + logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), + self.dirip, self.dirport) continue if int(entry['orport']) != self.orport: - logging.info('%s is not in the whitelist: fingerprint matches, but ' + - 'ORPort (%d) does not match entry ORPort (%d)', - self._fpr, self.orport, int(entry['orport'])) + logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['orport']), + self.dirip, self.orport) continue - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) if entry.has_key('ipv6') and self.has_ipv6(): # if both entry and fallback have an ipv6 address, compare them if entry['ipv6'] != ipv6: - logging.info('%s is not in the whitelist: fingerprint matches, ' + - 'but IPv6 (%s) does not match entry IPv6 (%s)', - self._fpr, ipv6, entry['ipv6']) + logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + + 'to %s?', self._fpr, entry['ipv6'], ipv6) continue # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.info('%s is not in the whitelist: fingerprint matches, but ' + - 'it has no IPv6, and entry has IPv6 (%s)', self._fpr, - entry['ipv6']) logging.warning('%s excluded: has it lost its former IPv6 address %s?', self._fpr, entry['ipv6']) continue elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.info('%s is not in the whitelist: fingerprint matches, but ' + - 'it has IPv6 (%s), and entry has no IPv6', self._fpr, - ipv6) logging.warning('%s excluded: has it gained an IPv6 address %s?', self._fpr, ipv6) continue

1 0

[tor/master] Limit fallbacks from the same operator
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit c157a31ee8bd84587e6e61b674b33c792154d74a Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Apr 14 18:40:06 2016 +1000 Limit fallbacks from the same operator Use IP address, effective family, and contact info to discover and limit fallbacks to one per operator. Also analyse netblock, ports, IP version, and Exit flag, and print the results. Don't exclude any fallbacks from the list because of netblocks, ports, IP version, or Exit flag. --- scripts/maint/updateFallbackDirs.py | 550 +++++++++++++++++++++++++++++++++--- 1 file changed, 510 insertions(+), 40 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 793ec7d..4cfee5d 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -2,9 +2,12 @@ # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc # Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . +# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) +# for netblock analysis, in PYTHONPATH, or just +# ln -s ../py2-ipaddress-3.4.1/ipaddress.py . # -# Then read the generated list to ensure no-one slipped anything funny into -# their name or contactinfo +# Then read the logs to make sure the fallbacks aren't dominated by a single +# netblock or port # Script by weasel, April 2015 # Portions by gsathya & karsten, 2013 @@ -34,6 +37,21 @@ import logging # INFO tells you why each relay was included or excluded # WARN tells you about potential misconfigurations logging.basicConfig(level=logging.WARNING) +logging.root.name = '' +# INFO tells you about each consensus download attempt +logging.getLogger('stem').setLevel(logging.WARNING) + +HAVE_IPADDRESS = False +try: + # python 3 builtin, or install package py2-ipaddress + # there are several ipaddress implementations for python 2 + # with slightly different semantics with str typed text + # fortunately, all our IP addresses are in unicode + import ipaddress + HAVE_IPADDRESS = True +except ImportError: + # if this happens, we avoid doing netblock analysis + logging.warning('Unable to import ipaddress, please install py2-ipaddress') ## Top-Level Configuration @@ -468,6 +486,9 @@ class Candidate(object): # relays without advertised bandwdith have it calculated from their # consensus weight details['advertised_bandwidth'] = 0 + if (not 'effective_family' in details + or details['effective_family'] is None): + details['effective_family'] = [] details['last_changed_address_or_port'] = parse_ts( details['last_changed_address_or_port']) self._data = details @@ -480,7 +501,7 @@ class Candidate(object): if self.orport is None: raise Exception("Failed to get an orport for %s."%(self._fpr,)) self._compute_ipv6addr() - if self.ipv6addr is None: + if not self.has_ipv6(): logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) def _stable_sort_or_addresses(self): @@ -584,14 +605,14 @@ class Candidate(object): (ipaddr, port) = i.rsplit(':', 1) if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr): self.ipv6addr = ipaddr - self.ipv6orport = port + self.ipv6orport = int(port) return # Choose the first IPv6 address in the list for i in self._data['or_addresses']: (ipaddr, port) = i.rsplit(':', 1) if Candidate.is_valid_ipv6_address(ipaddr): self.ipv6addr = ipaddr - self.ipv6orport = port + self.ipv6orport = int(port) return @staticmethod @@ -804,9 +825,10 @@ class Candidate(object): 'ORPort (%d) does not match entry ORPort (%d)', self._fpr, self.orport, int(entry['orport'])) continue - has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None - if (entry.has_key('ipv6') and has_ipv6): - ipv6 = self.ipv6addr + ':' + self.ipv6orport + ipv6 = None + if self.has_ipv6(): + ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) + if entry.has_key('ipv6') and self.has_ipv6(): # if both entry and fallback have an ipv6 address, compare them if entry['ipv6'] != ipv6: logging.info('%s is not in the whitelist: fingerprint matches, ' + @@ -815,14 +837,14 @@ class Candidate(object): continue # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not has_ipv6: + elif entry.has_key('ipv6') and not self.has_ipv6(): logging.info('%s is not in the whitelist: fingerprint matches, but ' + 'it has no IPv6, and entry has IPv6 (%s)', self._fpr, entry['ipv6']) logging.warning('%s excluded: has it lost its former IPv6 address %s?', self._fpr, entry['ipv6']) continue - elif not entry.has_key('ipv6') and has_ipv6: + elif not entry.has_key('ipv6') and self.has_ipv6(): logging.info('%s is not in the whitelist: fingerprint matches, but ' + 'it has IPv6 (%s), and entry has no IPv6', self._fpr, ipv6) @@ -871,9 +893,10 @@ class Candidate(object): 'entry has no DirPort or ORPort', self._fpr, self.dirip) return True - has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None - ipv6 = (self.ipv6addr + ':' + self.ipv6orport) if has_ipv6 else None - if (key == 'ipv6' and has_ipv6): + ipv6 = None + if self.has_ipv6(): + ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) + if (key == 'ipv6' and self.has_ipv6()): # if both entry and fallback have an ipv6 address, compare them, # otherwise, disregard ipv6 addresses if value == ipv6: @@ -889,18 +912,18 @@ class Candidate(object): logging.info('%s is in the blacklist: IPv6 (%s) matches, and' + 'entry has no DirPort', self._fpr, ipv6) return True - elif (key == 'ipv6' or has_ipv6): + elif (key == 'ipv6' or self.has_ipv6()): # only log if the fingerprint matches but the IPv6 doesn't if entry.has_key('id') and entry['id'] == self._fpr: logging.info('%s skipping IPv6 blacklist comparison: relay ' + 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, - '' if has_ipv6 else ' no', - (' (' + ipv6 + ')') if has_ipv6 else '', + '' if self.has_ipv6() else ' no', + (' (' + ipv6 + ')') if self.has_ipv6() else '', '' if key == 'ipv6' else ' no', (' (' + value + ')') if key == 'ipv6' else '') logging.warning('Has %s %s IPv6 address %s?', self._fpr, - 'gained an' if has_ipv6 else 'lost its former', - ipv6 if has_ipv6 else value) + 'gained an' if self.has_ipv6() else 'lost its former', + ipv6 if self.has_ipv6() else value) return False def cw_to_bw_factor(self): @@ -936,6 +959,101 @@ class Candidate(object): def is_running(self): return 'Running' in self._data['flags'] + # does this fallback have an IPv6 address and orport? + def has_ipv6(self): + return self.ipv6addr is not None and self.ipv6orport is not None + + # strip leading and trailing brackets from an IPv6 address + # safe to use on non-bracketed IPv6 and on IPv4 addresses + # also convert to unicode, and make None appear as '' + @staticmethod + def strip_ipv6_brackets(ip): + if ip is None: + return unicode('') + if len(ip) < 2: + return unicode(ip) + if ip[0] == '[' and ip[-1] == ']': + return unicode(ip[1:-1]) + return unicode(ip) + + # are ip_a and ip_b in the same netblock? + # mask_bits is the size of the netblock + # takes both IPv4 and IPv6 addresses + # the versions of ip_a and ip_b must be the same + # the mask must be valid for the IP version + @staticmethod + def netblocks_equal(ip_a, ip_b, mask_bits): + if ip_a is None or ip_b is None: + return False + ip_a = Candidate.strip_ipv6_brackets(ip_a) + ip_b = Candidate.strip_ipv6_brackets(ip_b) + a = ipaddress.ip_address(ip_a) + b = ipaddress.ip_address(ip_b) + if a.version != b.version: + raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) + if mask_bits > a.max_prefixlen: + logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + mask_bits = a.max_prefixlen + if mask_bits < 0: + logging.warning('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) + mask_bits = 0 + a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) + return b in a_net + + # is this fallback's IPv4 address (dirip) in the same netblock as other's + # IPv4 address? + # mask_bits is the size of the netblock + def ipv4_netblocks_equal(self, other, mask_bits): + return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits) + + # is this fallback's IPv6 address (ipv6addr) in the same netblock as + # other's IPv6 address? + # Returns False if either fallback has no IPv6 address + # mask_bits is the size of the netblock + def ipv6_netblocks_equal(self, other, mask_bits): + if not self.has_ipv6() or not other.has_ipv6(): + return False + return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits) + + # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort? + def dirport_equal(self, other): + return self.dirport == other.dirport + + # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort? + def ipv4_orport_equal(self, other): + return self.orport == other.orport + + # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort? + # Returns False if either fallback has no IPv6 address + def ipv6_orport_equal(self, other): + if not self.has_ipv6() or not other.has_ipv6(): + return False + return self.ipv6orport == other.ipv6orport + + # does this fallback have the same DirPort, IPv4 ORPort, or + # IPv6 ORPort as other? + # Ignores IPv6 ORPort if either fallback has no IPv6 address + def port_equal(self, other): + return (self.dirport_equal(other) or self.ipv4_orport_equal(other) + or self.ipv6_orport_equal(other)) + + # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present) + def port_list(self): + ports = [self.dirport, self.orport] + if self.has_ipv6() and not self.ipv6orport in ports: + ports.append(self.ipv6orport) + return ports + + # does this fallback share a port with other, regardless of whether the + # port types match? + # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80, + # return True + def port_shared(self, other): + for p in self.port_list(): + if p in other.port_list(): + return True + return False + # report how long it takes to download a consensus from dirip:dirport @staticmethod def fallback_consensus_download_speed(dirip, dirport, nickname, max_time): @@ -984,7 +1102,7 @@ class Candidate(object): self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS: + if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS: # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr, self.dirport, @@ -1086,9 +1204,8 @@ class Candidate(object): self.orport, cleanse_c_string(self._fpr)) s += '\n' - if self.ipv6addr is not None: - s += '" ipv6=%s:%s"'%( - cleanse_c_string(self.ipv6addr), cleanse_c_string(self.ipv6orport)) + if self.has_ipv6(): + s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport) s += '\n' s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT) if comment_string: @@ -1126,7 +1243,7 @@ class CandidateList(dict): d = fetch('details', fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + 'consensus_weight,advertised_bandwidth,or_addresses,' + - 'dir_address,recommended_version,flags')) + 'dir_address,recommended_version,flags,effective_family')) logging.debug('Loading details document done.') if not 'relays' in d: raise Exception("No relays found in document.") @@ -1163,19 +1280,19 @@ class CandidateList(dict): # lowest to highest # used to find the median cw_to_bw_factor() def sort_fallbacks_by_cw_to_bw_factor(self): - self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor(), self.fallbacks) + self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor()) # sort fallbacks by their measured bandwidth, highest to lowest # calculate_measured_bandwidth before calling this # this is useful for reviewing candidates in priority order def sort_fallbacks_by_measured_bandwidth(self): self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'], - self.fallbacks, reverse=True) + reverse=True) # sort fallbacks by their fingerprint, lowest to highest # this is useful for stable diffs of fallback lists def sort_fallbacks_by_fingerprint(self): - self.fallbacks.sort(key=lambda f: self[f]._fpr, self.fallbacks) + self.fallbacks.sort(key=lambda f: f._fpr) @staticmethod def load_relaylist(file_name): @@ -1341,6 +1458,91 @@ class CandidateList(dict): else: return None + # does exclusion_list contain attribute? + # if so, return False + # if not, return True + # if attribute is None or the empty string, always return True + @staticmethod + def allow(attribute, exclusion_list): + if attribute is None or attribute == '': + return True + elif attribute in exclusion_list: + return False + else: + return True + + # make sure there is only one fallback per IPv4 address, and per IPv6 address + # there is only one IPv4 address on each fallback: the IPv4 DirPort address + # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort) + # there is at most one IPv6 address on each fallback: the IPv6 ORPort address + # we try to match the IPv4 ORPort, but will use any IPv6 address if needed + # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but + # typically only use the IPv6 ORPort) + # if there is no IPv6 address, only the IPv4 address is checked + # return the number of candidates we excluded + def limit_fallbacks_same_ip(self): + ip_limit_fallbacks = [] + ip_list = [] + for f in self.fallbacks: + if (CandidateList.allow(f.dirip, ip_list) + and CandidateList.allow(f.ipv6addr, ip_list)): + ip_limit_fallbacks.append(f) + ip_list.append(f.dirip) + if f.has_ipv6(): + ip_list.append(f.ipv6addr) + elif not CandidateList.allow(f.dirip, ip_list): + logging.debug('Eliminated %s: already have fallback on IPv4 %s'%( + f._fpr, f.dirip)) + elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list): + logging.debug('Eliminated %s: already have fallback on IPv6 %s'%( + f._fpr, f.ipv6addr)) + original_count = len(self.fallbacks) + self.fallbacks = ip_limit_fallbacks + return original_count - len(self.fallbacks) + + # make sure there is only one fallback per ContactInfo + # if there is no ContactInfo, allow the fallback + # this check can be gamed by providing no ContactInfo, or by setting the + # ContactInfo to match another fallback + # However, given the likelihood that relays with the same ContactInfo will + # go down at similar times, its usefulness outweighs the risk + def limit_fallbacks_same_contact(self): + contact_limit_fallbacks = [] + contact_list = [] + for f in self.fallbacks: + if CandidateList.allow(f._data['contact'], contact_list): + contact_limit_fallbacks.append(f) + contact_list.append(f._data['contact']) + else: + logging.debug(('Eliminated %s: already have fallback on ' + + 'ContactInfo %s')%(f._fpr, f._data['contact'])) + original_count = len(self.fallbacks) + self.fallbacks = contact_limit_fallbacks + return original_count - len(self.fallbacks) + + # make sure there is only one fallback per effective family + # if there is no family, allow the fallback + # this check can't be gamed, because we use effective family, which ensures + # mutual family declarations + # if any indirect families exist, the result depends on the order in which + # fallbacks are sorted in the list + def limit_fallbacks_same_family(self): + family_limit_fallbacks = [] + fingerprint_list = [] + for f in self.fallbacks: + if CandidateList.allow(f._fpr, fingerprint_list): + family_limit_fallbacks.append(f) + fingerprint_list.append(f._fpr) + fingerprint_list.extend(f._data['effective_family']) + else: + # technically, we already have a fallback with this fallback in its + # effective family + logging.debug('Eliminated %s: already have fallback in effective ' + + 'family'%(f._fpr)) + original_count = len(self.fallbacks) + self.fallbacks = family_limit_fallbacks + return original_count - len(self.fallbacks) + # try a download check on each fallback candidate in order # stop after max_count successful downloads # but don't remove any candidates from the array @@ -1361,6 +1563,7 @@ class CandidateList(dict): # - eliminate failed candidates # - if there are more than max_count candidates, eliminate lowest bandwidth # - if there are fewer than max_count candidates, leave only successful + # Return the number of fallbacks that failed the consensus check def perform_download_consensus_checks(self, max_count): self.sort_fallbacks_by_measured_bandwidth() self.try_download_consensus_checks(max_count) @@ -1370,12 +1573,245 @@ class CandidateList(dict): self.try_download_consensus_checks(max_count) # now we have at least max_count successful candidates, # or we've tried them all + original_count = len(self.fallbacks) self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), self.fallbacks) + # some of these failed the check, others skipped the check, + # if we already had enough successful downloads + failed_count = original_count - len(self.fallbacks) self.fallbacks = self.fallbacks[:max_count] + return failed_count + + # return a string that describes a/b as a percentage + @staticmethod + def describe_percentage(a, b): + return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b) + + # return a dictionary of lists of fallbacks by IPv4 netblock + # the dictionary is keyed by the fingerprint of an arbitrary fallback + # in each netblock + # mask_bits is the size of the netblock + def fallbacks_by_ipv4_netblock(self, mask_bits): + netblocks = {} + for f in self.fallbacks: + found_netblock = False + for b in netblocks.keys(): + # we found an existing netblock containing this fallback + if f.ipv4_netblocks_equal(self[b], mask_bits): + # add it to the list + netblocks[b].append(f) + found_netblock = True + break + # make a new netblock based on this fallback's fingerprint + if not found_netblock: + netblocks[f._fpr] = [f] + return netblocks + + # return a dictionary of lists of fallbacks by IPv6 netblock + # where mask_bits is the size of the netblock + def fallbacks_by_ipv6_netblock(self, mask_bits): + netblocks = {} + for f in self.fallbacks: + # skip fallbacks without IPv6 addresses + if not f.has_ipv6(): + continue + found_netblock = False + for b in netblocks.keys(): + # we found an existing netblock containing this fallback + if f.ipv6_netblocks_equal(self[b], mask_bits): + # add it to the list + netblocks[b].append(f) + found_netblock = True + break + # make a new netblock based on this fallback's fingerprint + if not found_netblock: + netblocks[f._fpr] = [f] + return netblocks + + # log a message about the proportion of fallbacks in each IPv4 netblock, + # where mask_bits is the size of the netblock + def describe_fallback_ipv4_netblock_mask(self, mask_bits): + fallback_count = len(self.fallbacks) + shared_netblock_fallback_count = 0 + most_frequent_netblock = None + netblocks = self.fallbacks_by_ipv4_netblock(mask_bits) + for b in netblocks.keys(): + if len(netblocks[b]) > 1: + # how many fallbacks are in a netblock with other fallbacks? + shared_netblock_fallback_count += len(netblocks[b]) + # what's the netblock with the most fallbacks? + if (most_frequent_netblock is None + or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): + most_frequent_netblock = b + logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits)) + for f in netblocks[b]: + logging.debug('%s - %s', f.dirip, f._fpr) + if most_frequent_netblock is not None: + logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%( + CandidateList.describe_percentage( + len(netblocks[most_frequent_netblock]), + fallback_count), + mask_bits, + self[most_frequent_netblock].dirip)) + if shared_netblock_fallback_count > 0: + logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' + + 'fallbacks')%(CandidateList.describe_percentage( + shared_netblock_fallback_count, + fallback_count), + mask_bits)) + + # log a message about the proportion of fallbacks in each IPv6 netblock, + # where mask_bits is the size of the netblock + def describe_fallback_ipv6_netblock_mask(self, mask_bits): + fallback_count = len(self.fallbacks_with_ipv6()) + shared_netblock_fallback_count = 0 + most_frequent_netblock = None + netblocks = self.fallbacks_by_ipv6_netblock(mask_bits) + for b in netblocks.keys(): + if len(netblocks[b]) > 1: + # how many fallbacks are in a netblock with other fallbacks? + shared_netblock_fallback_count += len(netblocks[b]) + # what's the netblock with the most fallbacks? + if (most_frequent_netblock is None + or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): + most_frequent_netblock = b + logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits)) + for f in netblocks[b]: + logging.debug('%s - %s', f.ipv6addr, f._fpr) + if most_frequent_netblock is not None: + logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%( + CandidateList.describe_percentage( + len(netblocks[most_frequent_netblock]), + fallback_count), + mask_bits, + self[most_frequent_netblock].ipv6addr)) + if shared_netblock_fallback_count > 0: + logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' + + 'fallbacks')%(CandidateList.describe_percentage( + shared_netblock_fallback_count, + fallback_count), + mask_bits)) + + # log a message about the proportion of fallbacks in each IPv4 /8, /16, + # and /24 + def describe_fallback_ipv4_netblocks(self): + # this doesn't actually tell us anything useful + #self.describe_fallback_ipv4_netblock_mask(8) + self.describe_fallback_ipv4_netblock_mask(16) + self.describe_fallback_ipv4_netblock_mask(24) + + # log a message about the proportion of fallbacks in each IPv6 /12 (RIR), + # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host) + # https://www.iana.org/assignments/ipv6-unicast-address-assignments/ + def describe_fallback_ipv6_netblocks(self): + # these don't actually tell us anything useful + #self.describe_fallback_ipv6_netblock_mask(12) + #self.describe_fallback_ipv6_netblock_mask(23) + self.describe_fallback_ipv6_netblock_mask(32) + self.describe_fallback_ipv6_netblock_mask(48) + self.describe_fallback_ipv6_netblock_mask(64) + + # log a message about the proportion of fallbacks in each IPv4 and IPv6 + # netblock + def describe_fallback_netblocks(self): + self.describe_fallback_ipv4_netblocks() + self.describe_fallback_ipv6_netblocks() + + # return a list of fallbacks which are on the IPv4 ORPort port + def fallbacks_on_ipv4_orport(self, port): + return filter(lambda x: x.orport == port, self.fallbacks) + + # return a list of fallbacks which are on the IPv6 ORPort port + def fallbacks_on_ipv6_orport(self, port): + return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6()) + + # return a list of fallbacks which are on the DirPort port + def fallbacks_on_dirport(self, port): + return filter(lambda x: x.dirport == port, self.fallbacks) + + # log a message about the proportion of fallbacks on IPv4 ORPort port + # and return that count + def describe_fallback_ipv4_orport(self, port): + port_count = len(self.fallbacks_on_ipv4_orport(port)) + fallback_count = len(self.fallbacks) + logging.warning('%s of fallbacks are on IPv4 ORPort %d'%( + CandidateList.describe_percentage(port_count, + fallback_count), + port)) + return port_count + + # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port + # and return that count + def describe_fallback_ipv6_orport(self, port): + port_count = len(self.fallbacks_on_ipv6_orport(port)) + fallback_count = len(self.fallbacks_with_ipv6()) + logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%( + CandidateList.describe_percentage(port_count, + fallback_count), + port)) + return port_count + + # log a message about the proportion of fallbacks on DirPort port + # and return that count + def describe_fallback_dirport(self, port): + port_count = len(self.fallbacks_on_dirport(port)) + fallback_count = len(self.fallbacks) + logging.warning('%s of fallbacks are on DirPort %d'%( + CandidateList.describe_percentage(port_count, + fallback_count), + port)) + return port_count + + # log a message about the proportion of fallbacks on each dirport, + # each IPv4 orport, and each IPv6 orport + def describe_fallback_ports(self): + fallback_count = len(self.fallbacks) + ipv4_or_count = fallback_count + ipv4_or_count -= self.describe_fallback_ipv4_orport(443) + ipv4_or_count -= self.describe_fallback_ipv4_orport(9001) + logging.warning('%s of fallbacks are on other IPv4 ORPorts'%( + CandidateList.describe_percentage(ipv4_or_count, + fallback_count))) + ipv6_fallback_count = len(self.fallbacks_with_ipv6()) + ipv6_or_count = ipv6_fallback_count + ipv6_or_count -= self.describe_fallback_ipv6_orport(443) + ipv6_or_count -= self.describe_fallback_ipv6_orport(9001) + logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%( + CandidateList.describe_percentage(ipv6_or_count, + ipv6_fallback_count))) + dir_count = fallback_count + dir_count -= self.describe_fallback_dirport(80) + dir_count -= self.describe_fallback_dirport(9030) + logging.warning('%s of fallbacks are on other DirPorts'%( + CandidateList.describe_percentage(dir_count, + fallback_count))) + + # return a list of fallbacks which have the Exit flag + def fallbacks_with_exit(self): + return filter(lambda x: x.is_exit(), self.fallbacks) + + # log a message about the proportion of fallbacks with an Exit flag + def describe_fallback_exit_flag(self): + exit_falback_count = len(self.fallbacks_with_exit()) + fallback_count = len(self.fallbacks) + logging.warning('%s of fallbacks have the Exit flag'%( + CandidateList.describe_percentage(exit_falback_count, + fallback_count))) + + # return a list of fallbacks which have an IPv6 address + def fallbacks_with_ipv6(self): + return filter(lambda x: x.has_ipv6(), self.fallbacks) + + # log a message about the proportion of fallbacks on IPv6 + def describe_fallback_ip_family(self): + ipv6_falback_count = len(self.fallbacks_with_ipv6()) + fallback_count = len(self.fallbacks) + logging.warning('%s of fallbacks are on IPv6'%( + CandidateList.describe_percentage(ipv6_falback_count, + fallback_count))) - def summarise_fallbacks(self, eligible_count, guard_count, target_count, - max_count): + def summarise_fallbacks(self, eligible_count, operator_count, failed_count, + guard_count, target_count): # Report: # whether we checked consensus download times # the number of fallback directories (and limits/exclusions, if relevant) @@ -1399,17 +1835,23 @@ class CandidateList(dict): if FALLBACK_PROPORTION_OF_GUARDS is None: fallback_proportion = '' else: - fallback_proportion = ', Target %d (%d * %f)'%(target_count, guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d%s'%(fallback_count, - eligible_count, + fallback_proportion = ', Target %d (%d * %.2f)'%(target_count, + guard_count, + FALLBACK_PROPORTION_OF_GUARDS) + s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count, fallback_proportion) if MAX_FALLBACK_COUNT is not None: - s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) + s += ', Max %d'%(MAX_FALLBACK_COUNT) s += ')\n' if eligible_count != fallback_count: - s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( - eligible_count - fallback_count) + removed_count = eligible_count - fallback_count + excess_to_target_or_max = (eligible_count - operator_count - failed_count + - fallback_count) + # some 'Failed' failed the check, others 'Skipped' the check, + # if we already had enough successful downloads + s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' + + 'Excess %d)')%(removed_count, operator_count, failed_count, + excess_to_target_or_max) s += '\n' min_fb = self.fallback_min() min_bw = min_fb._data['measured_bandwidth'] @@ -1473,18 +1915,46 @@ def list_fallbacks(): # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, # separators=(',', ': '), default=json_util.default) + # impose mandatory conditions here, like one per contact, family, IP + # in measured bandwidth order + candidates.sort_fallbacks_by_measured_bandwidth() + operator_count = 0 + # only impose these limits on the final list - operators can nominate + # multiple candidate fallbacks, and then we choose the best set + if not OUTPUT_CANDIDATES: + operator_count += candidates.limit_fallbacks_same_ip() + operator_count += candidates.limit_fallbacks_same_contact() + operator_count += candidates.limit_fallbacks_same_family() + + # check if each candidate can serve a consensus + # there's a small risk we've eliminated relays from the same operator that + # can serve a consensus, in favour of one that can't + # but given it takes up to 15 seconds to check each consensus download, + # the risk is worth it + failed_count = candidates.perform_download_consensus_checks(max_count) + + # analyse and log interesting diversity metrics + # like netblock, ports, exit, IPv4-only + # (we can't easily analyse AS, and it's hard to accurately analyse country) + candidates.describe_fallback_ip_family() + # if we can't import the ipaddress module, we can't do netblock analysis + if HAVE_IPADDRESS: + candidates.describe_fallback_netblocks() + candidates.describe_fallback_ports() + candidates.describe_fallback_exit_flag() + + # output C comments summarising the fallback selection process if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, guard_count, - target_count, max_count) + print candidates.summarise_fallbacks(eligible_count, operator_count, + failed_count, guard_count, + target_count) else: print '/* No Fallbacks met criteria */' + # output C comments specifying the OnionOO data used to create the list for s in fetch_source_list(): print describe_fetch_source(s) - # check if each candidate can serve a consensus - candidates.perform_download_consensus_checks(max_count) - # if we're outputting the final fallback list, sort by fingerprint # this makes diffs much more stable # otherwise, leave sorted by bandwidth, which allows operators to be

1 0

[tor/master] Simplify fallback list output
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 999834324b47f920473b5e28a11b1833d6543d7c Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Mon Apr 11 14:32:57 2016 +1000 Simplify fallback list output When creating the list of fallbacks for a release: * don't output fallback name and contact * sort fallbacks by fingerprint --- scripts/maint/updateFallbackDirs.py | 151 ++++++++++++++++++++++++++---------- 1 file changed, 111 insertions(+), 40 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 44a7318..793ec7d 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -53,6 +53,9 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False +# Output fallback name, flags, and ContactInfo in a C comment? +OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False + # Output matching ContactInfo in fallbacks list or the blacklist? # Useful if you're trying to contact operators CONTACT_COUNT = True if OUTPUT_CANDIDATES else False @@ -933,8 +936,9 @@ class Candidate(object): def is_running(self): return 'Running' in self._data['flags'] + # report how long it takes to download a consensus from dirip:dirport @staticmethod - def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time): + def fallback_consensus_download_speed(dirip, dirport, nickname, max_time): download_failed = False downloader = DescriptorDownloader() start = datetime.datetime.utcnow() @@ -970,47 +974,60 @@ class Candidate(object): dirip, dirport, max_time) return download_failed - def fallback_consensus_dl_check(self): + # does this fallback download the consensus fast enough? + def check_fallback_download_consensus(self): # include the relay if we're not doing a check, or we can't check (IPv6) ipv4_failed = False ipv6_failed = False if PERFORM_IPV4_DIRPORT_CHECKS: - ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, + ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS: # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, - self.dirport, - self._data['nickname'], - CONSENSUS_DOWNLOAD_SPEED_MAX) - # Now retry the relay if it took too long the first time - if (PERFORM_IPV4_DIRPORT_CHECKS and ipv4_failed - and CONSENSUS_DOWNLOAD_RETRY): - ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, - self.dirport, - self._data['nickname'], - CONSENSUS_DOWNLOAD_SPEED_MAX) - if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS - and ipv6_failed and CONSENSUS_DOWNLOAD_RETRY): - ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) return ((not ipv4_failed) and (not ipv6_failed)) - def fallbackdir_line(self, dl_speed_ok, fallbacks, prefilter_fallbacks): + # if this fallback has not passed a download check, try it again, + # and record the result, available in get_fallback_download_consensus + def try_fallback_download_consensus(self): + if not self.get_fallback_download_consensus(): + self._data['download_check'] = self.check_fallback_download_consensus() + + # did this fallback pass the download check? + def get_fallback_download_consensus(self): + # if we're not performing checks, return True + if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS: + return True + # if we are performing checks, but haven't done one, return False + if not self._data.has_key('download_check'): + return False + return self._data['download_check'] + + # output an optional header comment and info for this fallback + # try_fallback_download_consensus before calling this + def fallbackdir_line(self, fallbacks, prefilter_fallbacks): + s = '' + if OUTPUT_COMMENTS: + s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks) + # if the download speed is ok, output a C string + # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string + if self.get_fallback_download_consensus() or OUTPUT_COMMENTS: + s += self.fallbackdir_info(self.get_fallback_download_consensus()) + return s + + # output a header comment for this fallback + def fallbackdir_comment(self, fallbacks, prefilter_fallbacks): # /* # nickname # flags # [contact] # [identical contact counts] # */ - # "address:dirport orport=port id=fingerprint" - # "[ipv6=addr:orport]" - # "weight=FALLBACK_OUTPUT_WEIGHT", - # # Multiline C comment s = '/*' s += '\n' @@ -1040,9 +1057,26 @@ class Candidate(object): s += '\n' s += '*/' s += '\n' + + # output the fallback info C string for this fallback + # this is the text that would go after FallbackDir in a torrc + # if this relay failed the download test and we OUTPUT_COMMENTS, + # comment-out the returned string + def fallbackdir_info(self, dl_speed_ok): + # "address:dirport orport=port id=fingerprint" + # "[ipv6=addr:orport]" + # "weight=FALLBACK_OUTPUT_WEIGHT", + # + # Do we want a C string, or a commented-out string? + c_string = dl_speed_ok + comment_string = not dl_speed_ok and OUTPUT_COMMENTS + # If we don't want either kind of string, bail + if not c_string and not comment_string: + return '' + s = '' # Comment out the fallback directory entry if it's too slow # See the debug output for which address and port is failing - if not dl_speed_ok: + if comment_string: s += '/* Consensus download failed or was too slow:\n' # Multi-Line C string with trailing comma (part of a string list) # This makes it easier to diff the file, and remove IPv6 lines using grep @@ -1057,7 +1091,7 @@ class Candidate(object): cleanse_c_string(self.ipv6addr), cleanse_c_string(self.ipv6orport)) s += '\n' s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT) - if not dl_speed_ok: + if comment_string: s += '\n' s += '*/' return s @@ -1129,13 +1163,19 @@ class CandidateList(dict): # lowest to highest # used to find the median cw_to_bw_factor() def sort_fallbacks_by_cw_to_bw_factor(self): - self.fallbacks.sort(key=lambda x: self[x].cw_to_bw_factor()) + self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor(), self.fallbacks) # sort fallbacks by their measured bandwidth, highest to lowest # calculate_measured_bandwidth before calling this + # this is useful for reviewing candidates in priority order def sort_fallbacks_by_measured_bandwidth(self): - self.fallbacks.sort(key=lambda x: self[x].self._data['measured_bandwidth'], - reverse=True) + self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'], + self.fallbacks, reverse=True) + + # sort fallbacks by their fingerprint, lowest to highest + # this is useful for stable diffs of fallback lists + def sort_fallbacks_by_fingerprint(self): + self.fallbacks.sort(key=lambda f: self[f]._fpr, self.fallbacks) @staticmethod def load_relaylist(file_name): @@ -1301,6 +1341,39 @@ class CandidateList(dict): else: return None + # try a download check on each fallback candidate in order + # stop after max_count successful downloads + # but don't remove any candidates from the array + def try_download_consensus_checks(self, max_count): + dl_ok_count = 0 + for f in self.fallbacks: + f.try_fallback_download_consensus() + if f.get_fallback_download_consensus(): + # this fallback downloaded a consensus ok + dl_ok_count += 1 + if dl_ok_count >= max_count: + # we have enough fallbacks + return + + # put max_count successful candidates in the fallbacks array: + # - perform download checks on each fallback candidate + # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set + # - eliminate failed candidates + # - if there are more than max_count candidates, eliminate lowest bandwidth + # - if there are fewer than max_count candidates, leave only successful + def perform_download_consensus_checks(self, max_count): + self.sort_fallbacks_by_measured_bandwidth() + self.try_download_consensus_checks(max_count) + if CONSENSUS_DOWNLOAD_RETRY: + # try unsuccessful candidates again + # we could end up with more than max_count successful candidates here + self.try_download_consensus_checks(max_count) + # now we have at least max_count successful candidates, + # or we've tried them all + self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), + self.fallbacks) + self.fallbacks = self.fallbacks[:max_count] + def summarise_fallbacks(self, eligible_count, guard_count, target_count, max_count): # Report: @@ -1393,9 +1466,6 @@ def list_fallbacks(): # then remove low-bandwidth relays candidates.calculate_measured_bandwidth() candidates.remove_low_bandwidth_relays() - # make sure the list is sorted by bandwidth when we output it - # so that we include the active fallbacks with the greatest bandwidth - candidates.sort_fallbacks_by_measured_bandwidth() # print the raw fallback list #for x in candidates.fallbacks: @@ -1412,17 +1482,18 @@ def list_fallbacks(): for s in fetch_source_list(): print describe_fetch_source(s) - active_count = 0 + # check if each candidate can serve a consensus + candidates.perform_download_consensus_checks(max_count) + + # if we're outputting the final fallback list, sort by fingerprint + # this makes diffs much more stable + # otherwise, leave sorted by bandwidth, which allows operators to be + # contacted in priority order + if not OUTPUT_CANDIDATES: + candidates.sort_fallbacks_by_fingerprint() + for x in candidates.fallbacks: - dl_speed_ok = x.fallback_consensus_dl_check() - print x.fallbackdir_line(dl_speed_ok, candidates.fallbacks, - prefilter_fallbacks) - if dl_speed_ok: - # this fallback is included in the list - active_count += 1 - if active_count >= max_count: - # we have enough fallbacks - break + print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks) if __name__ == "__main__": list_fallbacks()

1 0

[tor/master] Only count active fallbacks in updateFallbackDirs.py
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit e115523c98ce56973330c45c92939328f92642f7 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Apr 7 12:55:39 2016 +1000 Only count active fallbacks in updateFallbackDirs.py Previously, we would cut the list down to 100 fallbacks, then check if they could serve a consensus, and comment them out if they couldn't. This would leave us with fewer than 100 active fallbacks. Now, we stop when there are 100 active fallbacks. Also count fallbacks with identical contact info. Also fix minor logging issues. --- scripts/maint/updateFallbackDirs.py | 129 ++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 124f85f..3120496 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -26,6 +26,7 @@ import hashlib import dateutil.parser # bson_lazy provides bson #from bson import json_util +import copy from stem.descriptor.remote import DescriptorDownloader @@ -52,6 +53,11 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False +# Output matching ContactInfo in fallbacks list or the blacklist? +# Useful if you're trying to contact operators +CONTACT_COUNT = True if OUTPUT_CANDIDATES else False +CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False + ## OnionOO Settings ONIONOO = 'https://onionoo.torproject.org/' @@ -123,8 +129,9 @@ CONSENSUS_DOWNLOAD_RETRY = True _FB_POG = 0.2 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG +# We want exactly 100 fallbacks for the initial release # Limit the number of fallbacks (eliminating lowest by weight) -MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 500 +MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100 # Emit a C #error if the number of fallbacks is below MIN_FALLBACK_COUNT = 100 @@ -958,11 +965,12 @@ class Candidate(object): CONSENSUS_DOWNLOAD_SPEED_MAX) return ((not ipv4_failed) and (not ipv6_failed)) - def fallbackdir_line(self, dl_speed_ok): + def fallbackdir_line(self, dl_speed_ok, fallbacks, prefilter_fallbacks): # /* # nickname # flags # [contact] + # [identical contact counts] # */ # "address:dirport orport=port id=fingerprint" # "[ipv6=addr:orport]" @@ -978,6 +986,22 @@ class Candidate(object): s += '\n' if self._data['contact'] is not None: s += cleanse_c_multiline_comment(self._data['contact']) + if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT: + fallback_count = len([f for f in fallbacks + if f._data['contact'] == self._data['contact']]) + if fallback_count > 1: + s += '\n' + s += '%d identical contacts listed' % (fallback_count) + if CONTACT_BLACKLIST_COUNT: + prefilter_count = len([f for f in prefilter_fallbacks + if f._data['contact'] == self._data['contact']]) + filter_count = prefilter_count - fallback_count + if filter_count > 0: + if fallback_count > 1: + s += ' ' + else: + s += '\n' + s += '%d blacklisted' % (filter_count) s += '\n' s += '*/' s += '\n' @@ -1170,13 +1194,6 @@ class CandidateList(dict): return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( excluded_count, initial_count) - # Remove any fallbacks in excess of MAX_FALLBACK_COUNT, - # starting with the lowest-weighted fallbacks - # this changes total weight - def exclude_excess_fallbacks(self): - if MAX_FALLBACK_COUNT is not None: - self.fallbacks = self.fallbacks[:MAX_FALLBACK_COUNT] - def fallback_min_weight(self): if len(self.fallbacks) > 0: return self.fallbacks[-1] @@ -1192,63 +1209,55 @@ class CandidateList(dict): def summarise_fallbacks(self, eligible_count, guard_count, target_count, max_count): # Report: - # the number of fallback directories (with min & max limits); - # #error if below minimum count + # whether we checked consensus download times + # the number of fallback directories (and limits/exclusions, if relevant) # min & max fallback weights - # Multiline C comment with #error if things go bad - s = '/*' + # #error if below minimum count + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + s = '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( + 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', + ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS + and PERFORM_IPV6_DIRPORT_CHECKS) else '', + 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', + CONSENSUS_DOWNLOAD_SPEED_MAX) + else: + s = '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' s += '\n' - s += 'Fallback Directory Summary' + # Multiline C comment with #error if things go bad + s += '/*' s += '\n' # Integers don't need escaping in C comments fallback_count = len(self.fallbacks) if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = ' (none)' + fallback_proportion = '' else: - fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d, Target %d%s'%( - min(max_count, fallback_count), - eligible_count, - fallback_count, - fallback_proportion) + fallback_proportion = ', Target %d (%d * %f)'%(target_count, guard_count, + FALLBACK_PROPORTION_OF_GUARDS) + s += 'Final Count: %d (Eligible %d%s'%(fallback_count, + eligible_count, + fallback_proportion) if MAX_FALLBACK_COUNT is not None: s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) s += ')\n' - if fallback_count < MIN_FALLBACK_COUNT: - s += '*/' - s += '\n' - # We must have a minimum number of fallbacks so they are always - # reachable, and are in diverse locations - s += '#error Fallback Count %d is too low. '%(fallback_count) - s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to the whitelist, ' - s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' - s += '\n' - s += '/*' + if eligible_count != fallback_count: + s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( + eligible_count - fallback_count) s += '\n' min_fb = self.fallback_min_weight() min_weight = min_fb._data['consensus_weight'] max_fb = self.fallback_max_weight() max_weight = max_fb._data['consensus_weight'] - s += 'Max Weight: %d'%(max_weight) + s += 'Consensus Weight Range: %d - %d'%(min_weight, max_weight) s += '\n' - s += 'Min Weight: %d'%(min_weight) - s += '\n' - if eligible_count != fallback_count: - s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( - eligible_count - fallback_count) - s += '\n' s += '*/' - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( - 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', - ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS - and PERFORM_IPV6_DIRPORT_CHECKS) else '', - 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', - CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' + if fallback_count < MIN_FALLBACK_COUNT: + # We must have a minimum number of fallbacks so they are always + # reachable, and are in diverse locations + s += '\n' + s += '#error Fallback Count %d is too low. '%(fallback_count) + s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) + s += 'Try adding entries to the whitelist, ' + s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' return s ## Main Function @@ -1271,11 +1280,12 @@ def list_fallbacks(): # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count) # - the maximum fallback count (MAX_FALLBACK_COUNT) if MAX_FALLBACK_COUNT is None: - max_count = guard_count + max_count = target_count else: max_count = min(target_count, MAX_FALLBACK_COUNT) candidates.compute_fallbacks() + prefilter_fallbacks = copy.copy(candidates.fallbacks) # filter with the whitelist and blacklist initial_count = len(candidates.fallbacks) @@ -1286,9 +1296,8 @@ def list_fallbacks(): # print the raw fallback list #for x in candidates.fallbacks: # print x.fallbackdir_line(True) - - # exclude low-weight fallbacks if we have more than we want - candidates.exclude_excess_fallbacks() + # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, + # separators=(',', ': '), default=json_util.default) if len(candidates.fallbacks) > 0: print candidates.summarise_fallbacks(eligible_count, guard_count, @@ -1299,11 +1308,17 @@ def list_fallbacks(): for s in fetch_source_list(): print describe_fetch_source(s) - for x in candidates.fallbacks[:max_count]: + active_count = 0 + for x in candidates.fallbacks: dl_speed_ok = x.fallback_consensus_dl_check() - print x.fallbackdir_line(dl_speed_ok) - #print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) + print x.fallbackdir_line(dl_speed_ok, candidates.fallbacks, + prefilter_fallbacks) + if dl_speed_ok: + # this fallback is included in the list + active_count += 1 + if active_count >= max_count: + # we have enough fallbacks + break if __name__ == "__main__": list_fallbacks()

1 0

[tor/master] Select fallbacks by bandwidth rather than consensus weight
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 78ec782f76d4e228761128f24589316276e80473 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Fri Apr 8 23:53:24 2016 +1000 Select fallbacks by bandwidth rather than consensus weight But as advertised bandwidth is controlled by relays, use consensus weight and median weight to bandwidth ratio to approximate measured bandwidth. Includes minor comment changes and parameter reordering. --- scripts/maint/updateFallbackDirs.py | 208 +++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 52 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 3120496..44a7318 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -112,17 +112,13 @@ CUTOFF_GUARD = .95 # .00 means no bad exits PERMITTED_BADEXIT = .00 -# Clients will time out after 30 seconds trying to download a consensus -# So allow fallback directories half that to deliver a consensus -# The exact download times might change based on the network connection -# running this script, but only by a few seconds -# There is also about a second of python overhead -CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 -# If the relay fails a consensus check, retry the download -# This avoids delisting a relay due to transient network conditions -CONSENSUS_DOWNLOAD_RETRY = True +# older entries' weights are adjusted with ALPHA^(age in days) +AGE_ALPHA = 0.99 + +# this factor is used to scale OnionOO entries to [0,1] +ONIONOO_SCALE_ONE = 999. -## List Length Limits +## Fallback Count Limits # The target for these parameters is 20% of the guards in the network # This is around 200 as of October 2015 @@ -130,37 +126,53 @@ _FB_POG = 0.2 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG # We want exactly 100 fallbacks for the initial release -# Limit the number of fallbacks (eliminating lowest by weight) +# This gives us scope to add extra fallbacks to the list as needed +# Limit the number of fallbacks (eliminating lowest by advertised bandwidth) MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100 # Emit a C #error if the number of fallbacks is below MIN_FALLBACK_COUNT = 100 -## Fallback Weight Settings +## Fallback Bandwidth Requirements -# Any fallback with the Exit flag has its consensus weight multipled by this -EXIT_WEIGHT_FRACTION = 1.0 +# Any fallback with the Exit flag has its bandwidth multipled by this fraction +# to make sure we aren't further overloading exits +# (Set to 1.0, because we asked that only lightly loaded exits opt-in, +# and the extra load really isn't that much for large relays.) +EXIT_BANDWIDTH_FRACTION = 1.0 -# If a single fallback's consensus weight is too low, it's pointless adding it +# If a single fallback's bandwidth is too low, it's pointless adding it # We expect fallbacks to handle an extra 30 kilobytes per second of traffic -# Make sure they support a hundred times that -MIN_CONSENSUS_WEIGHT = 30.0 * 100.0 +# Make sure they can support a hundred times the expected extra load +# (Use 102.4 to make it come out nicely in MB/s) +# We convert this to a consensus weight before applying the filter, +# because all the bandwidth amounts are specified by the relay +MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0 + +# Clients will time out after 30 seconds trying to download a consensus +# So allow fallback directories half that to deliver a consensus +# The exact download times might change based on the network connection +# running this script, but only by a few seconds +# There is also about a second of python overhead +CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 +# If the relay fails a consensus check, retry the download +# This avoids delisting a relay due to transient network conditions +CONSENSUS_DOWNLOAD_RETRY = True + +## Fallback Weights for Client Selection # All fallback weights are equal, and set to the value below # Authorities are weighted 1.0 by default # Clients use these weights to select fallbacks and authorities at random # If there are 100 fallbacks and 9 authorities: -# - each fallback is chosen with probability 10/(1000 + 9) ~= 0.99% -# - each authority is chosen with probability 1/(1000 + 9) ~= 0.09% +# - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99% +# - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09% +# A client choosing a bootstrap directory server will choose a fallback for +# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for +# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts. +# (This disregards the bootstrap schedules, where clients start by choosing +# from fallbacks & authoritites, then later choose from only authorities.) FALLBACK_OUTPUT_WEIGHT = 10.0 -## Other Configuration Parameters - -# older entries' weights are adjusted with ALPHA^(age in days) -AGE_ALPHA = 0.99 - -# this factor is used to scale OnionOO entries to [0,1] -ONIONOO_SCALE_ONE = 999. - ## Parsing Functions def parse_ts(t): @@ -448,6 +460,11 @@ class Candidate(object): details['contact'] = None if not 'flags' in details or details['flags'] is None: details['flags'] = [] + if (not 'advertised_bandwidth' in details + or details['advertised_bandwidth'] is None): + # relays without advertised bandwdith have it calculated from their + # consensus weight + details['advertised_bandwidth'] = 0 details['last_changed_address_or_port'] = parse_ts( details['last_changed_address_or_port']) self._data = details @@ -462,10 +479,6 @@ class Candidate(object): self._compute_ipv6addr() if self.ipv6addr is None: logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) - # Reduce the weight of exits to EXIT_WEIGHT_FRACTION * consensus_weight - if self.is_exit(): - exit_weight = self._data['consensus_weight'] * EXIT_WEIGHT_FRACTION - self._data['consensus_weight'] = exit_weight def _stable_sort_or_addresses(self): # replace self._data['or_addresses'] with a stable ordering, @@ -754,11 +767,9 @@ class Candidate(object): logging.info('%s not a candidate: guard avg too low (%lf)', self._fpr, self._guard) return False - if (MIN_CONSENSUS_WEIGHT is not None - and self._data['consensus_weight'] < MIN_CONSENSUS_WEIGHT): - logging.info('%s not a candidate: consensus weight %.0f too low, must ' + - 'be at least %.0f', self._fpr, - self._data['consensus_weight'], MIN_CONSENSUS_WEIGHT) + if (not self._data.has_key('consensus_weight') + or self._data['consensus_weight'] < 1): + logging.info('%s not a candidate: consensus weight invalid', self._fpr) return False return True @@ -889,6 +900,30 @@ class Candidate(object): ipv6 if has_ipv6 else value) return False + def cw_to_bw_factor(self): + # any relays with a missing or zero consensus weight are not candidates + # any relays with a missing advertised bandwidth have it set to zero + return self._data['advertised_bandwidth'] / self._data['consensus_weight'] + + # since advertised_bandwidth is reported by the relay, it can be gamed + # to avoid this, use the median consensus weight to bandwidth factor to + # estimate this relay's measured bandwidth, and make that the upper limit + def measured_bandwidth(self, median_cw_to_bw_factor): + cw_to_bw= median_cw_to_bw_factor + # Reduce exit bandwidth to make sure we're not overloading them + if self.is_exit(): + cw_to_bw *= EXIT_BANDWIDTH_FRACTION + measured_bandwidth = self._data['consensus_weight'] * cw_to_bw + if self._data['advertised_bandwidth'] != 0: + # limit advertised bandwidth (if available) to measured bandwidth + return min(measured_bandwidth, self._data['advertised_bandwidth']) + else: + return measured_bandwidth + + def set_measured_bandwidth(self, median_cw_to_bw_factor): + self._data['measured_bandwidth'] = self.measured_bandwidth( + median_cw_to_bw_factor) + def is_exit(self): return 'Exit' in self._data['flags'] @@ -1056,8 +1091,8 @@ class CandidateList(dict): logging.debug('Loading details document.') d = fetch('details', fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + - 'consensus_weight,or_addresses,dir_address,' + - 'recommended_version,flags')) + 'consensus_weight,advertised_bandwidth,or_addresses,' + + 'dir_address,recommended_version,flags')) logging.debug('Loading details document done.') if not 'relays' in d: raise Exception("No relays found in document.") @@ -1083,15 +1118,24 @@ class CandidateList(dict): guard_count += 1 return guard_count - # Find fallbacks that fit the uptime, stability, and flags criteria + # Find fallbacks that fit the uptime, stability, and flags criteria, + # and make an array of them in self.fallbacks def compute_fallbacks(self): self.fallbacks = map(lambda x: self[x], - sorted( - filter(lambda x: self[x].is_candidate(), - self.keys()), - key=lambda x: self[x]._data['consensus_weight'], + filter(lambda x: self[x].is_candidate(), + self.keys())) + + # sort fallbacks by their consensus weight to advertised bandwidth factor, + # lowest to highest + # used to find the median cw_to_bw_factor() + def sort_fallbacks_by_cw_to_bw_factor(self): + self.fallbacks.sort(key=lambda x: self[x].cw_to_bw_factor()) + + # sort fallbacks by their measured bandwidth, highest to lowest + # calculate_measured_bandwidth before calling this + def sort_fallbacks_by_measured_bandwidth(self): + self.fallbacks.sort(key=lambda x: self[x].self._data['measured_bandwidth'], reverse=True) - ) @staticmethod def load_relaylist(file_name): @@ -1194,13 +1238,64 @@ class CandidateList(dict): return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( excluded_count, initial_count) - def fallback_min_weight(self): + # calculate each fallback's measured bandwidth based on the median + # consensus weight to advertised bandwdith ratio + def calculate_measured_bandwidth(self): + self.sort_fallbacks_by_cw_to_bw_factor() + median_fallback = self.fallback_median(True) + median_cw_to_bw_factor = median_fallback.cw_to_bw_factor() + for f in self.fallbacks: + f.set_measured_bandwidth(median_cw_to_bw_factor) + + # remove relays with low measured bandwidth from the fallback list + # calculate_measured_bandwidth for each relay before calling this + def remove_low_bandwidth_relays(self): + if MIN_BANDWIDTH is None: + return + above_min_bw_fallbacks = [] + for f in self.fallbacks: + if f._data['measured_bandwidth'] >= MIN_BANDWIDTH: + above_min_bw_fallbacks.append(f) + else: + # the bandwidth we log here is limited by the relay's consensus weight + # as well as its adverttised bandwidth. See set_measured_bandwidth + # for details + logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' + + 'be at least %.1fMB/s', f._fpr, + f._data['measured_bandwidth']/(1024.0*1024.0), + MIN_BANDWIDTH/(1024.0*1024.0)) + self.fallbacks = above_min_bw_fallbacks + + # the minimum fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_min(self): if len(self.fallbacks) > 0: return self.fallbacks[-1] else: return None - def fallback_max_weight(self): + # the median fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_median(self, require_advertised_bandwidth): + # use the low-median when there are an evan number of fallbacks, + # for consistency with the bandwidth authorities + if len(self.fallbacks) > 0: + median_position = (len(self.fallbacks) - 1) / 2 + if not require_advertised_bandwidth: + return self.fallbacks[median_position] + # if we need advertised_bandwidth but this relay doesn't have it, + # move to a fallback with greater consensus weight until we find one + while not self.fallbacks[median_position]._data['advertised_bandwidth']: + median_position += 1 + if median_position >= len(self.fallbacks): + return None + return self.fallbacks[median_position] + else: + return None + + # the maximum fallback in the list + # call one of the sort_fallbacks_* functions before calling this + def fallback_max(self): if len(self.fallbacks) > 0: return self.fallbacks[0] else: @@ -1211,7 +1306,7 @@ class CandidateList(dict): # Report: # whether we checked consensus download times # the number of fallback directories (and limits/exclusions, if relevant) - # min & max fallback weights + # min & max fallback bandwidths # #error if below minimum count if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: s = '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( @@ -1243,11 +1338,12 @@ class CandidateList(dict): s += 'Excluded: %d (Eligible Count Exceeded Target Count)'%( eligible_count - fallback_count) s += '\n' - min_fb = self.fallback_min_weight() - min_weight = min_fb._data['consensus_weight'] - max_fb = self.fallback_max_weight() - max_weight = max_fb._data['consensus_weight'] - s += 'Consensus Weight Range: %d - %d'%(min_weight, max_weight) + min_fb = self.fallback_min() + min_bw = min_fb._data['measured_bandwidth'] + max_fb = self.fallback_max() + max_bw = max_fb._data['measured_bandwidth'] + s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0), + max_bw/(1024.0*1024.0)) s += '\n' s += '*/' if fallback_count < MIN_FALLBACK_COUNT: @@ -1293,6 +1389,14 @@ def list_fallbacks(): print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) + # calculate the measured bandwidth of each relay, + # then remove low-bandwidth relays + candidates.calculate_measured_bandwidth() + candidates.remove_low_bandwidth_relays() + # make sure the list is sorted by bandwidth when we output it + # so that we include the active fallbacks with the greatest bandwidth + candidates.sort_fallbacks_by_measured_bandwidth() + # print the raw fallback list #for x in candidates.fallbacks: # print x.fallbackdir_line(True)

1 0

[tor/maint-0.2.8] Whitelist additional fallbacks after creating April 2016 list
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit de08f2d96bd1815a8da5c359a4f52310dc1905d7 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Mon Apr 18 22:20:06 2016 +1000 Whitelist additional fallbacks after creating April 2016 list --- scripts/maint/fallback.blacklist | 25 +++++++++++++++++++++++++ scripts/maint/fallback.whitelist | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist index f34986a..cc0c55d 100644 --- a/scripts/maint/fallback.blacklist +++ b/scripts/maint/fallback.blacklist @@ -170,3 +170,28 @@ # Email sent directly to teor, verified using relay contact info 104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001 + +# Email sent directly to teor, verified using relay contact info +46.252.25.249:9030 orport=443 id=80DCBB6EF4E86A7CD4FBCBDEE64979645509A610 + +# Email sent directly to teor, verified using relay contact info +176.10.99.200:8080 orport=443 id=2B44FD1742D26E4F28D4CACF1F0CF8A686270E45 +176.10.99.200:8000 orport=22 id=EB79F07792A065D3C534063773E83268E069F5EB +176.10.99.201:667 orport=666 id=3EAAAB35932610411E24FA4317603CB5780B80BC +176.10.99.201:990 orport=989 id=7C3A4CFF09C1981D41173CDE2A2ADD4A5CA109FD +176.10.99.202:992 orport=991 id=615EBC4B48F03858FA50A3E23E5AF569D0D2308A +176.10.99.202:994 orport=993 id=E34E25D958D46DDE5092385B14117C9B301DC0E9 +176.10.99.203:1194 orport=995 id=AD368442E9FF33C08C7407DF2DA7DB958F406CE2 +176.10.99.203:43 orport=53 id=79CF377F0ACEC5F0002D85335E4192B34202A269 +176.10.99.204:1755 orport=1723 id=69DF3CDA1CDA460C17ECAD9D6F0C117A42384FA0 +176.10.99.204:1293 orport=4321 id=3F061400B6FB1F55E7F19BB3C713884D677E55B7 +176.10.99.205:426 orport=425 id=C30B284784BF11D0D58C6A250240EE58D2084AD0 +176.10.99.205:109 orport=110 id=12D17D9F9E30FA901DE68806950A0EA278716CED +176.10.99.206:24 orport=23 id=2C804AAB0C02F971A4386B3A1F2AC00F9E080679 +176.10.99.206:20 orport=21 id=237588726AB6BEA37FF23CA00F5BD178586CA68E +176.10.99.207:3390 orport=3389 id=A838D5B8890B10172429ECE92EB5677DF93DC4DD +176.10.99.207:1415 orport=1414 id=377E5E817A84FAE0F4DC3427805DB2E8A6CBBFC0 +176.10.99.208:390 orport=389 id=7C288587BA0D99CC6B8537CDC2C4639FA827B907 +176.10.99.208:3307 orport=3306 id=1F0D2A44C56F42816DED2022EFD631878C29905B +176.10.99.209:1434 orport=1433 id=BDA7A91FF3806DE5109FDAE74CFEFB3BABB9E10F +176.10.99.209:220 orport=219 id=B8C2030001D832066A648269CFBA94171951D34B diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 61cb97a..06456c6 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -728,3 +728,43 @@ # Email sent directly to teor, verified using relay contact info 66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142 66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A + +# Email sent directly to teor, verified using relay contact info +92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82 + +# Email sent directly to teor, verified using relay contact info +212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E + +# Email sent directly to teor, verified using relay contact info +185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB + +# Email sent directly to teor, verified using relay contact info +166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812 + +# Emails sent directly to teor, verified using relay contact info +#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001 + +# Email sent directly to teor, verified using relay contact info +213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337 + +# Email sent directly to teor, verified using relay contact info +188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 + +# Email sent directly to teor, verified using relay contact info +88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 + +# Email sent directly to teor, verified using relay contact info +185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 +46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3 + +# Email sent directly to teor, verified using relay contact info +176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 +176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 +176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C +176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F + +# Email sent directly to teor, verified using relay contact info +107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA + +# Email sent directly to teor, verified using relay contact info +192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC

1 0

[tor/master] Improve fallback selection and output
by nickm＠torproject.org 26 Apr '16

26 Apr '16

commit 7e1b8ae79cc7da3b4a0989d6f9ba95aa5cd9b73b Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Wed Mar 30 12:41:57 2016 +1100 Improve fallback selection and output Improve the download test: * Allow IPv4 DirPort checks to be turned off. * Add a timeout to stem's consensus download. * Actually check for download errors, rather than ignoring them. * Simplify the timeout and download error checking logic. Tweak whitelist/blacklist checks to be more robust. Improve logging, make it warn by default. Cleanse fallback comments more thoroughly: * non-printables (yes, ContactInfo can have these) * // comments (don't rely on newlines to prevent // */ escapes) --- scripts/maint/updateFallbackDirs.py | 289 +++++++++++++++++++++++++----------- 1 file changed, 201 insertions(+), 88 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index d110335..4860700 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1,6 +1,7 @@ #!/usr/bin/python # Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc +# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . # # Then read the generated list to ensure no-one slipped anything funny into # their name or contactinfo @@ -29,17 +30,28 @@ import dateutil.parser from stem.descriptor.remote import DescriptorDownloader import logging -logging.basicConfig(level=logging.DEBUG) +# INFO tells you why each relay was included or excluded +# WARN tells you about potential misconfigurations +logging.basicConfig(level=logging.WARNING) ## Top-Level Configuration -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -PERFORM_IPV6_DIRPORT_CHECKS = False - # Output all candidate fallbacks, or only output selected fallbacks? OUTPUT_CANDIDATES = False +# Perform DirPort checks over IPv4? +# Change this to False if IPv4 doesn't work for you, or if you don't want to +# download a consensus for each fallback +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True + +# Perform DirPort checks over IPv6? +# If you know IPv6 works for you, set this to True +# This will exclude IPv6 relays without an IPv6 DirPort configured +# So it's best left at False until #18394 is implemented +# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True +PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False + ## OnionOO Settings ONIONOO = 'https://onionoo.torproject.org/' @@ -81,7 +93,7 @@ MAX_LIST_FILE_SIZE = 1024 * 1024 # Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted # This causes OnionOO to (correctly) reset its stability timer -# This issue is fixed in 0.2.7.7 and master. +# This issue will be fixed in 0.2.7.7 and 0.2.8.2 # Until then, the CUTOFFs below ensure a decent level of stability. ADDRESS_AND_PORT_STABLE_DAYS = 7 # What time-weighted-fraction of these flags must FallbackDirs @@ -157,36 +169,52 @@ def parse_ts(t): def remove_bad_chars(raw_string, bad_char_list): # Remove each character in the bad_char_list - escaped_string = raw_string + cleansed_string = raw_string for c in bad_char_list: - escaped_string = escaped_string.replace(c, '') - return escaped_string + cleansed_string = cleansed_string.replace(c, '') + return cleansed_string + +def cleanse_unprintable(raw_string): + # Remove all unprintable characters + cleansed_string = '' + for c in raw_string: + if (c in string.ascii_letters or c in string.digits + or c in string.punctuation or c in string.whitespace): + cleansed_string += c + return cleansed_string def cleanse_whitespace(raw_string): # Replace all whitespace characters with a space - escaped_string = raw_string + cleansed_string = raw_string for c in string.whitespace: - escaped_string = escaped_string.replace(c, ' ') - return escaped_string + cleansed_string = cleansed_string.replace(c, ' ') + return cleansed_string def cleanse_c_multiline_comment(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious / unanticipated string from breaking out # of a C-style multiline comment - # This removes '/*' and '*/' - # To deal with '//', the end comment must be on its own line - bad_char_list = '*' + # This removes '/*' and '*/' and '//' + bad_char_list = '*/' # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of comments # There isn't much we can do to cover every possible case # But comment-based directives are typically only advisory - return escaped_string + return cleansed_string def cleanse_c_string(raw_string): + cleansed_string = raw_string + # Embedded newlines should be removed by tor/onionoo, but let's be paranoid + cleansed_string = cleanse_whitespace(cleansed_string) + # ContactInfo and Version can be arbitrary binary data + cleansed_string = cleanse_unprintable(cleansed_string) # Prevent a malicious address/fingerprint string from breaking out # of a C-style string bad_char_list = '"' @@ -195,13 +223,11 @@ def cleanse_c_string(raw_string): # Prevent a malicious string from using C nulls bad_char_list += '\0' # Be safer by removing bad characters entirely - escaped_string = remove_bad_chars(raw_string, bad_char_list) - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - escaped_string = cleanse_whitespace(escaped_string) + cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) # Some compilers may further process the content of strings # There isn't much we can do to cover every possible case # But this typically only results in changes to the string data - return escaped_string + return cleansed_string ## OnionOO Source Functions @@ -244,11 +270,11 @@ def write_to_file(str, file_name, max_len): with open(file_name, 'w') as f: f.write(str[0:max_len]) except EnvironmentError, error: - logging.debug('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.warning('Writing file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) def read_from_file(file_name, max_len): try: @@ -256,11 +282,11 @@ def read_from_file(file_name, max_len): with open(file_name, 'r') as f: return f.read(max_len) except EnvironmentError, error: - logging.debug('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) + logging.info('Loading file %s failed: %d: %s'% + (file_name, + error.errno, + error.strerror) + ) return None def load_possibly_compressed_response_json(response): @@ -699,30 +725,37 @@ class Candidate(object): self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE def is_candidate(self): + must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS + or PERFORM_IPV6_DIRPORT_CHECKS) + if (must_be_running_now and not self.is_running()): + logging.info('%s not a candidate: not running now, unable to check ' + + 'DirPort consensus download', self._fpr) + return False if (self._data['last_changed_address_or_port'] > self.CUTOFF_ADDRESS_AND_PORT_STABLE): - logging.debug('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) + logging.info('%s not a candidate: changed address/port recently (%s)', + self._fpr, self._data['last_changed_address_or_port']) return False if self._running < CUTOFF_RUNNING: - logging.debug('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) + logging.info('%s not a candidate: running avg too low (%lf)', + self._fpr, self._running) return False if self._v2dir < CUTOFF_V2DIR: - logging.debug('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) + logging.info('%s not a candidate: v2dir avg too low (%lf)', + self._fpr, self._v2dir) return False if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - logging.debug('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) + logging.info('%s not a candidate: badexit avg too high (%lf)', + self._fpr, self._badexit) return False # if the relay doesn't report a version, also exclude the relay if (not self._data.has_key('recommended_version') or not self._data['recommended_version']): + logging.info('%s not a candidate: version not recommended', self._fpr) return False if self._guard < CUTOFF_GUARD: - logging.debug('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) + logging.info('%s not a candidate: guard avg too low (%lf)', + self._fpr, self._guard) return False return True @@ -736,24 +769,48 @@ class Candidate(object): If the fallback has an ipv6 key, the whitelist line must also have it, and vice versa, otherwise they don't match. """ for entry in relaylist: + if entry['id'] != self._fpr: + # can't log here, every relay's fingerprint is compared to the entry + continue if entry['ipv4'] != self.dirip: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'IPv4 (%s) does not match entry IPv4 (%s)', + self._fpr, self.dirip, entry['ipv4']) continue if int(entry['dirport']) != self.dirport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'DirPort (%d) does not match entry DirPort (%d)', + self._fpr, self.dirport, int(entry['dirport'])) continue if int(entry['orport']) != self.orport: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'ORPort (%d) does not match entry ORPort (%d)', + self._fpr, self.orport, int(entry['orport'])) continue - if entry['id'] != self._fpr: - continue - if (entry.has_key('ipv6') - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + if (entry.has_key('ipv6') and has_ipv6): + ipv6 = self.ipv6addr + ':' + self.ipv6orport # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != self.ipv6addr + ':' + self.ipv6orport: + if entry['ipv6'] != ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, ' + + 'but IPv6 (%s) does not match entry IPv6 (%s)', + self._fpr, ipv6, entry['ipv6']) continue # if the fallback has an IPv6 address but the whitelist entry # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and self.ipv6addr is None: + elif entry.has_key('ipv6') and not has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has no IPv6, and entry has IPv6 (%s)', self._fpr, + entry['ipv6']) + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) continue - elif not entry.has_key('ipv6') and self.ipv6addr is not None: + elif not entry.has_key('ipv6') and has_ipv6: + logging.info('%s is not in the whitelist: fingerprint matches, but ' + + 'it has IPv6 (%s), and entry has no IPv6', self._fpr, + ipv6) + logging.warning('%s excluded: has it gained an IPv6 address %s?', + self._fpr, ipv6) continue return True return False @@ -773,34 +830,60 @@ class Candidate(object): for entry in relaylist: for key in entry: value = entry[key] + if key == 'id' and value == self._fpr: + logging.info('%s is in the blacklist: fingerprint matches', + self._fpr) + return True if key == 'ipv4' and value == self.dirip: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'DirPort (%d) match', self._fpr, self.dirip, + self.dirport) return True # if the orport is present, check it too elif entry.has_key('orport'): if int(entry['orport']) == self.orport: + logging.info('%s is in the blacklist: IPv4 (%s) and ' + + 'ORPort (%d) match', self._fpr, self.dirip, + self.orport) return True else: + logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' + + 'entry has no DirPort or ORPort', self._fpr, + self.dirip) return True - if key == 'id' and value == self._fpr: - return True - if (key == 'ipv6' - and self.ipv6addr is not None and self.ipv6orport is not None): + has_ipv6 = self.ipv6addr is not None and self.ipv6orport is not None + ipv6 = (self.ipv6addr + ':' + self.ipv6orport) if has_ipv6 else None + if (key == 'ipv6' and has_ipv6): # if both entry and fallback have an ipv6 address, compare them, # otherwise, disregard ipv6 addresses - if value == self.ipv6addr + ':' + self.ipv6orport: + if value == ipv6: # if the dirport is present, check it too if entry.has_key('dirport'): if int(entry['dirport']) == self.dirport: + logging.info('%s is in the blacklist: IPv6 (%s) and ' + + 'DirPort (%d) match', self._fpr, ipv6, + self.dirport) return True - # if the orport is present, check it too - elif entry.has_key('orport'): - if int(entry['orport']) == self.orport: - return True + # we've already checked the ORPort, it's part of entry['ipv6'] else: + logging.info('%s is in the blacklist: IPv6 (%s) matches, and' + + 'entry has no DirPort', self._fpr, ipv6) return True + elif (key == 'ipv6' or has_ipv6): + # only log if the fingerprint matches but the IPv6 doesn't + if entry.has_key('id') and entry['id'] == self._fpr: + logging.info('%s skipping IPv6 blacklist comparison: relay ' + + 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, + '' if has_ipv6 else ' no', + (' (' + ipv6 + ')') if has_ipv6 else '', + '' if key == 'ipv6' else ' no', + (' (' + value + ')') if key == 'ipv6' else '') + logging.warning('Has %s %s IPv6 address %s?', self._fpr, + 'gained an' if has_ipv6 else 'lost its former', + ipv6 if has_ipv6 else value) return False def is_exit(self): @@ -809,6 +892,9 @@ class Candidate(object): def is_guard(self): return 'Guard' in self._data['flags'] + def is_running(self): + return 'Running' in self._data['flags'] + def fallback_weight_fraction(self, total_weight): return float(self._data['consensus_weight']) / total_weight @@ -825,53 +911,70 @@ class Candidate(object): @staticmethod def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time): + download_failed = False downloader = DescriptorDownloader() start = datetime.datetime.utcnow() + # some directory mirrors respond to requests in ways that hang python + # sockets, which is why we long this line here + logging.info('Initiating consensus download from %s (%s:%d).', nickname, + dirip, dirport) # there appears to be about 1 second of overhead when comparing stem's # internal trace time and the elapsed time calculated here - downloader.get_consensus(endpoints = [(dirip, dirport)]).run() + TIMEOUT_SLOP = 1.0 + try: + downloader.get_consensus(endpoints = [(dirip, dirport)], + timeout = (max_time + TIMEOUT_SLOP), + validate = True, + retries = 0, + fall_back_to_authority = False).run() + except Exception, stem_error: + logging.debug('Unable to retrieve a consensus from %s: %s', nickname, + stem_error) + status = 'error: "%s"' % (stem_error) + level = logging.WARNING + download_failed = True elapsed = (datetime.datetime.utcnow() - start).total_seconds() if elapsed > max_time: status = 'too slow' + level = logging.WARNING + download_failed = True else: status = 'ok' - logging.debug(('Consensus download: %0.2fs %s from %s (%s:%d), ' - + 'max download time %0.2fs.') % (elapsed, status, - nickname, dirip, dirport, - max_time)) - return elapsed + level = logging.DEBUG + logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' + + 'max download time %0.1fs.', elapsed, status, nickname, + dirip, dirport, max_time) + return download_failed def fallback_consensus_dl_check(self): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + # include the relay if we're not doing a check, or we can't check (IPv6) + ipv4_failed = False + ipv6_failed = False + if PERFORM_IPV4_DIRPORT_CHECKS: + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS: # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - ipv6_speed = None # Now retry the relay if it took too long the first time - if (ipv4_speed > CONSENSUS_DOWNLOAD_SPEED_MAX + if (PERFORM_IPV4_DIRPORT_CHECKS and ipv4_failed and CONSENSUS_DOWNLOAD_RETRY): - ipv4_speed = Candidate.fallback_consensus_dl_speed(self.dirip, + ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS - and ipv6_speed > CONSENSUS_DOWNLOAD_SPEED_MAX - and CONSENSUS_DOWNLOAD_RETRY): - ipv6_speed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, + and ipv6_failed and CONSENSUS_DOWNLOAD_RETRY): + ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr, self.dirport, self._data['nickname'], CONSENSUS_DOWNLOAD_SPEED_MAX) - - return (ipv4_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX - and (not PERFORM_IPV6_DIRPORT_CHECKS - or ipv6_speed <= CONSENSUS_DOWNLOAD_SPEED_MAX)) + return ((not ipv4_failed) and (not ipv6_failed)) def fallbackdir_line(self, total_weight, original_total_weight, dl_speed_ok): # /* @@ -1071,8 +1174,8 @@ class CandidateList(dict): if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES: # exclude excluded_count += 1 - logging.debug('Excluding %s: in both blacklist and whitelist.' % - f._fpr) + logging.warning('Excluding %s: in both blacklist and whitelist.', + f._fpr) else: # include filtered_fallbacks.append(f) @@ -1082,8 +1185,7 @@ class CandidateList(dict): elif in_blacklist: # exclude excluded_count += 1 - logging.debug('Excluding %s: in blacklist.' % - f._fpr) + logging.debug('Excluding %s: in blacklist.', f._fpr) else: if INCLUDE_UNLISTED_ENTRIES: # include @@ -1091,8 +1193,8 @@ class CandidateList(dict): else: # exclude excluded_count += 1 - logging.debug('Excluding %s: in neither blacklist nor whitelist.' % - f._fpr) + logging.info('Excluding %s: in neither blacklist nor whitelist.', + f._fpr) self.fallbacks = filtered_fallbacks return excluded_count @@ -1173,15 +1275,14 @@ class CandidateList(dict): # Integers don't need escaping in C comments fallback_count = len(self.fallbacks) if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' + fallback_proportion = ' (none)' else: - fallback_proportion = ' (%d * %f)'%(guard_count, + fallback_proportion = '%d (%d * %f)'%(target_count, guard_count, FALLBACK_PROPORTION_OF_GUARDS) s += 'Final Count: %d (Eligible %d, Usable %d, Target %d%s'%( min(max_count, fallback_count), eligible_count, fallback_count, - target_count, fallback_proportion) if MAX_FALLBACK_COUNT is not None: s += ', Clamped to %d'%(MAX_FALLBACK_COUNT) @@ -1242,6 +1343,16 @@ class CandidateList(dict): s += '#error ' + error_str else: s += '/* ' + error_str + ' */' + s += '\n' + if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: + s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( + 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', + ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS + and PERFORM_IPV6_DIRPORT_CHECKS) else '', + 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', + CONSENSUS_DOWNLOAD_SPEED_MAX) + else: + s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' return s ## Main Function @@ -1250,9 +1361,11 @@ def list_fallbacks(): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays """ + # find relays that could be fallbacks candidates = CandidateList() candidates.add_relays() + # work out how many fallbacks we want guard_count = candidates.count_guards() if FALLBACK_PROPORTION_OF_GUARDS is None: target_count = guard_count @@ -1268,10 +1381,10 @@ def list_fallbacks(): candidates.compute_fallbacks() + # filter with the whitelist and blacklist initial_count = len(candidates.fallbacks) excluded_count = candidates.apply_filter_lists() print candidates.summarise_filters(initial_count, excluded_count) - eligible_count = len(candidates.fallbacks) eligible_weight = candidates.fallback_weight_total()

1 0