- tor-commits - lists.torproject.org

[ooni-probe/master] Add details on "the hard way"
by art＠torproject.org 24 Jul '12

24 Jul '12

commit ac54aa0e98a7915f527f982f1792df09018767a8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 24 13:25:49 2012 +0200 Add details on "the hard way" --- INSTALL | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 48 insertions(+), 0 deletions(-) diff --git a/INSTALL b/INSTALL index 863c62b..bac1a39 100644 --- a/INSTALL +++ b/INSTALL @@ -2,6 +2,8 @@ * Twisted: http://twistedmatrix.com/trac/ * PyYAML: http://pyyaml.org/ * Scapy: http://www.secdev.org/projects/scapy/ + * pypcap: http://code.google.com/p/pypcap/ + * libdnet: http://code.google.com/p/libdnet/ Optional: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ @@ -12,3 +14,49 @@ On debian you can install all the dependecies with apt-get with this command: apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup +## The "hard" way + +This involves installing the dependencies installable via easy_install/pip and +the ones that are not by building them from source. + +"simple" dependencies via easy_install: + + sudo easy_install pyyaml + sudo easy_install twisted + sudo easy_install beautifulsoup + +"simple" dependencies via pip: + + sudo pip install pyyaml + sudo pip install twisted + sudo pip install beautifulsoup + + + +libdnet: + + wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz + tar xzf libdnet-1.12.tgz + cd libdnet-1.12 + ./configure && make + cd python/ + sudo python setup.py install + cd ../../ && rm -rf libdnet-1.12* + +pypcap: + + svn checkout http://pypcap.googlecode.com/svn/trunk/ pypcap-read-only + cd pypcap-read-only/ + sudo pip install pyrex + make + sudo python setup.py install + cd ../ && rm -rf pypcap-read-only + +scapy: + + wget http://www.secdev.org/projects/scapy/files/scapy-latest.zip + unzip scapy-latest.zip + cd scapy-2.2.0/ + sudo python setup.py install + cd ../ && rm -rf scapy-* +

1 0

[ooni-probe/master] Add installation guide. Fix bug in test.
by art＠torproject.org 24 Jul '12

24 Jul '12

commit 206bae4c2ac450c0eb7de591bf7ee39e944109ef Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 24 13:05:24 2012 +0200 Add installation guide. Fix bug in test. --- INSTALL | 14 ++++++++++++++ ooni/plugins/httphost.py | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..863c62b --- /dev/null +++ b/INSTALL @@ -0,0 +1,14 @@ +# Dependencies +* Twisted: http://twistedmatrix.com/trac/ +* PyYAML: http://pyyaml.org/ +* Scapy: http://www.secdev.org/projects/scapy/ + +Optional: +* BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ + +# Installation + +On debian you can install all the dependecies with apt-get with this command: + + apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup + diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index ccadff9..b57a50d 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -10,8 +10,10 @@ from datetime import datetime import urllib2 import httplib - -from BeautifulSoup import BeautifulSoup +try: + from BeautifulSoup import BeautifulSoup +except: + print "Beautiful Soup not installed. HTTPHost may not work" # XXX reduce boilerplating. from zope.interface import implements

1 0

[metrics-tasks/master] Bad exits are no exits (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit 15fd1870a94cc8203c30ff1a7480b10fc20d43bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 09:42:57 2012 +0200 Bad exits are no exits (#6232). --- task-6232/pyentropy.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 99d83da..08e0218 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -50,7 +50,7 @@ class Router: self.bandwidth = int(values[0].split('=')[1]) def add_flags(self, values): - if "Exit" in values: + if "Exit" in values and not "BadExit" in values: self.is_exit = True if "Guard" in values: self.is_guard = True

1 0

[metrics-tasks/master] Remove unused members of the Router class (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit cb16d7e2790c2a02a4315948b522a04f5464a3b0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 09:39:50 2012 +0200 Remove unused members of the Router class (#6232). --- task-6232/pyentropy.py | 30 ++++++++++-------------------- 1 files changed, 10 insertions(+), 20 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 8af09c6..99d83da 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -28,28 +28,19 @@ from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor class Router: def __init__(self): - self.lines = [] - self.nick = None - self.digest = None self.hex_digest = None self.bandwidth = None self.advertised_bw = None - self.flags = None - self.probability = None - self.ip = None self.country = None self.as_no = None - self.as_name = None self.is_exit = None self.is_guard = None def add_router_info(self, values): - self.nick = values[0] - self.digest = values[2] - self.hex_digest = b2a_hex(a2b_base64(self.digest+"=")) - self.ip = values[5] - self.country = gi_db.country_name_by_addr(self.ip) - self.as_no, self.as_name = self.get_as_details() + self.hex_digest = b2a_hex(a2b_base64(values[2]+"=")) + ip = values[5] + self.country = gi_db.country_name_by_addr(ip) + self.as_no = self.get_as_details(ip) def add_weights(self, values): self.advertised_bw = self.get_advertised_bw() @@ -59,18 +50,17 @@ class Router: self.bandwidth = int(values[0].split('=')[1]) def add_flags(self, values): - self.flags = values - if "Exit" in self.flags: + if "Exit" in values: self.is_exit = True - if "Guard" in self.flags: + if "Guard" in values: self.is_guard = True - def get_as_details(self): + def get_as_details(self, ip): try: - value = as_db.org_by_addr(str(self.ip)).split() - return value[0], value[1] + value = as_db.org_by_addr(str(ip)).split() + return value[0] except: - return None, None + return None def get_advertised_bw(self): try:

1 0

[metrics-tasks/master] No need to memorize country names if we can use country codes instead (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit 4b5dd9843fc063bdadcda1d18c5eccc797f73406 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 09:48:46 2012 +0200 No need to memorize country names if we can use country codes instead (#6232). --- task-6232/pyentropy.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 08e0218..ba11d24 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -39,7 +39,7 @@ class Router: def add_router_info(self, values): self.hex_digest = b2a_hex(a2b_base64(values[2]+"=")) ip = values[5] - self.country = gi_db.country_name_by_addr(ip) + self.country = gi_db.country_code_by_addr(ip) self.as_no = self.get_as_details(ip) def add_weights(self, values):

1 0

[metrics-tasks/master] Fix a bug in counting guards and exits (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit ca0339bd300854853e126e5402d50e4f527739b5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 09:59:15 2012 +0200 Fix a bug in counting guards and exits (#6232). --- task-6232/pyentropy.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 74c5699..76a3d52 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -117,6 +117,8 @@ def run(file_name): if router.is_guard and router.is_exit: total_guard_bw += Wgd*router.bandwidth total_exit_bw += Wed*router.bandwidth + guards_no += 1 + exits_no += 1 elif router.is_guard: total_guard_bw += Wgg*router.bandwidth guards_no += 1

1 0

[metrics-tasks/master] Fix the way how advertised bandwidth is supposed to be used (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit e7f15593954d88ad1de449c23581eb4b007ad539 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 09:53:29 2012 +0200 Fix the way how advertised bandwidth is supposed to be used (#6232). A relay's advertised bandwidth is not supposed to be used as replacement for a potentially missing consensus bandwidth. The idea is to compare path selection probability based on reported vs measured bandwidth. There should be an entropy (and max entropy) based on advertised bandwidths in addition to the same numbers for consensus weights. Also removing the hex_digest field, because we only need it once when looking up the advertised bandwidth. Oh, and changing the default advertised bandwidth from None to 0, so that we can calculate with the result. There may always be single server descriptors missing. --- task-6232/pyentropy.py | 16 ++++++---------- 1 files changed, 6 insertions(+), 10 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index ba11d24..74c5699 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -28,7 +28,6 @@ from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor class Router: def __init__(self): - self.hex_digest = None self.bandwidth = None self.advertised_bw = None self.country = None @@ -37,17 +36,14 @@ class Router: self.is_guard = None def add_router_info(self, values): - self.hex_digest = b2a_hex(a2b_base64(values[2]+"=")) + hex_digest = b2a_hex(a2b_base64(values[2]+"=")) + self.advertised_bw = self.get_advertised_bw(hex_digest) ip = values[5] self.country = gi_db.country_code_by_addr(ip) self.as_no = self.get_as_details(ip) def add_weights(self, values): - self.advertised_bw = self.get_advertised_bw() - if self.advertised_bw: - self.bandwidth = self.advertised_bw - else: - self.bandwidth = int(values[0].split('=')[1]) + self.bandwidth = int(values[0].split('=')[1]) def add_flags(self, values): if "Exit" in values and not "BadExit" in values: @@ -62,9 +58,9 @@ class Router: except: return None - def get_advertised_bw(self): + def get_advertised_bw(self, hex_digest): try: - with open(options.server_desc+self.hex_digest) as f: + with open(options.server_desc+hex_digest) as f: data = f.read() desc_iter = stem.descriptor.server_descriptor.parse_file(StringIO.StringIO(data)) @@ -72,7 +68,7 @@ class Router: desc = desc_entries[0] return min(desc.average_bandwidth, desc.burst_bandwidth, desc.observed_bandwidth) except: - return None + return 0 def parse_bw_weights(values): data = {}

1 0

[metrics-tasks/master] Ignore a consensus without relays even sooner (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit b8a3574dd5d5b96560d366b0082413484d7f05cc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 10:00:10 2012 +0200 Ignore a consensus without relays even sooner (#6232). --- task-6232/pyentropy.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 76a3d52..dd5e343 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -109,6 +109,9 @@ def run(file_name): except: pass + if len(routers) <= 0: + return + total_bw, total_exit_bw, total_guard_bw = 0, 0, 0 guards_no, exits_no = 0, 0 bw_countries, bw_as = {}, {} @@ -135,9 +138,6 @@ def run(file_name): else: bw_as[router.as_no] = router.bandwidth - if len(routers) <= 0: - return - entropy, entropy_exit, entropy_guard, entropy_country, entropy_as = 0.0, 0.0, 0.0, 0.0, 0.0 for router in routers: p = float(router.bandwidth) / float(total_bw)

1 0

[metrics-tasks/master] Skip consensuses that don't contain consensus weights (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit f5a5bad9dac005e416e09a97e4512ac7bf0a667f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 10:57:10 2012 +0200 Skip consensuses that don't contain consensus weights (#6232). --- task-6232/pyentropy.py | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 8e28210..e6d3214 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -116,6 +116,8 @@ def run(file_name): guards_no, exits_no = 0, 0 bw_countries, bw_as = {}, {} for router in routers: + if not router.bandwidth: + continue total_bw += router.bandwidth if router.is_guard and router.is_exit: total_guard_bw += Wgd*router.bandwidth @@ -137,6 +139,9 @@ def run(file_name): else: bw_as[router.as_no] = router.bandwidth + if total_bw == 0: + return + entropy, entropy_exit, entropy_guard, entropy_country, entropy_as = 0.0, 0.0, 0.0, 0.0, 0.0 for router in routers: p = float(router.bandwidth) / float(total_bw)

1 0

[metrics-tasks/master] Treat unresolved AS numbers correctly (#6232).
by karsten＠torproject.org 24 Jul '12

24 Jul '12

commit 4275170f5cbcd7332df77f7ca7a36618c27c269c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 24 10:12:07 2012 +0200 Treat unresolved AS numbers correctly (#6232). So far, relays that couldn't be found in the AS database were ignored in the AS entropy calculation. That's not correct. We have to treat them as "other" AS, similar to how we handle relays for which we couldn't find the country code. Otherwise the probability of all ASes wouldn't sum up to 1. --- task-6232/pyentropy.py | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index dd5e343..8e28210 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -56,7 +56,7 @@ class Router: value = as_db.org_by_addr(str(ip)).split() return value[0] except: - return None + return "" def get_advertised_bw(self, hex_digest): try: @@ -132,11 +132,10 @@ def run(file_name): bw_countries[router.country] += router.bandwidth else: bw_countries[router.country] = router.bandwidth - if router.as_no: - if bw_as.has_key(router.as_no): - bw_as[router.as_no] += router.bandwidth - else: - bw_as[router.as_no] = router.bandwidth + if bw_as.has_key(router.as_no): + bw_as[router.as_no] += router.bandwidth + else: + bw_as[router.as_no] = router.bandwidth entropy, entropy_exit, entropy_guard, entropy_country, entropy_as = 0.0, 0.0, 0.0, 0.0, 0.0 for router in routers:

1 0