November 2012 - tor-commits - lists.torproject.org

[metrics-tasks/master] add hex_digest attribute
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit 814c78aa6a6641c1be9efde1a6d58c6c6998fea0 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 22:24:12 2012 +0530 add hex_digest attribute --- task-1854/pylinf.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 1f50dc9..91ea2f8 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -25,6 +25,7 @@ class Router: def __init__(self): self.prob = None self.bandwidth = None + self.hex_digest = None self.advertised_bw = None self.country = None self.as_no = None @@ -32,8 +33,8 @@ class Router: self.is_guard = None def add_router_info(self, values): - hex_digest = b2a_hex(a2b_base64(values[2]+"=")) - self.advertised_bw = self.get_advertised_bw(hex_digest) + self.hex_digest = b2a_hex(a2b_base64(values[2]+"=")) + self.advertised_bw = self.get_advertised_bw(self.hex_digest) ip = values[5] #self.country = gi_db.country_code_by_addr(ip) #self.as_no = self.get_as_details(ip)

1 0

[metrics-tasks/master] pickle output
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit 9d02762d4e1cef6b24e5915d46a386fd0fee0f20 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Nov 24 00:03:32 2012 +0530 pickle output --- task-1854/pylinf.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 91ea2f8..9ccac13 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -13,6 +13,7 @@ import math import os import pygeoip import tarfile +import pickle import StringIO import stem.descriptor.server_descriptor as server_descriptor @@ -25,8 +26,8 @@ class Router: def __init__(self): self.prob = None self.bandwidth = None - self.hex_digest = None self.advertised_bw = None + self.hex_digest = None self.country = None self.as_no = None self.is_exit = None @@ -56,7 +57,7 @@ class Router: return "" def get_advertised_bw(self, hex_digest): - return descriptors[self.hex_digest] + return descriptors[self.hex_digest.upper()] def parse_bw_weights(values): data = {} @@ -219,6 +220,10 @@ if __name__ == "__main__": server_desc_files.append(os.path.join(options.server_desc, file_name)) load_server_desc(server_desc_files) + # Pickle descriptors + with open('data.pkl', 'wb') as output: + pickle.dump(descriptors, output) + with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus): file_path = os.path.join(options.consensus, file_name)

1 0

[metrics-tasks/master] Merge remote-tracking branch 'gsathya/bug_1854_v2'
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit 99a21d55b607daeb79033642ffda0ce288528992 Merge: 3f7a92b 6da2e5b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Nov 25 14:04:40 2012 -0500 Merge remote-tracking branch 'gsathya/bug_1854_v2' task-1854/README.txt | 17 +++--- task-1854/pylinf.py | 139 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 105 insertions(+), 51 deletions(-)

1 0

[metrics-tasks/master] don't crash if you can't find a desc
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit 6da2e5b741a96eb543cf1adcf9eb64a1c2877ec2 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Nov 24 15:23:42 2012 +0530 don't crash if you can't find a desc --- task-1854/pylinf.py | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index eba39ea..9d324d6 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -57,7 +57,9 @@ class Router: return "" def get_advertised_bw(self, hex_digest): - return descriptors[self.hex_digest.upper()] + try: + return descriptors[self.hex_digest.upper()] + except: return 0 def parse_bw_weights(values): data = {} @@ -193,7 +195,7 @@ def parse_args(): parser.add_option("-a", "--as", dest="as_db", default="GeoIPASNum.dat", help="Input AS GeoIP database") parser.add_option("-s", "--server_desc", dest="server_desc", - default="data/relay-descriptors/server-descriptors/", help="Server descriptors directory") + default=False, help="Server descriptors directory") parser.add_option("-o", "--output", dest="output", default="entropy.csv", help="Output filename") parser.add_option("-c", "--consensus", dest="consensus", default="in/consensus", @@ -216,7 +218,8 @@ if __name__ == "__main__": if options.pickled_data: with open('data.pkl', 'rb') as pkl_input: descriptors = pickle.load(pkl_input) - else: + + if options.server_desc: # load all server descs into memeory for file_name in os.listdir(options.server_desc): server_desc_files.append(os.path.join(options.server_desc, file_name))

1 0

[metrics-tasks/master] pickle data
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit 4167cc7499c7609f200417d03f34b3fc03000b9a Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Nov 24 09:24:30 2012 +0530 pickle data --- task-1854/pylinf.py | 46 +++++++++++++++++++++++----------------------- 1 files changed, 23 insertions(+), 23 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 9ccac13..eba39ea 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -67,7 +67,6 @@ def parse_bw_weights(values): data[key] = float(value) / 10000 return data except: - print "Unexpected error:", sys.exc_info()[0] return None def load_server_desc(tar_file_path): @@ -88,18 +87,16 @@ def load_server_desc(tar_file_path): tar_file_data=tar_fh.extractfile(member) data=tar_file_data.read() - try: - desc_iter = server_descriptor.parse_file(StringIO.StringIO(data), validate=False) - desc_entries = list(desc_iter) - desc = desc_entries[0] - - # currently we require only advertised_bw - descriptors[desc.digest()] = min(desc.average_bandwidth, - desc.burst_bandwidth, - desc.observed_bandwidth) - except: - print "Unexpected error:", sys.exc_info() - continue + + desc_iter = server_descriptor.parse_file(StringIO.StringIO(data), validate=False) + desc_entries = list(desc_iter) + desc = desc_entries[0] + + # currently we require only advertised_bw + descriptors[desc.digest()] = min(desc.average_bandwidth, + desc.burst_bandwidth, + desc.observed_bandwidth) + tar_fh.close() def run(data): @@ -114,8 +111,6 @@ def run(data): key = line.split()[0] values = line.split()[1:] except: - print "Unexpected error:", sys.exc_info()[0] - # we don't need sigs continue if key =='r': router = Router() @@ -203,6 +198,8 @@ def parse_args(): help="Output filename") parser.add_option("-c", "--consensus", dest="consensus", default="in/consensus", help="Input consensus dir") + parser.add_option("-p", "--pickled_data", dest="pickled_data", default=False, + help="Input pickled file") (options, args) = parser.parse_args() @@ -214,15 +211,18 @@ if __name__ == "__main__": as_db = pygeoip.GeoIP(options.as_db) server_desc_files = [] + global descriptors - # load all server descs into memeory - for file_name in os.listdir(options.server_desc): - server_desc_files.append(os.path.join(options.server_desc, file_name)) - load_server_desc(server_desc_files) - - # Pickle descriptors - with open('data.pkl', 'wb') as output: - pickle.dump(descriptors, output) + if options.pickled_data: + with open('data.pkl', 'rb') as pkl_input: + descriptors = pickle.load(pkl_input) + else: + # load all server descs into memeory + for file_name in os.listdir(options.server_desc): + server_desc_files.append(os.path.join(options.server_desc, file_name)) + load_server_desc(server_desc_files) + with open('data.pkl', 'wb') as output: + pickle.dump(descriptors, output) with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus):

1 0

[metrics-tasks/master] modify readme
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit c3a1ffe065ab8699c52022716171ee7c7a686d21 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:44:31 2012 +0530 modify readme --- task-1854/README.txt | 17 ++++++++--------- 1 files changed, 8 insertions(+), 9 deletions(-) diff --git a/task-1854/README.txt b/task-1854/README.txt index 646dfea..7cf8833 100644 --- a/task-1854/README.txt +++ b/task-1854/README.txt @@ -1,9 +1,8 @@ -Basic algorithm - -1) find probability of each relay in pristine consensus -2) find probability of each relay in modified consensus -3) for every relay in modified consensus, - calculate prob_diff where - prob_diff = prob_in_pristine_consensus[relay] - prob_in_modified_consensus[relay] -4) find largest prob_diff -5) remove the relays with lowest adv_bw -6) go to step 2 +To run: + +* python pylinf.py -c path/to/consensus -s path/to/server_descriptors + +where - path/to/consensus is a dir containing consensus tarfiles + - path/to/server_descriptor is a dir containing server descriptor tarfiles + +The output is present in entropy.csv \ No newline at end of file

1 0

[metrics-tasks/master] store all server descs in memory
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit d008ecfa68fee00fd63e75129677edd1b0502337 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Wed Nov 21 23:19:12 2012 +0530 store all server descs in memory --- task-1854/pylinf.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 files changed, 35 insertions(+), 9 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 3bc6de1..4dff224 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -12,12 +12,14 @@ import sys import math import os import pygeoip +import tarfile import StringIO -import stem.descriptor +import stem.descriptor.server_descriptor from optparse import OptionParser from binascii import b2a_hex, a2b_base64, a2b_hex -from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor + +descriptors = {} class Router: def __init__(self): @@ -54,13 +56,7 @@ class Router: def get_advertised_bw(self, hex_digest): try: - with open(options.server_desc+'/'+hex_digest) as f: - data = f.read() - - desc_iter = stem.descriptor.server_descriptor.parse_file(StringIO.StringIO(data)) - desc_entries = list(desc_iter) - desc = desc_entries[0] - return min(desc.average_bandwidth, desc.burst_bandwidth, desc.observed_bandwidth) + return descriptors[self.hex_digest] except: return 0 @@ -74,6 +70,33 @@ def parse_bw_weights(values): except: return None +def load_server_desc(tar_file_path): + """ + tar_file_path -> 'string' or 'list' + represents path{s} to tar file{s} + """ + global descriptors + + if type(tar_file_path) == str: tar_file_path = [tar_file_path] + + for file_path in tar_file_path: + with tarfile.open(file_path) as tar_fh: + for member in tar_fh: + if not member.isfile(): + continue + + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + desc_iter = stem.descriptor.server_descriptor.parse_file( + StringIO.StringIO(data)) + desc_entries = list(desc_iter) + desc = desc_entries[0] + + # currently we require only advertised_bw + descriptors[desc.digest()] = min(desc.average_bandwidth, + desc.burst_bandwidth, + desc.observed_bandwidth) + def run(file_name): routers = [] router = None @@ -180,6 +203,9 @@ if __name__ == "__main__": gi_db = pygeoip.GeoIP(options.gi_db) as_db = pygeoip.GeoIP(options.as_db) + # load all server descs into memeory + load_server_desc(options.server_desc) + with open(options.output, 'w') as f: for file_name in os.listdir(options.consensus): string = run(os.path.join(options.consensus, file_name))

1 0

[metrics-tasks/master] read server_desc dir
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit c7e0af8c2ab407c4043775792c1c4fee2e4d515a Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:53:42 2012 +0530 read server_desc dir --- task-1854/pylinf.py | 19 +++++++++---------- 1 files changed, 9 insertions(+), 10 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index eece3cd..1f50dc9 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -14,7 +14,7 @@ import os import pygeoip import tarfile import StringIO -import stem.descriptor.server_descriptor +import stem.descriptor.server_descriptor as server_descriptor from optparse import OptionParser from binascii import b2a_hex, a2b_base64, a2b_hex @@ -55,11 +55,7 @@ class Router: return "" def get_advertised_bw(self, hex_digest): - try: - return descriptors[self.hex_digest] - except: - print "Unexpected error:", sys.exc_info()[0] - return 0 + return descriptors[self.hex_digest] def parse_bw_weights(values): data = {} @@ -91,8 +87,7 @@ def load_server_desc(tar_file_path): data=tar_file_data.read() try: - desc_iter = stem.descriptor.server_descriptor.parse_file( - StringIO.StringIO(data)) + desc_iter = server_descriptor.parse_file(StringIO.StringIO(data), validate=False) desc_entries = list(desc_iter) desc = desc_entries[0] @@ -101,7 +96,7 @@ def load_server_desc(tar_file_path): desc.burst_bandwidth, desc.observed_bandwidth) except: - print "Unexpected error:", sys.exc_info()[0] + print "Unexpected error:", sys.exc_info() continue tar_fh.close() @@ -216,8 +211,12 @@ if __name__ == "__main__": gi_db = pygeoip.GeoIP(options.gi_db) as_db = pygeoip.GeoIP(options.as_db) + server_desc_files = [] + # load all server descs into memeory - load_server_desc(options.server_desc) + for file_name in os.listdir(options.server_desc): + server_desc_files.append(os.path.join(options.server_desc, file_name)) + load_server_desc(server_desc_files) with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus):

1 0

[metrics-tasks/master] read consensus tarfiles, don't complain too much
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit cc8fab6bca860a6bd525ceffc727517433abccdc Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:36:54 2012 +0530 read consensus tarfiles, don't complain too much --- task-1854/pylinf.py | 89 +++++++++++++++++++++++++++++++------------------- 1 files changed, 55 insertions(+), 34 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 4dff224..eece3cd 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -35,8 +35,8 @@ class Router: hex_digest = b2a_hex(a2b_base64(values[2]+"=")) self.advertised_bw = self.get_advertised_bw(hex_digest) ip = values[5] - self.country = gi_db.country_code_by_addr(ip) - self.as_no = self.get_as_details(ip) + #self.country = gi_db.country_code_by_addr(ip) + #self.as_no = self.get_as_details(ip) def add_weights(self, values): self.bandwidth = int(values[0].split('=')[1]) @@ -58,6 +58,7 @@ class Router: try: return descriptors[self.hex_digest] except: + print "Unexpected error:", sys.exc_info()[0] return 0 def parse_bw_weights(values): @@ -68,6 +69,7 @@ def parse_bw_weights(values): data[key] = float(value) / 10000 return data except: + print "Unexpected error:", sys.exc_info()[0] return None def load_server_desc(tar_file_path): @@ -80,13 +82,15 @@ def load_server_desc(tar_file_path): if type(tar_file_path) == str: tar_file_path = [tar_file_path] for file_path in tar_file_path: - with tarfile.open(file_path) as tar_fh: - for member in tar_fh: - if not member.isfile(): - continue + tar_fh = tarfile.open(file_path) + for member in tar_fh: + if not member.isfile(): + continue - tar_file_data=tar_fh.extractfile(member) - data=tar_file_data.read() + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + + try: desc_iter = stem.descriptor.server_descriptor.parse_file( StringIO.StringIO(data)) desc_entries = list(desc_iter) @@ -96,36 +100,45 @@ def load_server_desc(tar_file_path): descriptors[desc.digest()] = min(desc.average_bandwidth, desc.burst_bandwidth, desc.observed_bandwidth) + except: + print "Unexpected error:", sys.exc_info()[0] + continue + tar_fh.close() -def run(file_name): +def run(data): routers = [] router = None result_string = [] Wed, Wee, Wgd, Wgg = 1, 1, 1, 1 + # parse consensus - with open(file_name, 'r') as f: - for line in f.readlines(): + for line in data.split("\n"): + try: key = line.split()[0] values = line.split()[1:] - if key =='r': - router = Router() - routers.append(router) - router.add_router_info(values) - elif key == 's': - router.add_flags(values) - elif key == 'w': - router.add_weights(values) - elif key == 'valid-after': - valid_after = ' '.join(values) - elif key == 'bandwidth-weights': - data = parse_bw_weights(values) - try: - Wed = data['Wed'] - Wee = data['Wee'] - Wgd = data['Wgd'] - Wgg = data['Wgg'] - except: - pass + except: + print "Unexpected error:", sys.exc_info()[0] + # we don't need sigs + continue + if key =='r': + router = Router() + routers.append(router) + router.add_router_info(values) + elif key == 's': + router.add_flags(values) + elif key == 'w': + router.add_weights(values) + elif key == 'valid-after': + valid_after = ' '.join(values) + elif key == 'bandwidth-weights': + data = parse_bw_weights(values) + try: + Wed = data['Wed'] + Wee = data['Wee'] + Wgd = data['Wgd'] + Wgg = data['Wgg'] + except: + pass if len(routers) <= 0: return @@ -206,8 +219,16 @@ if __name__ == "__main__": # load all server descs into memeory load_server_desc(options.server_desc) - with open(options.output, 'w') as f: + with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus): - string = run(os.path.join(options.consensus, file_name)) - if string: - f.write("%s\n" % (string)) + file_path = os.path.join(options.consensus, file_name) + tar_fh = tarfile.open(file_path) + for member in tar_fh: + if not member.isfile(): + continue + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + output_string = run(data) + if output_string: + out_fh.write("%s\n" % (output_string)) + tar_fh.close()

1 0

[ooni-probe/master] Fix bug that mistakenly reports unsupported twisted version
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 113fd7f69afab9d5e4f8e98b195f4969b69920a5 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 17:07:36 2012 +0100 Fix bug that mistakenly reports unsupported twisted version --- ooni/utils/txagentwithsocks.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 7b000fc..49e5a3c 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -262,7 +262,7 @@ class Agent(client.Agent): self._socksport = socksport def request(self, method, uri, headers=None, bodyProducer=None): - if uri.startswith('shttp') or uri.startswith('httpo'): + if (uri.startswith('shttp') or uri.startswith('httpo')) and not HTTPConnectionPool: log.err("Requests over SOCKS are supported only with versions of Twisted >= 12.1.0") raise UnsupportedTwistedVersion return client.Agent.request(self, method, uri, headers, bodyProducer)

1 0