- tor-commits - lists.torproject.org

[metrics-tasks/master] modify readme
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit c3a1ffe065ab8699c52022716171ee7c7a686d21 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:44:31 2012 +0530 modify readme --- task-1854/README.txt | 17 ++++++++--------- 1 files changed, 8 insertions(+), 9 deletions(-) diff --git a/task-1854/README.txt b/task-1854/README.txt index 646dfea..7cf8833 100644 --- a/task-1854/README.txt +++ b/task-1854/README.txt @@ -1,9 +1,8 @@ -Basic algorithm - -1) find probability of each relay in pristine consensus -2) find probability of each relay in modified consensus -3) for every relay in modified consensus, - calculate prob_diff where - prob_diff = prob_in_pristine_consensus[relay] - prob_in_modified_consensus[relay] -4) find largest prob_diff -5) remove the relays with lowest adv_bw -6) go to step 2 +To run: + +* python pylinf.py -c path/to/consensus -s path/to/server_descriptors + +where - path/to/consensus is a dir containing consensus tarfiles + - path/to/server_descriptor is a dir containing server descriptor tarfiles + +The output is present in entropy.csv \ No newline at end of file

1 0

[metrics-tasks/master] store all server descs in memory
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit d008ecfa68fee00fd63e75129677edd1b0502337 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Wed Nov 21 23:19:12 2012 +0530 store all server descs in memory --- task-1854/pylinf.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 files changed, 35 insertions(+), 9 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 3bc6de1..4dff224 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -12,12 +12,14 @@ import sys import math import os import pygeoip +import tarfile import StringIO -import stem.descriptor +import stem.descriptor.server_descriptor from optparse import OptionParser from binascii import b2a_hex, a2b_base64, a2b_hex -from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor + +descriptors = {} class Router: def __init__(self): @@ -54,13 +56,7 @@ class Router: def get_advertised_bw(self, hex_digest): try: - with open(options.server_desc+'/'+hex_digest) as f: - data = f.read() - - desc_iter = stem.descriptor.server_descriptor.parse_file(StringIO.StringIO(data)) - desc_entries = list(desc_iter) - desc = desc_entries[0] - return min(desc.average_bandwidth, desc.burst_bandwidth, desc.observed_bandwidth) + return descriptors[self.hex_digest] except: return 0 @@ -74,6 +70,33 @@ def parse_bw_weights(values): except: return None +def load_server_desc(tar_file_path): + """ + tar_file_path -> 'string' or 'list' + represents path{s} to tar file{s} + """ + global descriptors + + if type(tar_file_path) == str: tar_file_path = [tar_file_path] + + for file_path in tar_file_path: + with tarfile.open(file_path) as tar_fh: + for member in tar_fh: + if not member.isfile(): + continue + + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + desc_iter = stem.descriptor.server_descriptor.parse_file( + StringIO.StringIO(data)) + desc_entries = list(desc_iter) + desc = desc_entries[0] + + # currently we require only advertised_bw + descriptors[desc.digest()] = min(desc.average_bandwidth, + desc.burst_bandwidth, + desc.observed_bandwidth) + def run(file_name): routers = [] router = None @@ -180,6 +203,9 @@ if __name__ == "__main__": gi_db = pygeoip.GeoIP(options.gi_db) as_db = pygeoip.GeoIP(options.as_db) + # load all server descs into memeory + load_server_desc(options.server_desc) + with open(options.output, 'w') as f: for file_name in os.listdir(options.consensus): string = run(os.path.join(options.consensus, file_name))

1 0

[metrics-tasks/master] read server_desc dir
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit c7e0af8c2ab407c4043775792c1c4fee2e4d515a Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:53:42 2012 +0530 read server_desc dir --- task-1854/pylinf.py | 19 +++++++++---------- 1 files changed, 9 insertions(+), 10 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index eece3cd..1f50dc9 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -14,7 +14,7 @@ import os import pygeoip import tarfile import StringIO -import stem.descriptor.server_descriptor +import stem.descriptor.server_descriptor as server_descriptor from optparse import OptionParser from binascii import b2a_hex, a2b_base64, a2b_hex @@ -55,11 +55,7 @@ class Router: return "" def get_advertised_bw(self, hex_digest): - try: - return descriptors[self.hex_digest] - except: - print "Unexpected error:", sys.exc_info()[0] - return 0 + return descriptors[self.hex_digest] def parse_bw_weights(values): data = {} @@ -91,8 +87,7 @@ def load_server_desc(tar_file_path): data=tar_file_data.read() try: - desc_iter = stem.descriptor.server_descriptor.parse_file( - StringIO.StringIO(data)) + desc_iter = server_descriptor.parse_file(StringIO.StringIO(data), validate=False) desc_entries = list(desc_iter) desc = desc_entries[0] @@ -101,7 +96,7 @@ def load_server_desc(tar_file_path): desc.burst_bandwidth, desc.observed_bandwidth) except: - print "Unexpected error:", sys.exc_info()[0] + print "Unexpected error:", sys.exc_info() continue tar_fh.close() @@ -216,8 +211,12 @@ if __name__ == "__main__": gi_db = pygeoip.GeoIP(options.gi_db) as_db = pygeoip.GeoIP(options.as_db) + server_desc_files = [] + # load all server descs into memeory - load_server_desc(options.server_desc) + for file_name in os.listdir(options.server_desc): + server_desc_files.append(os.path.join(options.server_desc, file_name)) + load_server_desc(server_desc_files) with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus):

1 0

[metrics-tasks/master] read consensus tarfiles, don't complain too much
by karsten＠torproject.org 25 Nov '12

25 Nov '12

commit cc8fab6bca860a6bd525ceffc727517433abccdc Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Fri Nov 23 17:36:54 2012 +0530 read consensus tarfiles, don't complain too much --- task-1854/pylinf.py | 89 +++++++++++++++++++++++++++++++------------------- 1 files changed, 55 insertions(+), 34 deletions(-) diff --git a/task-1854/pylinf.py b/task-1854/pylinf.py index 4dff224..eece3cd 100644 --- a/task-1854/pylinf.py +++ b/task-1854/pylinf.py @@ -35,8 +35,8 @@ class Router: hex_digest = b2a_hex(a2b_base64(values[2]+"=")) self.advertised_bw = self.get_advertised_bw(hex_digest) ip = values[5] - self.country = gi_db.country_code_by_addr(ip) - self.as_no = self.get_as_details(ip) + #self.country = gi_db.country_code_by_addr(ip) + #self.as_no = self.get_as_details(ip) def add_weights(self, values): self.bandwidth = int(values[0].split('=')[1]) @@ -58,6 +58,7 @@ class Router: try: return descriptors[self.hex_digest] except: + print "Unexpected error:", sys.exc_info()[0] return 0 def parse_bw_weights(values): @@ -68,6 +69,7 @@ def parse_bw_weights(values): data[key] = float(value) / 10000 return data except: + print "Unexpected error:", sys.exc_info()[0] return None def load_server_desc(tar_file_path): @@ -80,13 +82,15 @@ def load_server_desc(tar_file_path): if type(tar_file_path) == str: tar_file_path = [tar_file_path] for file_path in tar_file_path: - with tarfile.open(file_path) as tar_fh: - for member in tar_fh: - if not member.isfile(): - continue + tar_fh = tarfile.open(file_path) + for member in tar_fh: + if not member.isfile(): + continue - tar_file_data=tar_fh.extractfile(member) - data=tar_file_data.read() + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + + try: desc_iter = stem.descriptor.server_descriptor.parse_file( StringIO.StringIO(data)) desc_entries = list(desc_iter) @@ -96,36 +100,45 @@ def load_server_desc(tar_file_path): descriptors[desc.digest()] = min(desc.average_bandwidth, desc.burst_bandwidth, desc.observed_bandwidth) + except: + print "Unexpected error:", sys.exc_info()[0] + continue + tar_fh.close() -def run(file_name): +def run(data): routers = [] router = None result_string = [] Wed, Wee, Wgd, Wgg = 1, 1, 1, 1 + # parse consensus - with open(file_name, 'r') as f: - for line in f.readlines(): + for line in data.split("\n"): + try: key = line.split()[0] values = line.split()[1:] - if key =='r': - router = Router() - routers.append(router) - router.add_router_info(values) - elif key == 's': - router.add_flags(values) - elif key == 'w': - router.add_weights(values) - elif key == 'valid-after': - valid_after = ' '.join(values) - elif key == 'bandwidth-weights': - data = parse_bw_weights(values) - try: - Wed = data['Wed'] - Wee = data['Wee'] - Wgd = data['Wgd'] - Wgg = data['Wgg'] - except: - pass + except: + print "Unexpected error:", sys.exc_info()[0] + # we don't need sigs + continue + if key =='r': + router = Router() + routers.append(router) + router.add_router_info(values) + elif key == 's': + router.add_flags(values) + elif key == 'w': + router.add_weights(values) + elif key == 'valid-after': + valid_after = ' '.join(values) + elif key == 'bandwidth-weights': + data = parse_bw_weights(values) + try: + Wed = data['Wed'] + Wee = data['Wee'] + Wgd = data['Wgd'] + Wgg = data['Wgg'] + except: + pass if len(routers) <= 0: return @@ -206,8 +219,16 @@ if __name__ == "__main__": # load all server descs into memeory load_server_desc(options.server_desc) - with open(options.output, 'w') as f: + with open(options.output, 'w') as out_fh: for file_name in os.listdir(options.consensus): - string = run(os.path.join(options.consensus, file_name)) - if string: - f.write("%s\n" % (string)) + file_path = os.path.join(options.consensus, file_name) + tar_fh = tarfile.open(file_path) + for member in tar_fh: + if not member.isfile(): + continue + tar_file_data=tar_fh.extractfile(member) + data=tar_file_data.read() + output_string = run(data) + if output_string: + out_fh.write("%s\n" % (output_string)) + tar_fh.close()

1 0

[ooni-probe/master] Fix bug that mistakenly reports unsupported twisted version
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 113fd7f69afab9d5e4f8e98b195f4969b69920a5 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 17:07:36 2012 +0100 Fix bug that mistakenly reports unsupported twisted version --- ooni/utils/txagentwithsocks.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 7b000fc..49e5a3c 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -262,7 +262,7 @@ class Agent(client.Agent): self._socksport = socksport def request(self, method, uri, headers=None, bodyProducer=None): - if uri.startswith('shttp') or uri.startswith('httpo'): + if (uri.startswith('shttp') or uri.startswith('httpo')) and not HTTPConnectionPool: log.err("Requests over SOCKS are supported only with versions of Twisted >= 12.1.0") raise UnsupportedTwistedVersion return client.Agent.request(self, method, uri, headers, bodyProducer)

1 0

[ooni-probe/master] Fix bug that lead processPacket being called even if no packet has been received
by art＠torproject.org 25 Nov '12

25 Nov '12

commit fc24e4f29d25a2cef91525e4d47c39da3c6b4d8d Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:49:06 2012 +0100 Fix bug that lead processPacket being called even if no packet has been received --- ooni/utils/txscapy.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index fbe9c38..7902133 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -113,8 +113,9 @@ class ScapyFactory(abstract.FileDescriptor): def doRead(self): packet = self.super_socket.recv(MTU) - for protocol in self.protocols: - protocol.packetReceived(packet) + if packet: + for protocol in self.protocols: + protocol.packetReceived(packet) def registerProtocol(self, protocol): if not self.connected:

1 0

[ooni-probe/master] Handle the case in which we do not get a DNS lookup response and include it in the report
by art＠torproject.org 25 Nov '12

25 Nov '12

commit b2b2b54306f503335cc52edc82426baea45da242 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:30:17 2012 +0100 Handle the case in which we do not get a DNS lookup response and include it in the report --- nettests/core/dnsspoof.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nettests/core/dnsspoof.py b/nettests/core/dnsspoof.py index 48991ea..5c50c2f 100644 --- a/nettests/core/dnsspoof.py +++ b/nettests/core/dnsspoof.py @@ -38,8 +38,13 @@ class DNSSpoof(scapyt.ScapyTest): This is not tested, but the concept is that if the two responses match up then spoofing is occuring. """ - test_answer = report['test_a_lookup']['answered_packets'][0][1] - control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + try: + test_answer = report['test_a_lookup']['answered_packets'][0][1] + control_answer = report['test_control_a_lookup']['answered_packets'][0][1] + except IndexError: + self.report['spoofing'] = 'no_answer' + return + if test_answer[UDP] == control_answer[UDP]: self.report['spoofing'] = True else:

1 0

[ooni-probe/master] The dicts and lists used for keeping track of answers should be instance attributes not class attributes
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 15c501d57c1876a21ee60b53f0f38ee845ebe782 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:28:16 2012 +0100 The dicts and lists used for keeping track of answers should be instance attributes not class attributes --- ooni/utils/txscapy.py | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 8e479f8..fbe9c38 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -148,16 +148,7 @@ class ScapyProtocol(object): class ScapySender(ScapyProtocol): timeout = 5 - # This dict is used to store the unique hashes that allow scapy to - # match up request with answer - hr_sent_packets = {} - - # These are the packets we have received as answer to the ones we sent - answered_packets = [] - - # These are the packets we send - sent_packets = [] - + # This deferred will fire when we have finished sending a receiving packets. # Should we look for multiple answers for the same sent packet? multi = False @@ -222,6 +213,16 @@ class ScapySender(ScapyProtocol): self.factory.send(packet) def startSending(self, packets): + # This dict is used to store the unique hashes that allow scapy to + # match up request with answer + self.hr_sent_packets = {} + + # These are the packets we have received as answer to the ones we sent + self.answered_packets = [] + + # These are the packets we send + self.sent_packets = [] + self._start_time = time.time() self.d = defer.Deferred() self.sendPackets(packets)

1 0

[ooni-probe/master] Register scapyFactor if it is not registered
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 29505b25df7b133d634c9c0b5b5f33be12be7adf Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:31:47 2012 +0100 Register scapyFactor if it is not registered * Better debugging messages --- ooni/config.py | 1 + ooni/templates/scapyt.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index 5168c34..b017e87 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -14,6 +14,7 @@ from ooni.utils import Storage reports = Storage() basic = None cmd_line_options = None +scapyFactory = None def get_root_path(): this_directory = os.path.dirname(__file__) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index cb02300..a1eade4 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -16,7 +16,7 @@ from ooni.nettest import NetTestCase from ooni.utils import log from ooni import config -from ooni.utils.txscapy import ScapySender, getDefaultIface +from ooni.utils.txscapy import ScapySender, getDefaultIface, ScapyFactory class BaseScapyTest(NetTestCase): """ @@ -42,6 +42,10 @@ class BaseScapyTest(NetTestCase): ] def _setUp(self): + if not config.scapyFactory: + log.debug("Scapy factoring not set, registering it.") + config.scapyFactory = ScapyFactory(config.advanced.interface) + self.report['answer_flags'] = [] if self.localOptions['ipsrc']: config.checkIPsrc = 0 @@ -98,6 +102,7 @@ class BaseScapyTest(NetTestCase): scapySender = ScapySender() config.scapyFactory.registerProtocol(scapySender) + log.debug("Using sending with hash %s" % scapySender.__hash__) d = scapySender.startSending(packets) d.addCallback(self.finishedSendReceive)

1 0

[ooni-probe/master] Randomize source port by default in traceroute test and include source port in report
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 2e06b7c0a6c779f174d047d6a22dac36fe6d4922 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 16:31:04 2012 +0100 Randomize source port by default in traceroute test and include source port in report --- nettests/core/traceroute.py | 19 +++++++------------ 1 files changed, 7 insertions(+), 12 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index 45bbb67..f8311fd 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -20,8 +20,6 @@ class UsageOptions(usage.Options): ['srcport', 'p', None, 'Set the source port to a specific value (only applies to TCP and UDP)'] ] - optFlags = [['randomize','r', 'Randomize the source port']] - class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" author = "Arturo Filastò" @@ -34,12 +32,8 @@ class TracerouteTest(scapyt.BaseScapyTest): def get_sport(protocol): if self.localOptions['srcport']: return int(self.localOptions['srcport']) - elif self.localOptions['randomize']: + else: return random.randint(1024, 65535) - elif protocol == 'tcp': - return 80 - elif protocol == 'udp': - return 53 self.get_sport = get_sport @@ -65,11 +59,12 @@ class TracerouteTest(scapyt.BaseScapyTest): def finished(packets, port): log.debug("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets - self.report['hops_'+str(port)] = [] + self.report['hops_'+str(port)] = [] for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time, + 'sport': snd[UDP].sport } log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) @@ -98,7 +93,8 @@ class TracerouteTest(scapyt.BaseScapyTest): for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time, + 'sport': snd[UDP].sport } log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) @@ -126,7 +122,7 @@ class TracerouteTest(scapyt.BaseScapyTest): for snd, rcv in answered: report = {'ttl': snd.ttl, 'address': rcv.src, - 'rtt': rcv.time - snd.time + 'rtt': rcv.time - snd.time } log.debug("%s" % (report)) self.report['hops'].append(report) @@ -139,4 +135,3 @@ class TracerouteTest(scapyt.BaseScapyTest): d.addCallback(finished) return d -

1 0