July 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Main concept of b0wser implemented
by art＠torproject.org 20 Jul '12

20 Jul '12

commit db0e863b9133f7e0d17de365e55c432dbf448878 Author: Arturo Filastò <hellais(a)torproject.org> Date: Fri Jul 13 21:15:58 2012 +0200 Main concept of b0wser implemented --- ooni/example_plugins/examplescapy.py | 2 +- ooni/plugins/chinatrigger.py | 2 +- ooni/protocols/b0wser.py | 45 ++++++++++++++++++---------- ooni/protocols/scapy.py | 55 ---------------------------------- oonib/b0wser.py | 3 +- 5 files changed, 33 insertions(+), 74 deletions(-) diff --git a/ooni/example_plugins/examplescapy.py b/ooni/example_plugins/examplescapy.py index 144afee..21a919d 100644 --- a/ooni/example_plugins/examplescapy.py +++ b/ooni/example_plugins/examplescapy.py @@ -6,7 +6,7 @@ from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset from ooni.utils import log -from ooni.protocols.scapy import ScapyTest +from ooni.protocols.scapyproto import ScapyTest from ooni.lib.txscapy import txsr, txsend diff --git a/ooni/plugins/chinatrigger.py b/ooni/plugins/chinatrigger.py index 538ef0b..9873518 100644 --- a/ooni/plugins/chinatrigger.py +++ b/ooni/plugins/chinatrigger.py @@ -10,7 +10,7 @@ from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset from ooni.utils import log -from ooni.protocols.scapy import ScapyTest +from ooni.protocols.scapyproto import ScapyTest from ooni.lib.txscapy import txsr, txsend diff --git a/ooni/protocols/b0wser.py b/ooni/protocols/b0wser.py index 94e535f..6a01d96 100644 --- a/ooni/protocols/b0wser.py +++ b/ooni/protocols/b0wser.py @@ -1,7 +1,11 @@ +from twisted.internet import protocol, defer +from twisted.internet.error import ConnectionDone + from ooni.utils import log +from ooni.plugoo import reports import sys -from scapy.all import * # XXX recommended way of importing scapy? +from scapy.all import * import yaml def get_b0wser_dictionary_from_pcap(filename): @@ -23,12 +27,12 @@ def get_b0wser_dictionary_from_pcap(filename): """ pcap assumptions: - pcap only contains packets exchanged between a Tor client and a Tor - server. (This assumption makes sure that there are only two IP - addresses in the pcap file) + pcap only contains packets exchanged between a Tor client and a Tor server. + (This assumption makes sure that there are only two IP addresses in the + pcap file) - The first packet of the pcap is sent from the client to the server. - (This assumption is used to get the IP address of the client.) + The first packet of the pcap is sent from the client to the server. (This + assumption is used to get the IP address of the client.) All captured packets are TLS packets: that is TCP session establishment/teardown packets should be filtered out (no SYN/SYN+ACK) @@ -59,7 +63,7 @@ def get_b0wser_dictionary_from_pcap(filename): else: raise("Detected third IP address! pcap is corrupted.") - return yaml.dump(messages) + return messages class Mutator: idx = 0 @@ -115,7 +119,8 @@ class Mutator: returns True if another mutation is available. returns False if all the possible mutations have been done. """ - if self.step > len(self.steps): + if (self.step + 1) > len(self.steps): + # Hack to stop once we have gone through all the steps self.waiting = True return False @@ -123,7 +128,10 @@ class Mutator: current_idx = self.idx current_step = self.step current_data = self.steps[current_step]['data'] - data_to_receive = self.steps[current_step]['wait'] + try: + data_to_receive = len(self.steps[current_step +1 ]['data']) + except: + print "No more data to receive" if self.waiting and self.waiting_step == data_to_receive: log.debug("I am no longer waiting.") @@ -170,14 +178,14 @@ class B0wserProtocol(protocol.Protocol): For every instance of protocol there is only 1 mutation. Once the last step is reached the connection is closed on the serverside. """ - steps = [{'data': "STEP1", 'wait': 4}, - {'data': "STEP2", 'wait': 4}, - {'data': "STEP3", 'wait': 4}] + steps = [] mutator = None + role = 'client' state = 0 total_states = len(steps) - 1 received_data = 0 + to_receive_data = 0 report = reports.Report('b0wser', 'b0wser.yamlooni') def next_state(self): @@ -185,8 +193,12 @@ class B0wserProtocol(protocol.Protocol): This is called once I have completed one step of the protocol and need to proceed to the next step. """ - data = self.mutator.get_mutation(self.state) - self.transport.write(data) + if self.role is self.steps[self.state]['sender']: + data = self.mutator.get_mutation(self.state) + self.transport.write(data) + self.to_receive_data = 0 + else: + self.to_receive_data = len(self.steps[self.state]['data']) self.state += 1 self.received_data = 0 @@ -204,8 +216,9 @@ class B0wserProtocol(protocol.Protocol): self.report(report) self.transport.loseConnection() return + self.received_data += len(data) - if self.received_data >= self.steps[self.state]['wait']: + if self.received_data >= self.to_receive_data: print "Moving to next state %s" % self.state self.next_state() @@ -234,7 +247,7 @@ class B0wserProtocol(protocol.Protocol): or it may be because of a censorship event. """ report = {'reason': reason, 'proto_state': self.state, - 'mutator_state': self.mutator.state(), 'trigger': None} + 'trigger': None, 'mutator_state': self.mutator.state()} if self.state < self.total_states: report['trigger'] = 'did not finish state walk' diff --git a/ooni/protocols/scapy.py b/ooni/protocols/scapy.py deleted file mode 100644 index 4166146..0000000 --- a/ooni/protocols/scapy.py +++ /dev/null @@ -1,55 +0,0 @@ -import random -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni.utils import log - -from ooni.lib.txscapy import txsr, txsend - -class ScapyTest(OONITest): - """ - A utility class for writing scapy driven OONI tests. - """ - - receive = True - timeout = None - pcapfile = 'scapytest.pcap' - def initialize(self, reactor=None): - - if not self.reactor: - from twisted.internet import reactor - self.reactor = reactor - - self.request = {} - self.response = {} - - def experiment(self, args): - log.msg("Running experiment") - if self.receive: - log.msg("Sending and receiving packets.") - d = txsr(self.build_packets(), pcapfile=self.pcapfile, - timeout=self.timeout) - else: - log.msg("Sending packets.") - d = txsend(self.build_packets()) - - def finished(data): - log.msg("Finished sending") - return data - - d.addCallback(finished) - return d - - def build_packets(self): - """ - Override this method to build scapy packets. - """ - from scapy.all import IP, TCP - return IP()/TCP() - - def load_assets(self): - return {} - diff --git a/oonib/b0wser.py b/oonib/b0wser.py index 33501de..0bb89fb 100644 --- a/oonib/b0wser.py +++ b/oonib/b0wser.py @@ -25,6 +25,7 @@ class B0wserServer(protocol.ServerFactory): print "Moving on to next mutation" if not self.mutations[addr.host].next_mutation(): self.mutations.pop(addr.host) - p.mutator = self.mutations[addr.host] + else: + p.mutator = self.mutations[addr.host] return p

1 0

[ooni-probe/master] Add a pcap parsing function.
by art＠torproject.org 20 Jul '12

20 Jul '12

commit b9b80c43e84bd37c95037a7a73dad0c29d74c3fe Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Jul 12 18:26:27 2012 +0200 Add a pcap parsing function. --- ooni/protocols/b0wser.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 61 insertions(+), 0 deletions(-) diff --git a/ooni/protocols/b0wser.py b/ooni/protocols/b0wser.py index ae6b002..ed82781 100644 --- a/ooni/protocols/b0wser.py +++ b/ooni/protocols/b0wser.py @@ -1,5 +1,66 @@ from ooni.utils import log +import sys +from scapy.all import * # XXX recommended way of importing scapy? +import yaml + +def get_b0wser_dictionary_from_pcap(filename): + """ + @param filename: Filesystem path to the pcap. + + Returns: + [{"sender": "client", "data": "\x17\x52\x15"}, {"sender": "server", "data": "\x17\x15\x13"}] + """ + packets = rdpcap(filename) + + checking_first_packet = True + client_ip_addr = None + server_ip_addr = None + + ssl_packets = [] + messages = [] + + """ + pcap assumptions: + + pcap only contains packets exchanged between a Tor client and a Tor + server. (This assumption makes sure that there are only two IP + addresses in the pcap file) + + The first packet of the pcap is sent from the client to the server. + (This assumption is used to get the IP address of the client.) + + All captured packets are TLS packets: that is TCP session + establishment/teardown packets should be filtered out (no SYN/SYN+ACK) + """ + + """Minimally validate the pcap and also find out what's the client + and server IP addresses.""" + for packet in packets: + if checking_first_packet: + client_ip_addr = packet[IP].src + checking_first_packet = False + else: + if packet[IP].src != client_ip_addr: + server_ip_addr = packet[IP].src + + try: + if (packet[Raw]): + ssl_packets.append(packet) + except IndexError: + pass + + """Form our list.""" + for packet in ssl_packets: + if packet[IP].src == client_ip_addr: + messages.append({"sender": "client", "data": str(packet[Raw])}) + elif packet[IP].src == server_ip_addr: + messages.append({"sender": "server", "data": str(packet[Raw])}) + else: + raise("Detected third IP address! pcap is corrupted.") + + return yaml.dump(messages) + class Mutator: idx = 0 step = 0

1 0

[ooni-probe/master] Implement client side component of b0wser
by art＠torproject.org 20 Jul '12

20 Jul '12

commit 7d604af457a17fdf6cb433ce46a3f691ad566658 Author: Arturo Filastò <hellais(a)torproject.org> Date: Fri Jul 13 21:16:26 2012 +0200 Implement client side component of b0wser --- ooni/plugins/b0wser.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 85 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/b0wser.py b/ooni/plugins/b0wser.py new file mode 100644 index 0000000..5d6687b --- /dev/null +++ b/ooni/plugins/b0wser.py @@ -0,0 +1,85 @@ +""" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +""" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, endpoints + +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni.protocols import b0wser +from ooni.utils import log + +class B0wserClientProtocol(b0wser.B0wserProtocol): + def connectionMade(self): + self.next_state() + + def connectionLost(self, reason): + print "LOST!" + +class B0wserClientFactory(protocol.ClientFactory): + protocol = B0wserClientProtocol + mutator = None + steps = None + + def buildProtocol(self, addr): + p = self.protocol() + p.factory = self + if self.steps: + p.steps = self.steps + + if not self.mutator: + self.mutator = b0wser.Mutator(p.steps) + p.mutator = self.mutator + else: + print "Moving on to next mutation" + self.mutator.next_mutation() + return p + + def clientConnectionFailed(self, reason): + print "We failed connecting the the OONIB" + print "Cannot perform test. Perhaps it got blocked?" + print "Please report this to tor-assistants(a)torproject.org" + + def clientConnectionLost(self, reason): + print "Connection Lost." + +class b0wserArgs(usage.Options): + optParameters = [['pcap', 'f', None, 'PCAP file to take as input'], + ['host', 'h', None, 'Target Hostname'], + ['port', 'p', None, 'Target port number'], + ['resume', 'r', 0, 'Resume at this index']] + +class b0wserTest(OONITest): + implements(IPlugin, ITest) + + shortName = "b0wser" + description = "b0wser" + requirements = None + options = b0wserArgs + blocking = False + + def initialize(self): + #pass + self.factory = B0wserClientFactory() + + def experiment(self, args): + steps = b0wser.get_b0wser_dictionary_from_pcap(self.local_options['pcap']) + print steps + self.factory.steps = steps + host = self.local_options['host'] + port = int(self.local_options['port']) + log.msg("Connecting to %s:%s" % (host, port)) + endpoint = endpoints.TCP4ClientEndpoint(self.reactor, host, port) + return endpoint.connect(self.factory) + #return endpoint.connect(B0wserClientFactory) + + def load_assets(self): + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +b0wsertest = b0wserTest(None, None, None)

1 0

[ooni-probe/master] Get b0wser to start doing something useful
by art＠torproject.org 20 Jul '12

20 Jul '12

commit 4dbb7426ec0147e6efa946e287f5e6a7f682252f Author: Arturo Filastò <hellais(a)torproject.org> Date: Tue Jul 17 22:10:24 2012 +0200 Get b0wser to start doing something useful --- ooni/plugins/b0wser.py | 23 +++++++++++++++++++---- ooni/plugoo/tests.py | 6 ++++-- ooni/protocols/b0wser.py | 28 +++++++++++++++++++++++++--- oonib/b0wser.py | 6 ++++-- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/ooni/plugins/b0wser.py b/ooni/plugins/b0wser.py index 5d6687b..3121378 100644 --- a/ooni/plugins/b0wser.py +++ b/ooni/plugins/b0wser.py @@ -62,14 +62,23 @@ class b0wserTest(OONITest): options = b0wserArgs blocking = False + local_options = None + + steps = None def initialize(self): + if not self.local_options: + return #pass self.factory = B0wserClientFactory() + self.steps = b0wser.get_b0wser_dictionary_from_pcap(self.local_options['pcap']) + + def control(self, exp_res, args): + mutation = self.factory.mutator.get_mutation(0) + return {'mutation_number': args['mutation'], 'value': mutation} def experiment(self, args): - steps = b0wser.get_b0wser_dictionary_from_pcap(self.local_options['pcap']) - print steps - self.factory.steps = steps + log.msg("Doing mutation %s" % args['mutation']) + self.factory.steps = self.steps host = self.local_options['host'] port = int(self.local_options['port']) log.msg("Connecting to %s:%s" % (host, port)) @@ -78,7 +87,13 @@ class b0wserTest(OONITest): #return endpoint.connect(B0wserClientFactory) def load_assets(self): - return {} + if not self.steps: + print "No asset!" + return {} + mutations = 0 + for x in self.steps: + mutations += len(x['data']) + return {'mutation': range(mutations)} # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index a99a144..9b6ea26 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -24,6 +24,7 @@ class OONITest(object): """ # By default we set this to False, meaning that we don't block blocking = False + reactor = None def __init__(self, local_options, global_options, report, ooninet=None, reactor=None): @@ -31,13 +32,14 @@ class OONITest(object): self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options - self.assets = self.load_assets() self.report = report #self.ooninet = ooninet self.reactor = reactor - self.initialize() self.result = {} + self.initialize() + self.assets = self.load_assets() + def initialize(self): """ Override this method if you are interested in having some extra diff --git a/ooni/protocols/b0wser.py b/ooni/protocols/b0wser.py index 6a01d96..5d55c94 100644 --- a/ooni/protocols/b0wser.py +++ b/ooni/protocols/b0wser.py @@ -101,7 +101,9 @@ class Mutator: Returns a dict containg the packet index and the step number. """ - return {'idx': self.idx, 'step': self.step} + print "[Mutator.state()] Giving out my internal state." + current_state = {'idx': self.idx, 'step': self.step} + return current_state def next_mutation(self): """ @@ -121,6 +123,8 @@ class Mutator: """ if (self.step + 1) > len(self.steps): # Hack to stop once we have gone through all the steps + print "[Mutator.next_mutation()] I believe I have gone over all steps" + print " Stopping!" self.waiting = True return False @@ -130,20 +134,24 @@ class Mutator: current_data = self.steps[current_step]['data'] try: data_to_receive = len(self.steps[current_step +1 ]['data']) + print "[Mutator.next_mutation()] Managed to receive some data." except: - print "No more data to receive" + print "[Mutator.next_mutation()] No more data to receive." if self.waiting and self.waiting_step == data_to_receive: + print "[Mutator.next_mutation()] I am no longer waiting" log.debug("I am no longer waiting.") self.waiting = False self.waiting_step = 0 self.idx = 0 elif self.waiting: + print "[Mutator.next_mutation()] Waiting some more." log.debug("Waiting some more.") self.waiting_step += 1 elif current_idx >= len(current_data): + print "[Mutator.next_mutation()] Entering waiting mode." log.debug("Entering waiting mode.") self.step += 1 self.idx = 0 @@ -164,7 +172,7 @@ class Mutator: returns the mutated packet for the specified step. """ if step != self.step or self.waiting: - log.debug("I am not going to do anything :)") + log.debug("[Mutator.get_mutation()] I am not going to do anything :)") return self.steps[step]['data'] data = self.steps[step]['data'] @@ -193,12 +201,19 @@ class B0wserProtocol(protocol.Protocol): This is called once I have completed one step of the protocol and need to proceed to the next step. """ + if not self.mutator: + print "[B0wserProtocol.next_state] No mutator. There is no point to stay on this earth." + self.transport.loseConnection() + return if self.role is self.steps[self.state]['sender']: + print "[B0wserProtocol.next_state] I am a sender" data = self.mutator.get_mutation(self.state) self.transport.write(data) self.to_receive_data = 0 else: + print "[B0wserProtocol.next_state] I am a receiver" self.to_receive_data = len(self.steps[self.state]['data']) + self.state += 1 self.received_data = 0 @@ -209,6 +224,10 @@ class B0wserProtocol(protocol.Protocol): @param data: the data that has been sent by the client. """ + if not self.mutator: + print "I don't have a mutator. My life means nothing." + self.transport.loseConnection() + return if len(self.steps) <= self.state: print "I have reached the end of the state machine" print "Censorship fingerprint bruteforced!" @@ -246,6 +265,9 @@ class B0wserProtocol(protocol.Protocol): The connection was closed. This may be because of a legittimate reason or it may be because of a censorship event. """ + if not self.mutator: + print "Terminated because of little interest in life." + return report = {'reason': reason, 'proto_state': self.state, 'trigger': None, 'mutator_state': self.mutator.state()} diff --git a/oonib/b0wser.py b/oonib/b0wser.py index 0bb89fb..4500075 100644 --- a/oonib/b0wser.py +++ b/oonib/b0wser.py @@ -25,7 +25,9 @@ class B0wserServer(protocol.ServerFactory): print "Moving on to next mutation" if not self.mutations[addr.host].next_mutation(): self.mutations.pop(addr.host) - else: - p.mutator = self.mutations[addr.host] + try: + p.mutator = self.mutations[addr.host] + except: + pass return p

1 0

[metrics-tasks/master] Parse consensus_weight_fraction from details.json (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 415d92c63c90ac6ea3d47d61945c75068fe8286a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 12:44:58 2012 +0200 Parse consensus_weight_fraction from details.json (#6329). --- task-6329/tor-relays-stats.py | 67 +++++++++++++++++------------------------ 1 files changed, 28 insertions(+), 39 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 05d9373..0fbe3aa 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -21,14 +21,6 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_total_consensus_weight(self, relays=None): - if relays is None: - relays = self.get_relays() - total_consensus_weight = 0 - for relay in relays: - total_consensus_weight += relay['consensus_weight'] - return total_consensus_weight - def get_relays(self, flags=[], countries='', as_sets=[]): relays = [] for relay in self.data['relays']: @@ -52,23 +44,22 @@ class RelayStats(object): country = relay.get('country', None) if country not in countries: countries[country] = 0 - countries[country] += relay['consensus_weight'] + countries[country] += relay['consensus_weight_fraction'] ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) ranking.reverse() - total_consensus_weight = self.get_total_consensus_weight() for country, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0 / total_consensus_weight, country) + print "%8.4f%% %s" % (weight * 100.0, country) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for as_set, weight in ranking[count:]: - other_consensus_weight += weight - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - selection_consensus_weight = 0 + other_consensus_weight_fraction += weight + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + selection_consensus_weight_fraction = 0 for as_set, weight in ranking: - selection_consensus_weight += weight - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + selection_consensus_weight_fraction += weight + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) def output_as_sets(self, count='10', flags='', countries=''): count = int(count) @@ -79,23 +70,22 @@ class RelayStats(object): as_set = relay.get('as_name', 'Unknown') if as_set not in as_sets: as_sets[as_set] = 0 - as_sets[as_set] += relay['consensus_weight'] + as_sets[as_set] += relay['consensus_weight_fraction'] - total_consensus_weight = self.get_total_consensus_weight() ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) ranking.reverse() for as_set, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) + print "%8.4f%% %s" % (weight * 100.0, as_set) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for as_set, weight in ranking[count:]: - other_consensus_weight += weight - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - selection_consensus_weight = 0 + other_consensus_weight_fraction += weight + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + selection_consensus_weight_fraction = 0 for as_set, weight in ranking: - selection_consensus_weight += weight - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + selection_consensus_weight_fraction += weight + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count) @@ -103,21 +93,20 @@ class RelayStats(object): as_sets = as_sets.split() relays = self.get_relays(flags, countries, as_sets) - total_consensus_weight = self.get_total_consensus_weight() - ranking = sorted(relays, key=operator.itemgetter('consensus_weight')) + ranking = sorted(relays, key=operator.itemgetter('consensus_weight_fraction')) ranking.reverse() - selection_consensus_weight = 0 + selection_consensus_weight_fraction = 0 for relay in ranking[:count]: - selection_consensus_weight += relay['consensus_weight'] - print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight'] * 100.0 / total_consensus_weight, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) + selection_consensus_weight_fraction += relay['consensus_weight_fraction'] + print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight_fraction'] * 100.0, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for relay in ranking[count:]: - other_consensus_weight += relay['consensus_weight'] - selection_consensus_weight += relay['consensus_weight'] - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + other_consensus_weight_fraction += relay['consensus_weight_fraction'] + selection_consensus_weight_fraction += relay['consensus_weight_fraction'] + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) OUTPUTS = { 'countries': 'output_countries',

1 0

[metrics-tasks/master] Unify code for grouped and non-grouped results (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit ccf3f1952318cdfce88b378419b4aa4ab9e6117c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 14:10:27 2012 +0200 Unify code for grouped and non-grouped results (#6329). --- task-6329/tor-relays-stats.py | 122 +++++++++++++++++++++++------------------ 1 files changed, 69 insertions(+), 53 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 0fbe3aa..038a22d 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -35,78 +35,94 @@ class RelayStats(object): relays.append(relay) return relays - def output_countries(self, count='10', flags=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags) - countries = {} + def group_relays(self, relays, by_country=False, by_as_number=False): + grouped_relays = {} for relay in relays: - country = relay.get('country', None) - if country not in countries: - countries[country] = 0 - countries[country] += relay['consensus_weight_fraction'] - - ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) - ranking.reverse() - for country, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0, country) - if len(ranking) > count: + if by_country and by_as_number: + key = (relay.get('country', None), relay.get('as_number', None)) + elif by_country: + key = relay.get('country', None) + elif by_as_number: + key = relay.get('as_number', None) + else: + key = relay.get('fingerprint') + if key not in grouped_relays: + grouped_relays[key] = [] + grouped_relays[key].append(relay) + return grouped_relays + + def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False): + formatted_groups = {} + for group in grouped_relays.viewvalues(): + group_consensus_weight_fraction = 0 + relays_in_group = 0 + for relay in group: + group_consensus_weight_fraction += relay.get('consensus_weight_fraction', 0) + nickname = relay['nickname'] + fingerprint = relay['fingerprint'] + exit = 'Exit' if 'Exit' in set(relay['flags']) else '' + guard = 'Guard' if 'Guard' in set(relay['flags']) else '' + country = relay.get('country', '') + as_number = relay.get('as_number', '') + as_name = relay.get('as_name', '') + relays_in_group += 1 + if by_country or by_as_number: + nickname = "*" + fingerprint = "* %5d relays" % relays_in_group + exit = "*" + guard = "*" + if by_country and not by_as_number: + as_number = "*" + as_name = "*" + if by_as_number and not by_country: + country = "*" + formatted_group = "%8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( + group_consensus_weight_fraction * 100.0, nickname, fingerprint, + exit, guard, country, as_number, as_name) + formatted_groups[formatted_group] = group_consensus_weight_fraction + sorted_groups = sorted(formatted_groups.iteritems(), key=operator.itemgetter(1)) + sorted_groups.reverse() + return sorted_groups + + def print_groups(self, sorted_groups, count=10): + print " CW Nickname Fingerprint Exit Guard CC AS_num AS_name" + for formatted_group, _ in sorted_groups[:count]: + print formatted_group + if len(sorted_groups) > count: other_consensus_weight_fraction = 0 - for as_set, weight in ranking[count:]: + for _, weight in sorted_groups[count:]: other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(sorted_groups) - count) selection_consensus_weight_fraction = 0 - for as_set, weight in ranking: + for _, weight in sorted_groups: selection_consensus_weight_fraction += weight if selection_consensus_weight_fraction < 0.999: print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + def output_countries(self, count='10', flags=''): + count = int(count) + flags = flags.split() + relays = self.get_relays(flags) + grouped_relays = self.group_relays(relays, by_country=True) + sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) + self.print_groups(sorted_groups, count) + def output_as_sets(self, count='10', flags='', countries=''): count = int(count) flags = flags.split() relays = self.get_relays(flags, countries) - as_sets = {} - for relay in relays: - as_set = relay.get('as_name', 'Unknown') - if as_set not in as_sets: - as_sets[as_set] = 0 - as_sets[as_set] += relay['consensus_weight_fraction'] - - ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) - ranking.reverse() - for as_set, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0, as_set) - if len(ranking) > count: - other_consensus_weight_fraction = 0 - for as_set, weight in ranking[count:]: - other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) - selection_consensus_weight_fraction = 0 - for as_set, weight in ranking: - selection_consensus_weight_fraction += weight - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + grouped_relays = self.group_relays(relays, by_as_number=True) + sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) + self.print_groups(sorted_groups, count) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count) flags = flags.split() as_sets = as_sets.split() relays = self.get_relays(flags, countries, as_sets) - - ranking = sorted(relays, key=operator.itemgetter('consensus_weight_fraction')) - ranking.reverse() - selection_consensus_weight_fraction = 0 - for relay in ranking[:count]: - selection_consensus_weight_fraction += relay['consensus_weight_fraction'] - print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight_fraction'] * 100.0, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) - if len(ranking) > count: - other_consensus_weight_fraction = 0 - for relay in ranking[count:]: - other_consensus_weight_fraction += relay['consensus_weight_fraction'] - selection_consensus_weight_fraction += relay['consensus_weight_fraction'] - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + grouped_relays = self.group_relays(relays) + sorted_groups = self.format_and_sort_groups(grouped_relays) + self.print_groups(sorted_groups, count) OUTPUTS = { 'countries': 'output_countries',

1 0

[metrics-tasks/master] Display all five path selection probabilities (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 4c9568f88b012aefa183b918555e50b4f725e591 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 14:46:54 2012 +0200 Display all five path selection probabilities (#6329). --- task-6329/tor-relays-stats.py | 67 ++++++++++++++++++++++++++++------------- 1 files changed, 46 insertions(+), 21 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 038a22d..4d10488 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -54,10 +54,15 @@ class RelayStats(object): def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False): formatted_groups = {} for group in grouped_relays.viewvalues(): - group_consensus_weight_fraction = 0 + group_weights = (0, 0, 0, 0, 0) relays_in_group = 0 for relay in group: - group_consensus_weight_fraction += relay.get('consensus_weight_fraction', 0) + weights = (relay.get('consensus_weight_fraction', 0), + relay.get('advertised_bandwidth_fraction', 0), + relay.get('guard_probability', 0), + relay.get('middle_probability', 0), + relay.get('exit_probability', 0)) + group_weights = tuple(sum(x) for x in zip(group_weights, weights)) nickname = relay['nickname'] fingerprint = relay['fingerprint'] exit = 'Exit' if 'Exit' in set(relay['flags']) else '' @@ -67,8 +72,8 @@ class RelayStats(object): as_name = relay.get('as_name', '') relays_in_group += 1 if by_country or by_as_number: - nickname = "*" - fingerprint = "* %5d relays" % relays_in_group + nickname = "(%d relays)" % relays_in_group + fingerprint = "*" exit = "*" guard = "*" if by_country and not by_as_number: @@ -76,28 +81,48 @@ class RelayStats(object): as_name = "*" if by_as_number and not by_country: country = "*" - formatted_group = "%8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( - group_consensus_weight_fraction * 100.0, nickname, fingerprint, + formatted_group = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( + group_weights[0] * 100.0, + group_weights[1] * 100.0, + group_weights[2] * 100.0, + group_weights[3] * 100.0, + group_weights[4] * 100.0, + nickname, fingerprint, exit, guard, country, as_number, as_name) - formatted_groups[formatted_group] = group_consensus_weight_fraction + formatted_groups[formatted_group] = group_weights sorted_groups = sorted(formatted_groups.iteritems(), key=operator.itemgetter(1)) sorted_groups.reverse() return sorted_groups - def print_groups(self, sorted_groups, count=10): - print " CW Nickname Fingerprint Exit Guard CC AS_num AS_name" - for formatted_group, _ in sorted_groups[:count]: + def print_groups(self, sorted_groups, count=10, by_country=False, by_as_number=False): + print " CW adv_bw P_guard P_middle P_exit Nickname Fingerprint Exit Guard CC AS_num AS_name" + + for formatted_group, weight in sorted_groups[:count]: print formatted_group if len(sorted_groups) > count: - other_consensus_weight_fraction = 0 - for _, weight in sorted_groups[count:]: - other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(sorted_groups) - count) - selection_consensus_weight_fraction = 0 - for _, weight in sorted_groups: - selection_consensus_weight_fraction += weight - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + if by_country and by_as_number: + type = "countries and ASes" + elif by_country: + type = "countries" + elif by_as_number: + type = "ASes" + else: + type = "relays" + other_weights = (0, 0, 0, 0, 0) + for _, weights in sorted_groups[count:]: + other_weights = tuple(sum(x) for x in zip(other_weights, weights)) + print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (%d other %s)" % ( + other_weights[0] * 100.0, other_weights[1] * 100.0, + other_weights[2] * 100.0, other_weights[3] * 100.0, + other_weights[4] * 100.0, len(sorted_groups) - count, type) + selection_weights = (0, 0, 0, 0, 0) + for _, weights in sorted_groups: + selection_weights = tuple(sum(x) for x in zip(selection_weights, weights)) + if selection_weights[0] < 0.999: + print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (total in selection)" % ( + selection_weights[0] * 100.0, selection_weights[1] * 100.0, + selection_weights[2] * 100.0, selection_weights[3] * 100.0, + selection_weights[4] * 100.0) def output_countries(self, count='10', flags=''): count = int(count) @@ -105,7 +130,7 @@ class RelayStats(object): relays = self.get_relays(flags) grouped_relays = self.group_relays(relays, by_country=True) sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) - self.print_groups(sorted_groups, count) + self.print_groups(sorted_groups, count, by_country=True) def output_as_sets(self, count='10', flags='', countries=''): count = int(count) @@ -113,7 +138,7 @@ class RelayStats(object): relays = self.get_relays(flags, countries) grouped_relays = self.group_relays(relays, by_as_number=True) sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) - self.print_groups(sorted_groups, count) + self.print_groups(sorted_groups, count, by_as_number=True) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count)

1 0

[metrics-tasks/master] Accept upper-case country codes, too (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 5ca7405e9f423fcb85a21557ae9ee82ef78bb028 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:35:53 2012 +0200 Accept upper-case country codes, too (#6329). --- task-6329/tor-relays-stats.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index e62782a..a2086f4 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -24,6 +24,8 @@ class RelayStats(object): def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False): relays = [] + if countries: + countries = [x.lower() for x in countries] for relay in self.data['relays']: if not relay['running']: continue

1 0

[metrics-tasks/master] Use optparse (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 1178fa61567a24c98c385b75aaf0c9d753959e97 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:32:13 2012 +0200 Use optparse (#6329). --- task-6329/tor-relays-stats.py | 113 +++++++++++++++++------------------------ 1 files changed, 46 insertions(+), 67 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 4d10488..e62782a 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -10,6 +10,7 @@ import json import operator import sys import os.path +from optparse import OptionParser, OptionGroup class RelayStats(object): def __init__(self): @@ -21,17 +22,19 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_relays(self, flags=[], countries='', as_sets=[]): + def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False): relays = [] for relay in self.data['relays']: if not relay['running']: continue - if set(flags) & set(relay['flags']) != set(flags): - continue if countries and not relay.get('country', ' ') in countries: continue if as_sets and not relay.get('as_number', ' ') in as_sets: continue + if exits_only and not relay.get('exit_probability', -1) > 0.0: + continue + if guards_only and not relay.get('guard_probability', -1) > 0.0: + continue relays.append(relay) return relays @@ -124,70 +127,46 @@ class RelayStats(object): selection_weights[2] * 100.0, selection_weights[3] * 100.0, selection_weights[4] * 100.0) - def output_countries(self, count='10', flags=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags) - grouped_relays = self.group_relays(relays, by_country=True) - sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) - self.print_groups(sorted_groups, count, by_country=True) - - def output_as_sets(self, count='10', flags='', countries=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags, countries) - grouped_relays = self.group_relays(relays, by_as_number=True) - sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) - self.print_groups(sorted_groups, count, by_as_number=True) - - def output_relays(self, count='10', flags='', countries='', as_sets=''): - count = int(count) - flags = flags.split() - as_sets = as_sets.split() - relays = self.get_relays(flags, countries, as_sets) - grouped_relays = self.group_relays(relays) - sorted_groups = self.format_and_sort_groups(grouped_relays) - self.print_groups(sorted_groups, count) - -OUTPUTS = { - 'countries': 'output_countries', - 'as-sets': 'output_as_sets', - 'relays': 'output_relays', -} - -def usage(): - print >>sys.stderr, """Usage: %(progname)s <output> [args ...] - -Where <output> is one of: - - countries [COUNT] [FLAGS] - relative percentage of the consensus in each countries - - as-sets [COUNT] [FLAGS] [COUNTRIES] - relative percentage of the consensus in each AS sets - - relays [COUNT] [FLAGS] [COUNTRIES] [AS_SETS] - list relays ranked by their place in the whole consensus - -Examples: - - - To get the top five exit nodes in France: - %(progname)s top 5 Exit fr - - To get weights of the top ten AS of all relays in Germany: - %(progname)s as-sets 10 Running de - -This script expect to have a file called 'details.json' in the -current directory. In order to retrieve the needed data, one -can issue the following command: - - curl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true' -""" % { 'progname': sys.argv[0] } - sys.exit(1) - if '__main__' == __name__: + parser = OptionParser() + group = OptionGroup(parser, "Filtering options") + group.add_option("-a", "--as", dest="ases", action="append", + help="select only relays from autonomous system number AS", + metavar="AS") + group.add_option("-c", "--country", action="append", + help="select only relays from country with code CC", metavar="CC") + group.add_option("-e", "--exits-only", action="store_true", + help="select only relays suitable for exit position") + group.add_option("-g", "--guards-only", action="store_true", + help="select only relays suitable for guard position") + parser.add_option_group(group) + group = OptionGroup(parser, "Grouping options") + group.add_option("-A", "--by-as", action="store_true", default=False, + help="group relays by AS") + group.add_option("-C", "--by-country", action="store_true", default=False, + help="group relays by country") + parser.add_option_group(group) + group = OptionGroup(parser, "Display options") + group.add_option("-t", "--top", type="int", default=10, metavar="NUM", + help="display only the top results (default: %default)") + parser.add_option_group(group) + (options, args) = parser.parse_args() + if len(args) > 0: + parser.error("Did not understand positional argument(s), use options instead.") if not os.path.exists('details.json'): - usage() - if len(sys.argv) == 1: - usage() - func = OUTPUTS.get(sys.argv[1], None) - if not func: - usage() + parser.error("Did not find details.json. Please download this file using the following command:\ncurl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true'") + stats = RelayStats() - getattr(stats, func)(*sys.argv[2:]) + relays = stats.get_relays(countries=options.country, + as_sets=options.ases, + exits_only=options.exits_only, + guards_only=options.guards_only) + grouped_relays = stats.group_relays(relays, + by_country=options.by_country, + by_as_number=options.by_as) + sorted_groups = stats.format_and_sort_groups(grouped_relays, + by_country=options.by_country, + by_as_number=options.by_as) + stats.print_groups(sorted_groups, options.top, + by_country=options.by_country, + by_as_number=options.by_as)

1 0

[metrics-tasks/master] Hide totals if there's only 1 list element (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 694d75f8a436f7dbca1e6bb53f5a1b48a139c20f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:37:08 2012 +0200 Hide totals if there's only 1 list element (#6329). --- task-6329/tor-relays-stats.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index a2086f4..4fedf92 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -123,7 +123,7 @@ class RelayStats(object): selection_weights = (0, 0, 0, 0, 0) for _, weights in sorted_groups: selection_weights = tuple(sum(x) for x in zip(selection_weights, weights)) - if selection_weights[0] < 0.999: + if len(sorted_groups) > 1 and selection_weights[0] < 0.999: print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (total in selection)" % ( selection_weights[0] * 100.0, selection_weights[1] * 100.0, selection_weights[2] * 100.0, selection_weights[3] * 100.0,

1 0