- tor-commits - lists.torproject.org

[ooni-probe/master] Get b0wser to start doing something useful
by art＠torproject.org 20 Jul '12

20 Jul '12

commit 4dbb7426ec0147e6efa946e287f5e6a7f682252f Author: Arturo Filastò <hellais(a)torproject.org> Date: Tue Jul 17 22:10:24 2012 +0200 Get b0wser to start doing something useful --- ooni/plugins/b0wser.py | 23 +++++++++++++++++++---- ooni/plugoo/tests.py | 6 ++++-- ooni/protocols/b0wser.py | 28 +++++++++++++++++++++++++--- oonib/b0wser.py | 6 ++++-- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/ooni/plugins/b0wser.py b/ooni/plugins/b0wser.py index 5d6687b..3121378 100644 --- a/ooni/plugins/b0wser.py +++ b/ooni/plugins/b0wser.py @@ -62,14 +62,23 @@ class b0wserTest(OONITest): options = b0wserArgs blocking = False + local_options = None + + steps = None def initialize(self): + if not self.local_options: + return #pass self.factory = B0wserClientFactory() + self.steps = b0wser.get_b0wser_dictionary_from_pcap(self.local_options['pcap']) + + def control(self, exp_res, args): + mutation = self.factory.mutator.get_mutation(0) + return {'mutation_number': args['mutation'], 'value': mutation} def experiment(self, args): - steps = b0wser.get_b0wser_dictionary_from_pcap(self.local_options['pcap']) - print steps - self.factory.steps = steps + log.msg("Doing mutation %s" % args['mutation']) + self.factory.steps = self.steps host = self.local_options['host'] port = int(self.local_options['port']) log.msg("Connecting to %s:%s" % (host, port)) @@ -78,7 +87,13 @@ class b0wserTest(OONITest): #return endpoint.connect(B0wserClientFactory) def load_assets(self): - return {} + if not self.steps: + print "No asset!" + return {} + mutations = 0 + for x in self.steps: + mutations += len(x['data']) + return {'mutation': range(mutations)} # We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index a99a144..9b6ea26 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -24,6 +24,7 @@ class OONITest(object): """ # By default we set this to False, meaning that we don't block blocking = False + reactor = None def __init__(self, local_options, global_options, report, ooninet=None, reactor=None): @@ -31,13 +32,14 @@ class OONITest(object): self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options - self.assets = self.load_assets() self.report = report #self.ooninet = ooninet self.reactor = reactor - self.initialize() self.result = {} + self.initialize() + self.assets = self.load_assets() + def initialize(self): """ Override this method if you are interested in having some extra diff --git a/ooni/protocols/b0wser.py b/ooni/protocols/b0wser.py index 6a01d96..5d55c94 100644 --- a/ooni/protocols/b0wser.py +++ b/ooni/protocols/b0wser.py @@ -101,7 +101,9 @@ class Mutator: Returns a dict containg the packet index and the step number. """ - return {'idx': self.idx, 'step': self.step} + print "[Mutator.state()] Giving out my internal state." + current_state = {'idx': self.idx, 'step': self.step} + return current_state def next_mutation(self): """ @@ -121,6 +123,8 @@ class Mutator: """ if (self.step + 1) > len(self.steps): # Hack to stop once we have gone through all the steps + print "[Mutator.next_mutation()] I believe I have gone over all steps" + print " Stopping!" self.waiting = True return False @@ -130,20 +134,24 @@ class Mutator: current_data = self.steps[current_step]['data'] try: data_to_receive = len(self.steps[current_step +1 ]['data']) + print "[Mutator.next_mutation()] Managed to receive some data." except: - print "No more data to receive" + print "[Mutator.next_mutation()] No more data to receive." if self.waiting and self.waiting_step == data_to_receive: + print "[Mutator.next_mutation()] I am no longer waiting" log.debug("I am no longer waiting.") self.waiting = False self.waiting_step = 0 self.idx = 0 elif self.waiting: + print "[Mutator.next_mutation()] Waiting some more." log.debug("Waiting some more.") self.waiting_step += 1 elif current_idx >= len(current_data): + print "[Mutator.next_mutation()] Entering waiting mode." log.debug("Entering waiting mode.") self.step += 1 self.idx = 0 @@ -164,7 +172,7 @@ class Mutator: returns the mutated packet for the specified step. """ if step != self.step or self.waiting: - log.debug("I am not going to do anything :)") + log.debug("[Mutator.get_mutation()] I am not going to do anything :)") return self.steps[step]['data'] data = self.steps[step]['data'] @@ -193,12 +201,19 @@ class B0wserProtocol(protocol.Protocol): This is called once I have completed one step of the protocol and need to proceed to the next step. """ + if not self.mutator: + print "[B0wserProtocol.next_state] No mutator. There is no point to stay on this earth." + self.transport.loseConnection() + return if self.role is self.steps[self.state]['sender']: + print "[B0wserProtocol.next_state] I am a sender" data = self.mutator.get_mutation(self.state) self.transport.write(data) self.to_receive_data = 0 else: + print "[B0wserProtocol.next_state] I am a receiver" self.to_receive_data = len(self.steps[self.state]['data']) + self.state += 1 self.received_data = 0 @@ -209,6 +224,10 @@ class B0wserProtocol(protocol.Protocol): @param data: the data that has been sent by the client. """ + if not self.mutator: + print "I don't have a mutator. My life means nothing." + self.transport.loseConnection() + return if len(self.steps) <= self.state: print "I have reached the end of the state machine" print "Censorship fingerprint bruteforced!" @@ -246,6 +265,9 @@ class B0wserProtocol(protocol.Protocol): The connection was closed. This may be because of a legittimate reason or it may be because of a censorship event. """ + if not self.mutator: + print "Terminated because of little interest in life." + return report = {'reason': reason, 'proto_state': self.state, 'trigger': None, 'mutator_state': self.mutator.state()} diff --git a/oonib/b0wser.py b/oonib/b0wser.py index 0bb89fb..4500075 100644 --- a/oonib/b0wser.py +++ b/oonib/b0wser.py @@ -25,7 +25,9 @@ class B0wserServer(protocol.ServerFactory): print "Moving on to next mutation" if not self.mutations[addr.host].next_mutation(): self.mutations.pop(addr.host) - else: - p.mutator = self.mutations[addr.host] + try: + p.mutator = self.mutations[addr.host] + except: + pass return p

1 0

[metrics-tasks/master] Parse consensus_weight_fraction from details.json (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 415d92c63c90ac6ea3d47d61945c75068fe8286a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 12:44:58 2012 +0200 Parse consensus_weight_fraction from details.json (#6329). --- task-6329/tor-relays-stats.py | 67 +++++++++++++++++------------------------ 1 files changed, 28 insertions(+), 39 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 05d9373..0fbe3aa 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -21,14 +21,6 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_total_consensus_weight(self, relays=None): - if relays is None: - relays = self.get_relays() - total_consensus_weight = 0 - for relay in relays: - total_consensus_weight += relay['consensus_weight'] - return total_consensus_weight - def get_relays(self, flags=[], countries='', as_sets=[]): relays = [] for relay in self.data['relays']: @@ -52,23 +44,22 @@ class RelayStats(object): country = relay.get('country', None) if country not in countries: countries[country] = 0 - countries[country] += relay['consensus_weight'] + countries[country] += relay['consensus_weight_fraction'] ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) ranking.reverse() - total_consensus_weight = self.get_total_consensus_weight() for country, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0 / total_consensus_weight, country) + print "%8.4f%% %s" % (weight * 100.0, country) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for as_set, weight in ranking[count:]: - other_consensus_weight += weight - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - selection_consensus_weight = 0 + other_consensus_weight_fraction += weight + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + selection_consensus_weight_fraction = 0 for as_set, weight in ranking: - selection_consensus_weight += weight - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + selection_consensus_weight_fraction += weight + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) def output_as_sets(self, count='10', flags='', countries=''): count = int(count) @@ -79,23 +70,22 @@ class RelayStats(object): as_set = relay.get('as_name', 'Unknown') if as_set not in as_sets: as_sets[as_set] = 0 - as_sets[as_set] += relay['consensus_weight'] + as_sets[as_set] += relay['consensus_weight_fraction'] - total_consensus_weight = self.get_total_consensus_weight() ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) ranking.reverse() for as_set, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) + print "%8.4f%% %s" % (weight * 100.0, as_set) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for as_set, weight in ranking[count:]: - other_consensus_weight += weight - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - selection_consensus_weight = 0 + other_consensus_weight_fraction += weight + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + selection_consensus_weight_fraction = 0 for as_set, weight in ranking: - selection_consensus_weight += weight - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + selection_consensus_weight_fraction += weight + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count) @@ -103,21 +93,20 @@ class RelayStats(object): as_sets = as_sets.split() relays = self.get_relays(flags, countries, as_sets) - total_consensus_weight = self.get_total_consensus_weight() - ranking = sorted(relays, key=operator.itemgetter('consensus_weight')) + ranking = sorted(relays, key=operator.itemgetter('consensus_weight_fraction')) ranking.reverse() - selection_consensus_weight = 0 + selection_consensus_weight_fraction = 0 for relay in ranking[:count]: - selection_consensus_weight += relay['consensus_weight'] - print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight'] * 100.0 / total_consensus_weight, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) + selection_consensus_weight_fraction += relay['consensus_weight_fraction'] + print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight_fraction'] * 100.0, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) if len(ranking) > count: - other_consensus_weight = 0 + other_consensus_weight_fraction = 0 for relay in ranking[count:]: - other_consensus_weight += relay['consensus_weight'] - selection_consensus_weight += relay['consensus_weight'] - print "%8.4f%% (%d others)" % (other_consensus_weight * 100.0 / total_consensus_weight, len(ranking) - count) - if selection_consensus_weight < total_consensus_weight: - print "%8.4f%% (total in selection)" % (selection_consensus_weight * 100.0 / total_consensus_weight) + other_consensus_weight_fraction += relay['consensus_weight_fraction'] + selection_consensus_weight_fraction += relay['consensus_weight_fraction'] + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + if selection_consensus_weight_fraction < 0.999: + print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) OUTPUTS = { 'countries': 'output_countries',

1 0

[metrics-tasks/master] Unify code for grouped and non-grouped results (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit ccf3f1952318cdfce88b378419b4aa4ab9e6117c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 14:10:27 2012 +0200 Unify code for grouped and non-grouped results (#6329). --- task-6329/tor-relays-stats.py | 122 +++++++++++++++++++++++------------------ 1 files changed, 69 insertions(+), 53 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 0fbe3aa..038a22d 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -35,78 +35,94 @@ class RelayStats(object): relays.append(relay) return relays - def output_countries(self, count='10', flags=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags) - countries = {} + def group_relays(self, relays, by_country=False, by_as_number=False): + grouped_relays = {} for relay in relays: - country = relay.get('country', None) - if country not in countries: - countries[country] = 0 - countries[country] += relay['consensus_weight_fraction'] - - ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) - ranking.reverse() - for country, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0, country) - if len(ranking) > count: + if by_country and by_as_number: + key = (relay.get('country', None), relay.get('as_number', None)) + elif by_country: + key = relay.get('country', None) + elif by_as_number: + key = relay.get('as_number', None) + else: + key = relay.get('fingerprint') + if key not in grouped_relays: + grouped_relays[key] = [] + grouped_relays[key].append(relay) + return grouped_relays + + def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False): + formatted_groups = {} + for group in grouped_relays.viewvalues(): + group_consensus_weight_fraction = 0 + relays_in_group = 0 + for relay in group: + group_consensus_weight_fraction += relay.get('consensus_weight_fraction', 0) + nickname = relay['nickname'] + fingerprint = relay['fingerprint'] + exit = 'Exit' if 'Exit' in set(relay['flags']) else '' + guard = 'Guard' if 'Guard' in set(relay['flags']) else '' + country = relay.get('country', '') + as_number = relay.get('as_number', '') + as_name = relay.get('as_name', '') + relays_in_group += 1 + if by_country or by_as_number: + nickname = "*" + fingerprint = "* %5d relays" % relays_in_group + exit = "*" + guard = "*" + if by_country and not by_as_number: + as_number = "*" + as_name = "*" + if by_as_number and not by_country: + country = "*" + formatted_group = "%8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( + group_consensus_weight_fraction * 100.0, nickname, fingerprint, + exit, guard, country, as_number, as_name) + formatted_groups[formatted_group] = group_consensus_weight_fraction + sorted_groups = sorted(formatted_groups.iteritems(), key=operator.itemgetter(1)) + sorted_groups.reverse() + return sorted_groups + + def print_groups(self, sorted_groups, count=10): + print " CW Nickname Fingerprint Exit Guard CC AS_num AS_name" + for formatted_group, _ in sorted_groups[:count]: + print formatted_group + if len(sorted_groups) > count: other_consensus_weight_fraction = 0 - for as_set, weight in ranking[count:]: + for _, weight in sorted_groups[count:]: other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) + print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(sorted_groups) - count) selection_consensus_weight_fraction = 0 - for as_set, weight in ranking: + for _, weight in sorted_groups: selection_consensus_weight_fraction += weight if selection_consensus_weight_fraction < 0.999: print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + def output_countries(self, count='10', flags=''): + count = int(count) + flags = flags.split() + relays = self.get_relays(flags) + grouped_relays = self.group_relays(relays, by_country=True) + sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) + self.print_groups(sorted_groups, count) + def output_as_sets(self, count='10', flags='', countries=''): count = int(count) flags = flags.split() relays = self.get_relays(flags, countries) - as_sets = {} - for relay in relays: - as_set = relay.get('as_name', 'Unknown') - if as_set not in as_sets: - as_sets[as_set] = 0 - as_sets[as_set] += relay['consensus_weight_fraction'] - - ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) - ranking.reverse() - for as_set, weight in ranking[:count]: - print "%8.4f%% %s" % (weight * 100.0, as_set) - if len(ranking) > count: - other_consensus_weight_fraction = 0 - for as_set, weight in ranking[count:]: - other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) - selection_consensus_weight_fraction = 0 - for as_set, weight in ranking: - selection_consensus_weight_fraction += weight - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + grouped_relays = self.group_relays(relays, by_as_number=True) + sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) + self.print_groups(sorted_groups, count) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count) flags = flags.split() as_sets = as_sets.split() relays = self.get_relays(flags, countries, as_sets) - - ranking = sorted(relays, key=operator.itemgetter('consensus_weight_fraction')) - ranking.reverse() - selection_consensus_weight_fraction = 0 - for relay in ranking[:count]: - selection_consensus_weight_fraction += relay['consensus_weight_fraction'] - print "%8.4f%% %-19s %-2s %-4s %-5s %s %-9s %s" % (relay['consensus_weight_fraction'] * 100.0, relay['nickname'], relay['fingerprint'], 'Exit' if 'Exit' in set(relay['flags']) else '', 'Guard' if 'Guard' in set(relay['flags']) else '', relay.get('country', ' '), relay.get('as_number', ''), relay.get('as_name', '')) - if len(ranking) > count: - other_consensus_weight_fraction = 0 - for relay in ranking[count:]: - other_consensus_weight_fraction += relay['consensus_weight_fraction'] - selection_consensus_weight_fraction += relay['consensus_weight_fraction'] - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(ranking) - count) - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + grouped_relays = self.group_relays(relays) + sorted_groups = self.format_and_sort_groups(grouped_relays) + self.print_groups(sorted_groups, count) OUTPUTS = { 'countries': 'output_countries',

1 0

[metrics-tasks/master] Display all five path selection probabilities (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 4c9568f88b012aefa183b918555e50b4f725e591 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 14:46:54 2012 +0200 Display all five path selection probabilities (#6329). --- task-6329/tor-relays-stats.py | 67 ++++++++++++++++++++++++++++------------- 1 files changed, 46 insertions(+), 21 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 038a22d..4d10488 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -54,10 +54,15 @@ class RelayStats(object): def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False): formatted_groups = {} for group in grouped_relays.viewvalues(): - group_consensus_weight_fraction = 0 + group_weights = (0, 0, 0, 0, 0) relays_in_group = 0 for relay in group: - group_consensus_weight_fraction += relay.get('consensus_weight_fraction', 0) + weights = (relay.get('consensus_weight_fraction', 0), + relay.get('advertised_bandwidth_fraction', 0), + relay.get('guard_probability', 0), + relay.get('middle_probability', 0), + relay.get('exit_probability', 0)) + group_weights = tuple(sum(x) for x in zip(group_weights, weights)) nickname = relay['nickname'] fingerprint = relay['fingerprint'] exit = 'Exit' if 'Exit' in set(relay['flags']) else '' @@ -67,8 +72,8 @@ class RelayStats(object): as_name = relay.get('as_name', '') relays_in_group += 1 if by_country or by_as_number: - nickname = "*" - fingerprint = "* %5d relays" % relays_in_group + nickname = "(%d relays)" % relays_in_group + fingerprint = "*" exit = "*" guard = "*" if by_country and not by_as_number: @@ -76,28 +81,48 @@ class RelayStats(object): as_name = "*" if by_as_number and not by_country: country = "*" - formatted_group = "%8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( - group_consensus_weight_fraction * 100.0, nickname, fingerprint, + formatted_group = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( + group_weights[0] * 100.0, + group_weights[1] * 100.0, + group_weights[2] * 100.0, + group_weights[3] * 100.0, + group_weights[4] * 100.0, + nickname, fingerprint, exit, guard, country, as_number, as_name) - formatted_groups[formatted_group] = group_consensus_weight_fraction + formatted_groups[formatted_group] = group_weights sorted_groups = sorted(formatted_groups.iteritems(), key=operator.itemgetter(1)) sorted_groups.reverse() return sorted_groups - def print_groups(self, sorted_groups, count=10): - print " CW Nickname Fingerprint Exit Guard CC AS_num AS_name" - for formatted_group, _ in sorted_groups[:count]: + def print_groups(self, sorted_groups, count=10, by_country=False, by_as_number=False): + print " CW adv_bw P_guard P_middle P_exit Nickname Fingerprint Exit Guard CC AS_num AS_name" + + for formatted_group, weight in sorted_groups[:count]: print formatted_group if len(sorted_groups) > count: - other_consensus_weight_fraction = 0 - for _, weight in sorted_groups[count:]: - other_consensus_weight_fraction += weight - print "%8.4f%% (%d others)" % (other_consensus_weight_fraction * 100.0, len(sorted_groups) - count) - selection_consensus_weight_fraction = 0 - for _, weight in sorted_groups: - selection_consensus_weight_fraction += weight - if selection_consensus_weight_fraction < 0.999: - print "%8.4f%% (total in selection)" % (selection_consensus_weight_fraction * 100.0) + if by_country and by_as_number: + type = "countries and ASes" + elif by_country: + type = "countries" + elif by_as_number: + type = "ASes" + else: + type = "relays" + other_weights = (0, 0, 0, 0, 0) + for _, weights in sorted_groups[count:]: + other_weights = tuple(sum(x) for x in zip(other_weights, weights)) + print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (%d other %s)" % ( + other_weights[0] * 100.0, other_weights[1] * 100.0, + other_weights[2] * 100.0, other_weights[3] * 100.0, + other_weights[4] * 100.0, len(sorted_groups) - count, type) + selection_weights = (0, 0, 0, 0, 0) + for _, weights in sorted_groups: + selection_weights = tuple(sum(x) for x in zip(selection_weights, weights)) + if selection_weights[0] < 0.999: + print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (total in selection)" % ( + selection_weights[0] * 100.0, selection_weights[1] * 100.0, + selection_weights[2] * 100.0, selection_weights[3] * 100.0, + selection_weights[4] * 100.0) def output_countries(self, count='10', flags=''): count = int(count) @@ -105,7 +130,7 @@ class RelayStats(object): relays = self.get_relays(flags) grouped_relays = self.group_relays(relays, by_country=True) sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) - self.print_groups(sorted_groups, count) + self.print_groups(sorted_groups, count, by_country=True) def output_as_sets(self, count='10', flags='', countries=''): count = int(count) @@ -113,7 +138,7 @@ class RelayStats(object): relays = self.get_relays(flags, countries) grouped_relays = self.group_relays(relays, by_as_number=True) sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) - self.print_groups(sorted_groups, count) + self.print_groups(sorted_groups, count, by_as_number=True) def output_relays(self, count='10', flags='', countries='', as_sets=''): count = int(count)

1 0

[metrics-tasks/master] Accept upper-case country codes, too (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 5ca7405e9f423fcb85a21557ae9ee82ef78bb028 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:35:53 2012 +0200 Accept upper-case country codes, too (#6329). --- task-6329/tor-relays-stats.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index e62782a..a2086f4 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -24,6 +24,8 @@ class RelayStats(object): def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False): relays = [] + if countries: + countries = [x.lower() for x in countries] for relay in self.data['relays']: if not relay['running']: continue

1 0

[metrics-tasks/master] Use optparse (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 1178fa61567a24c98c385b75aaf0c9d753959e97 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:32:13 2012 +0200 Use optparse (#6329). --- task-6329/tor-relays-stats.py | 113 +++++++++++++++++------------------------ 1 files changed, 46 insertions(+), 67 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 4d10488..e62782a 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -10,6 +10,7 @@ import json import operator import sys import os.path +from optparse import OptionParser, OptionGroup class RelayStats(object): def __init__(self): @@ -21,17 +22,19 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_relays(self, flags=[], countries='', as_sets=[]): + def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False): relays = [] for relay in self.data['relays']: if not relay['running']: continue - if set(flags) & set(relay['flags']) != set(flags): - continue if countries and not relay.get('country', ' ') in countries: continue if as_sets and not relay.get('as_number', ' ') in as_sets: continue + if exits_only and not relay.get('exit_probability', -1) > 0.0: + continue + if guards_only and not relay.get('guard_probability', -1) > 0.0: + continue relays.append(relay) return relays @@ -124,70 +127,46 @@ class RelayStats(object): selection_weights[2] * 100.0, selection_weights[3] * 100.0, selection_weights[4] * 100.0) - def output_countries(self, count='10', flags=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags) - grouped_relays = self.group_relays(relays, by_country=True) - sorted_groups = self.format_and_sort_groups(grouped_relays, by_country=True) - self.print_groups(sorted_groups, count, by_country=True) - - def output_as_sets(self, count='10', flags='', countries=''): - count = int(count) - flags = flags.split() - relays = self.get_relays(flags, countries) - grouped_relays = self.group_relays(relays, by_as_number=True) - sorted_groups = self.format_and_sort_groups(grouped_relays, by_as_number=True) - self.print_groups(sorted_groups, count, by_as_number=True) - - def output_relays(self, count='10', flags='', countries='', as_sets=''): - count = int(count) - flags = flags.split() - as_sets = as_sets.split() - relays = self.get_relays(flags, countries, as_sets) - grouped_relays = self.group_relays(relays) - sorted_groups = self.format_and_sort_groups(grouped_relays) - self.print_groups(sorted_groups, count) - -OUTPUTS = { - 'countries': 'output_countries', - 'as-sets': 'output_as_sets', - 'relays': 'output_relays', -} - -def usage(): - print >>sys.stderr, """Usage: %(progname)s <output> [args ...] - -Where <output> is one of: - - countries [COUNT] [FLAGS] - relative percentage of the consensus in each countries - - as-sets [COUNT] [FLAGS] [COUNTRIES] - relative percentage of the consensus in each AS sets - - relays [COUNT] [FLAGS] [COUNTRIES] [AS_SETS] - list relays ranked by their place in the whole consensus - -Examples: - - - To get the top five exit nodes in France: - %(progname)s top 5 Exit fr - - To get weights of the top ten AS of all relays in Germany: - %(progname)s as-sets 10 Running de - -This script expect to have a file called 'details.json' in the -current directory. In order to retrieve the needed data, one -can issue the following command: - - curl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true' -""" % { 'progname': sys.argv[0] } - sys.exit(1) - if '__main__' == __name__: + parser = OptionParser() + group = OptionGroup(parser, "Filtering options") + group.add_option("-a", "--as", dest="ases", action="append", + help="select only relays from autonomous system number AS", + metavar="AS") + group.add_option("-c", "--country", action="append", + help="select only relays from country with code CC", metavar="CC") + group.add_option("-e", "--exits-only", action="store_true", + help="select only relays suitable for exit position") + group.add_option("-g", "--guards-only", action="store_true", + help="select only relays suitable for guard position") + parser.add_option_group(group) + group = OptionGroup(parser, "Grouping options") + group.add_option("-A", "--by-as", action="store_true", default=False, + help="group relays by AS") + group.add_option("-C", "--by-country", action="store_true", default=False, + help="group relays by country") + parser.add_option_group(group) + group = OptionGroup(parser, "Display options") + group.add_option("-t", "--top", type="int", default=10, metavar="NUM", + help="display only the top results (default: %default)") + parser.add_option_group(group) + (options, args) = parser.parse_args() + if len(args) > 0: + parser.error("Did not understand positional argument(s), use options instead.") if not os.path.exists('details.json'): - usage() - if len(sys.argv) == 1: - usage() - func = OUTPUTS.get(sys.argv[1], None) - if not func: - usage() + parser.error("Did not find details.json. Please download this file using the following command:\ncurl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true'") + stats = RelayStats() - getattr(stats, func)(*sys.argv[2:]) + relays = stats.get_relays(countries=options.country, + as_sets=options.ases, + exits_only=options.exits_only, + guards_only=options.guards_only) + grouped_relays = stats.group_relays(relays, + by_country=options.by_country, + by_as_number=options.by_as) + sorted_groups = stats.format_and_sort_groups(grouped_relays, + by_country=options.by_country, + by_as_number=options.by_as) + stats.print_groups(sorted_groups, options.top, + by_country=options.by_country, + by_as_number=options.by_as)

1 0

[metrics-tasks/master] Hide totals if there's only 1 list element (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 694d75f8a436f7dbca1e6bb53f5a1b48a139c20f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:37:08 2012 +0200 Hide totals if there's only 1 list element (#6329). --- task-6329/tor-relays-stats.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index a2086f4..4fedf92 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -123,7 +123,7 @@ class RelayStats(object): selection_weights = (0, 0, 0, 0, 0) for _, weights in sorted_groups: selection_weights = tuple(sum(x) for x in zip(selection_weights, weights)) - if selection_weights[0] < 0.999: + if len(sorted_groups) > 1 and selection_weights[0] < 0.999: print "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% (total in selection)" % ( selection_weights[0] * 100.0, selection_weights[1] * 100.0, selection_weights[2] * 100.0, selection_weights[3] * 100.0,

1 0

[metrics-tasks/master] Add --download option (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 00693abce05d177a388aa7adef6a2d371579e008 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:45:38 2012 +0200 Add --download option (#6329). --- task-6329/tor-relays-stats.py | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 4fedf92..fc9b14c 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -11,6 +11,7 @@ import operator import sys import os.path from optparse import OptionParser, OptionGroup +import urllib class RelayStats(object): def __init__(self): @@ -129,8 +130,17 @@ class RelayStats(object): selection_weights[2] * 100.0, selection_weights[3] * 100.0, selection_weights[4] * 100.0) +def download_details_file(): + url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay&running=true') + details_file = open("details.json", 'w') + details_file.write(url.read()) + url.close() + details_file.close() + if '__main__' == __name__: parser = OptionParser() + parser.add_option("-d", "--download", action="store_true", + help="download details.json from Onionoo service") group = OptionGroup(parser, "Filtering options") group.add_option("-a", "--as", dest="ases", action="append", help="select only relays from autonomous system number AS", @@ -155,8 +165,12 @@ if '__main__' == __name__: (options, args) = parser.parse_args() if len(args) > 0: parser.error("Did not understand positional argument(s), use options instead.") + + if options.download: + download_details_file() + if not os.path.exists('details.json'): - parser.error("Did not find details.json. Please download this file using the following command:\ncurl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true'") + parser.error("Did not find details.json. Re-run with --download.") stats = RelayStats() relays = stats.get_relays(countries=options.country,

1 0

[metrics-tasks/master] Accept "AS"-prefixed ASes or just AS numbers (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit a0a2e0e0900832f78ce14d13fd5fa41417562717 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:48:20 2012 +0200 Accept "AS"-prefixed ASes or just AS numbers (#6329). --- task-6329/tor-relays-stats.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index fc9b14c..09bf98a 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -27,6 +27,8 @@ class RelayStats(object): relays = [] if countries: countries = [x.lower() for x in countries] + if as_sets: + as_sets = [x if not x.isdigit() else "AS" + x for x in as_sets] for relay in self.data['relays']: if not relay['running']: continue

1 0

[metrics-tasks/master] Update README (#6329).
by karsten＠torproject.org 20 Jul '12

20 Jul '12

commit 0c7660b62ea92ef8ee4312ba5317dbf21d8a6a5a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 20 15:57:44 2012 +0200 Update README (#6329). --- task-6329/README | 23 +++++++++-------------- 1 files changed, 9 insertions(+), 14 deletions(-) diff --git a/task-6329/README b/task-6329/README index 0c6c501..bbb36ae 100644 --- a/task-6329/README +++ b/task-6329/README @@ -7,29 +7,23 @@ based on the current consensus. Usage ----- -Usage: tor-relays-stats.py <output> [args ...] +Run the script with --help to learn about its usage: -Where <output> is one of: - - countries [COUNT] [FLAGS] - relative percentage of the consensus in each countries - - as-sets [COUNT] [FLAGS] [COUNTRIES] - relative percentage of the consensus in each AS sets - - relays [COUNT] [FLAGS] [COUNTRIES] [AS_SETS] - list relays ranked by their place in the whole consensus + ./tor-relays-stats.py -h Examples: - To get the top five exit nodes in France: - tor-relays-stats.py top 5 Exit fr + ./tor-relays-stats.py --top=5 --exits-only --country=fr - To get weights of the top ten AS of all relays in Germany: - tor-relays-stats.py as-sets 10 Running de + ./tor-relays-stats.py --top=10 --by-as --country=de Retrieving the data ------------------- This script expect to have a file called 'details.json' in the -current directory. In order to retrieve the needed data, one -can issue the following command: +current directory. Run with the --download option or issue the following +command to obtain that file: curl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true' @@ -38,10 +32,11 @@ Dependencies Written and tested on Python 2.6.6. -Author ------- +Authors +------- * delber <delber(a)riseup.net> + * Karsten Loesing <karsten(a)torproject.org> Licence -------

1 0