- tor-commits - lists.torproject.org

[metrics-tasks/master] task-6329: Remove extra whitespaces
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit a339cf586cb320223d996666cc27767affb82cff Author: delber <delber(a)riseup.net> Date: Sat Jul 14 13:07:13 2012 +0000 task-6329: Remove extra whitespaces --- task-6329/tor-relays-stats.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 0aa4854..822317a 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -86,7 +86,6 @@ class RelayStats(object): top = sorted(relays, key=operator.itemgetter('consensus_weight')) total_consensus_weight = self.get_total_consensus_weight() for relay in top[-count:]: - print "%3.4f%% %-20s %s" % (relay['consensus_weight'] * 100.0 / total_consensus_weight, relay['nickname'], relay['fingerprint']) OUTPUTS = {

1 0

[metrics-tasks/master] task-6329: Ignore 'details.json'
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 1ed2512c20b3c4227999a5c8d07cd7e7de368bae Author: delber <delber(a)riseup.net> Date: Sat Jul 14 12:48:21 2012 +0000 task-6329: Ignore 'details.json' --- task-6329/.gitignore | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/task-6329/.gitignore b/task-6329/.gitignore new file mode 100644 index 0000000..0002bfe --- /dev/null +++ b/task-6329/.gitignore @@ -0,0 +1 @@ +details.json

1 0

[metrics-tasks/master] task-6329: Add tor-relay-stats.py as posted on tor-relays@ on 2012-07-12
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 8417025094b60f41b28d64d1b62a801a618c9bca Author: delber <delber(a)riseup.net> Date: Sat Jul 14 11:59:41 2012 +0000 task-6329: Add tor-relay-stats.py as posted on tor-relays@ on 2012-07-12 --- task-6329/tor-relays-stats.py | 125 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 125 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py new file mode 100755 index 0000000..0aa4854 --- /dev/null +++ b/task-6329/tor-relays-stats.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +# tor-relay-stats.py: extract some statistics about relays +# +# This script expect to have a file called 'details.json' in the +# current directory. In order to retrieve the needed data, one +# can issue the following command: +# +# curl -o details.json https://onionoo.torproject.org/details +# +# +# Author: delber(a)riseup.net +# +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms of the Do What The Fuck You Want +# To Public License, Version 2, as published by Sam Hocevar. See +# http://sam.zoy.org/wtfpl/COPYING for more details. + +import json +import operator +import sys + +class RelayStats(object): + def __init__(self): + self._data = None + + @property + def data(self): + if not self._data: + self._data = json.load(file('details.json')) + return self._data + + def get_total_consensus_weight(self, relays=None): + if relays is None: + relays = self.get_relays() + total_consensus_weight = 0 + for relay in relays: + total_consensus_weight += relay['consensus_weight'] + return total_consensus_weight + + def get_relays(self, flags=[], countries=''): + relays = [] + for relay in self.data['relays']: + if not relay['running']: + continue + if set(flags) & set(relay['flags']) != set(flags): + continue + if countries and not relay.get('country', '') in countries: + continue + relays.append(relay) + return relays + + def output_countries(self, flags=''): + flags = flags.split() + relays = self.get_relays(flags) + countries = {} + for relay in relays: + country = relay.get('country', None) + if country not in countries: + countries[country] = 0 + countries[country] += relay['consensus_weight'] + + total_consensus_weight = self.get_total_consensus_weight(relays) + for country, weight in sorted(countries.iteritems(), key=operator.itemgetter(1)): + print "%3.2f%% %s" % (weight * 100.0 / total_consensus_weight, country) + + def output_as_sets(self, flags='', countries=''): + flags = flags.split() + relays = self.get_relays(flags, countries) + as_sets = {} + for relay in relays: + as_set = relay.get('as_name', 'Unknown') + if as_set not in as_sets: + as_sets[as_set] = 0 + as_sets[as_set] += relay['consensus_weight'] + + total_consensus_weight = self.get_total_consensus_weight(relays) + for as_set, weight in sorted(as_sets.iteritems(), key=operator.itemgetter(1)): + print "%3.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) + + def output_top(self, count='10', flags='', countries=''): + count = int(count) + flags = flags.split() + relays = self.get_relays(flags, countries) + top = sorted(relays, key=operator.itemgetter('consensus_weight')) + total_consensus_weight = self.get_total_consensus_weight() + for relay in top[-count:]: + + print "%3.4f%% %-20s %s" % (relay['consensus_weight'] * 100.0 / total_consensus_weight, relay['nickname'], relay['fingerprint']) + +OUTPUTS = { + 'countries': 'output_countries', + 'as-sets': 'output_as_sets', + 'top': 'output_top', +} + +def usage(): + print >>sys.stderr, """Usage: %(progname)s <output> [args ...] + +Where <output> is one of: + - countries [FLAGS] + relative percentage of the consensus in each countries + - as-sets [FLAGS] [COUNTRIES] + relative percentage of the consensus in each AS sets + - top [COUNT] [FLAGS] [COUNTRIES] + top relays according their place in the whole consensus + +Examples: + + - To get the top five exit nodes in France: + %(progname)s top 5 Exit fr + - To get weights of each AS of all relays in Germany: + %(progname)s as-sets Running de +""" % { 'progname': sys.argv[0] } + sys.exit(1) + +if '__main__' == __name__: + if len(sys.argv) == 1: + usage() + func = OUTPUTS.get(sys.argv[1], None) + if not func: + usage() + stats = RelayStats() + getattr(stats, func)(*sys.argv[2:])

1 0

[metrics-tasks/master] task-6329: Suggest an optimized Onionoo URL
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 8d7f697c14c83a4e7aa75ccab9351f4db0fc4d87 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 13:00:11 2012 +0000 task-6329: Suggest an optimized Onionoo URL As we are only interested in running relays, this saves 3 MB of download according to Karsten. --- task-6329/README | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6329/README b/task-6329/README index a09e7ab..d5b4c0f 100644 --- a/task-6329/README +++ b/task-6329/README @@ -31,7 +31,7 @@ This script expect to have a file called 'details.json' in the current directory. In order to retrieve the needed data, one can issue the following command: - curl -o details.json https://onionoo.torproject.org/details + curl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true' Dependencies ------------

1 0

[metrics-tasks/master] task-6329: Rename 'top' output to 'relays'
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit fd3fbf2b3c58ea87d29b0e8b628319d3054f0707 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 12:55:21 2012 +0000 task-6329: Rename 'top' output to 'relays' All outputs now only display the top ten by defaults, so let's rename the output that list relays, 'relays', as suggested by Karsten. --- task-6329/README | 4 ++-- task-6329/tor-relays-stats.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/task-6329/README b/task-6329/README index 443a3d6..a09e7ab 100644 --- a/task-6329/README +++ b/task-6329/README @@ -14,8 +14,8 @@ Where <output> is one of: relative percentage of the consensus in each countries - as-sets [COUNT] [FLAGS] [COUNTRIES] relative percentage of the consensus in each AS sets - - top [COUNT] [FLAGS] [COUNTRIES] - top relays according their place in the whole consensus + - relays [COUNT] [FLAGS] [COUNTRIES] + list relays ranked by their place in the whole consensus Examples: diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index d6b3e86..fd30792 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -74,7 +74,7 @@ class RelayStats(object): for as_set, weight in ranking[:count]: print "%3.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) - def output_top(self, count='10', flags='', countries=''): + def output_relays(self, count='10', flags='', countries=''): count = int(count) flags = flags.split() relays = self.get_relays(flags, countries) @@ -88,7 +88,7 @@ class RelayStats(object): OUTPUTS = { 'countries': 'output_countries', 'as-sets': 'output_as_sets', - 'top': 'output_top', + 'relays': 'output_relays', } def usage(): @@ -99,8 +99,8 @@ Where <output> is one of: relative percentage of the consensus in each countries - as-sets [COUNT] [FLAGS] [COUNTRIES] relative percentage of the consensus in each AS sets - - top [COUNT] [FLAGS] [COUNTRIES] - top relays according their place in the whole consensus + - relays [COUNT] [FLAGS] [COUNTRIES] + list relays ranked by their place in the whole consensus Examples:

1 0

[metrics-tasks/master] task-6329: Add the curl command to usage output for those who do not read README
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit a7bcab237396e0446cf269fd0c2343d4ea17a4e9 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 13:02:42 2012 +0000 task-6329: Add the curl command to usage output for those who do not read README --- task-6329/tor-relays-stats.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index fd30792..5f6f633 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -108,6 +108,12 @@ Examples: %(progname)s top 5 Exit fr - To get weights of the top ten AS of all relays in Germany: %(progname)s as-sets 10 Running de + +This script expect to have a file called 'details.json' in the +current directory. In order to retrieve the needed data, one +can issue the following command: + + curl -o details.json 'https://onionoo.torproject.org/details?type=relay&running=true' """ % { 'progname': sys.argv[0] } sys.exit(1)

1 0

[metrics-tasks/master] task-6329: Display usage when details.json cannot be found
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit c91ad8eaf8a05f839a8270986a72599ccc2b8ca9 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 13:03:49 2012 +0000 task-6329: Display usage when details.json cannot be found --- task-6329/tor-relays-stats.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 5f6f633..cbb70ac 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -9,6 +9,7 @@ import json import operator import sys +import os.path class RelayStats(object): def __init__(self): @@ -118,6 +119,8 @@ can issue the following command: sys.exit(1) if '__main__' == __name__: + if not os.path.exists('details.json'): + usage() if len(sys.argv) == 1: usage() func = OUTPUTS.get(sys.argv[1], None)

1 0

[metrics-tasks/master] task-6329: Add a 'count' parameter to 'as-sets' and 'countries' output
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 39fe1e46b9c4a13bd181edbde285a3e82a5cadd9 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 12:53:25 2012 +0000 task-6329: Add a 'count' parameter to 'as-sets' and 'countries' output --- task-6329/README | 8 ++++---- task-6329/tor-relays-stats.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/task-6329/README b/task-6329/README index 51f9dd7..443a3d6 100644 --- a/task-6329/README +++ b/task-6329/README @@ -10,9 +10,9 @@ Usage Usage: tor-relays-stats.py <output> [args ...] Where <output> is one of: - - countries [FLAGS] + - countries [COUNT] [FLAGS] relative percentage of the consensus in each countries - - as-sets [FLAGS] [COUNTRIES] + - as-sets [COUNT] [FLAGS] [COUNTRIES] relative percentage of the consensus in each AS sets - top [COUNT] [FLAGS] [COUNTRIES] top relays according their place in the whole consensus @@ -21,8 +21,8 @@ Examples: - To get the top five exit nodes in France: tor-relays-stats.py top 5 Exit fr - - To get weights of each AS of all relays in Germany: - tor-relays-stats.py as-sets Running de + - To get weights of the top ten AS of all relays in Germany: + tor-relays-stats.py as-sets 10 Running de Retrieving the data ------------------- diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index ed81bb5..d6b3e86 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -40,7 +40,8 @@ class RelayStats(object): relays.append(relay) return relays - def output_countries(self, flags=''): + def output_countries(self, count='10', flags=''): + count = int(count) flags = flags.split() relays = self.get_relays(flags) countries = {} @@ -53,10 +54,11 @@ class RelayStats(object): ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) ranking.reverse() total_consensus_weight = self.get_total_consensus_weight(relays) - for country, weight in ranking: + for country, weight in ranking[:count]: print "%3.2f%% %s" % (weight * 100.0 / total_consensus_weight, country) - def output_as_sets(self, flags='', countries=''): + def output_as_sets(self, count='10', flags='', countries=''): + count = int(count) flags = flags.split() relays = self.get_relays(flags, countries) as_sets = {} @@ -69,7 +71,7 @@ class RelayStats(object): total_consensus_weight = self.get_total_consensus_weight(relays) ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) ranking.reverse() - for as_set, weight in ranking: + for as_set, weight in ranking[:count]: print "%3.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) def output_top(self, count='10', flags='', countries=''): @@ -93,9 +95,9 @@ def usage(): print >>sys.stderr, """Usage: %(progname)s <output> [args ...] Where <output> is one of: - - countries [FLAGS] + - countries [COUNT] [FLAGS] relative percentage of the consensus in each countries - - as-sets [FLAGS] [COUNTRIES] + - as-sets [COUNT] [FLAGS] [COUNTRIES] relative percentage of the consensus in each AS sets - top [COUNT] [FLAGS] [COUNTRIES] top relays according their place in the whole consensus @@ -104,8 +106,8 @@ Examples: - To get the top five exit nodes in France: %(progname)s top 5 Exit fr - - To get weights of each AS of all relays in Germany: - %(progname)s as-sets Running de + - To get weights of the top ten AS of all relays in Germany: + %(progname)s as-sets 10 Running de """ % { 'progname': sys.argv[0] } sys.exit(1)

1 0

[metrics-tasks/master] task-6329: Sort results in descending order
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 83b2b0be3f9a12ca8f9e8e447b97850519d04f4a Author: delber <delber(a)riseup.net> Date: Sat Jul 14 12:47:52 2012 +0000 task-6329: Sort results in descending order --- task-6329/tor-relays-stats.py | 14 ++++++++++---- 1 files changed, 10 insertions(+), 4 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 245865f..ed81bb5 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -50,8 +50,10 @@ class RelayStats(object): countries[country] = 0 countries[country] += relay['consensus_weight'] + ranking = sorted(countries.iteritems(), key=operator.itemgetter(1)) + ranking.reverse() total_consensus_weight = self.get_total_consensus_weight(relays) - for country, weight in sorted(countries.iteritems(), key=operator.itemgetter(1)): + for country, weight in ranking: print "%3.2f%% %s" % (weight * 100.0 / total_consensus_weight, country) def output_as_sets(self, flags='', countries=''): @@ -65,16 +67,20 @@ class RelayStats(object): as_sets[as_set] += relay['consensus_weight'] total_consensus_weight = self.get_total_consensus_weight(relays) - for as_set, weight in sorted(as_sets.iteritems(), key=operator.itemgetter(1)): + ranking = sorted(as_sets.iteritems(), key=operator.itemgetter(1)) + ranking.reverse() + for as_set, weight in ranking: print "%3.4f%% %s" % (weight * 100.0 / total_consensus_weight, as_set) def output_top(self, count='10', flags='', countries=''): count = int(count) flags = flags.split() relays = self.get_relays(flags, countries) - top = sorted(relays, key=operator.itemgetter('consensus_weight')) + total_consensus_weight = self.get_total_consensus_weight() - for relay in top[-count:]: + ranking = sorted(relays, key=operator.itemgetter('consensus_weight')) + ranking.reverse() + for relay in ranking[:count]: print "%3.4f%% %-20s %s" % (relay['consensus_weight'] * 100.0 / total_consensus_weight, relay['nickname'], relay['fingerprint']) OUTPUTS = {

1 0

[metrics-tasks/master] task-6329: Add a proper README file
by karsten＠torproject.org 15 Jul '12

15 Jul '12

commit 5af3e23d0160c097f02b1342821fa59174834b26 Author: delber <delber(a)riseup.net> Date: Sat Jul 14 12:05:15 2012 +0000 task-6329: Add a proper README file --- task-6329/README | 53 +++++++++++++++++++++++++++++++++++++++++ task-6329/tor-relays-stats.py | 11 -------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/task-6329/README b/task-6329/README new file mode 100644 index 0000000..51f9dd7 --- /dev/null +++ b/task-6329/README @@ -0,0 +1,53 @@ +tor-relay-stats.py +================== + +This script extract some statistics about relays using aggregate data +based on the current consensus. + +Usage +----- + +Usage: tor-relays-stats.py <output> [args ...] + +Where <output> is one of: + - countries [FLAGS] + relative percentage of the consensus in each countries + - as-sets [FLAGS] [COUNTRIES] + relative percentage of the consensus in each AS sets + - top [COUNT] [FLAGS] [COUNTRIES] + top relays according their place in the whole consensus + +Examples: + + - To get the top five exit nodes in France: + tor-relays-stats.py top 5 Exit fr + - To get weights of each AS of all relays in Germany: + tor-relays-stats.py as-sets Running de + +Retrieving the data +------------------- + +This script expect to have a file called 'details.json' in the +current directory. In order to retrieve the needed data, one +can issue the following command: + + curl -o details.json https://onionoo.torproject.org/details + +Dependencies +------------ + +Written and tested on Python 2.6.6. + +Author +------ + + * delber <delber(a)riseup.net> + +Licence +------- + +This program is free software. It comes without any warranty, to +the extent permitted by applicable law. You can redistribute it +and/or modify it under the terms of the Do What The Fuck You Want +To Public License, Version 2, as published by Sam Hocevar. See +http://sam.zoy.org/wtfpl/COPYING for more details. diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 822317a..245865f 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -1,16 +1,5 @@ #!/usr/bin/env python # -# tor-relay-stats.py: extract some statistics about relays -# -# This script expect to have a file called 'details.json' in the -# current directory. In order to retrieve the needed data, one -# can issue the following command: -# -# curl -o details.json https://onionoo.torproject.org/details -# -# -# Author: delber(a)riseup.net -# # This program is free software. It comes without any warranty, to # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms of the Do What The Fuck You Want

1 0