September 2013 - tor-commits - lists.torproject.org

[oonib/master] Add some basic inputs descriptors and add makefile to fetch them
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d3749779dcc7691e5e14e9b2b3d60cf923757e70 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:02:58 2013 +0200 Add some basic inputs descriptors and add makefile to fetch them --- ...36f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc | 2 ++ data/inputs/Makefile | 7 +++++++ ...8bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc | 2 ++ 3 files changed, 11 insertions(+) diff --git a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc new file mode 100644 index 0000000..feecc5a --- /dev/null +++ b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc @@ -0,0 +1,2 @@ +{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites'} + diff --git a/data/inputs/Makefile b/data/inputs/Makefile new file mode 100644 index 0000000..8e53144 --- /dev/null +++ b/data/inputs/Makefile @@ -0,0 +1,7 @@ +lists: + curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz + tar xzf input-pack.tar.gz + rm input-pack.tar.gz + mv input-pack/alexa-top-1k.txt 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1 + mv input-pack/alexa-top-1m.txt e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290 + rm -rf input-pack diff --git a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc new file mode 100644 index 0000000..07030c6 --- /dev/null +++ b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc @@ -0,0 +1,2 @@ +{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites'} +

1 0

[oonib/master] Only consider descriptors
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 2d1a18e80be7c4cd860dad8fc510a524a0f0453b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:09:39 2013 +0200 Only consider descriptors --- oonib/inputs/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index cf056ba..373e275 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -26,7 +26,7 @@ class InputsDescHandler(OONIBHandler): class InputsListHandler(OONIBHandler): def get(self): - path = os.path.abspath(config.main.inputs_dir) + "/*" + path = os.path.abspath(config.main.inputs_dir) + "/*.desc" inputnames = map(os.path.basename, glob.iglob(path)) inputList = [] for inputname in inputnames:

1 0

[oonib/master] Fix missing imports
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 56e54b5f0e6168c6bd8139be2cc3ab6dc2cf6f75 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 22:51:10 2013 +0200 Fix missing imports --- oonib/deck/handlers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index edbe361..b84d1de 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,4 +1,11 @@ +import glob +import json +import os +import re +import yaml + from oonib.handlers import OONIBHandler +from oonib import config, log class DeckDescHandler(OONIBHandler): def get(self, deckID): @@ -21,6 +28,7 @@ class DeckListHandler(OONIBHandler): if not config.main.deck_dir: return path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) + decknames = filter(lambda y: re.match("[a-z0-9]{40}", y), decknames) deckList = [] for deckname in decknames: f = open(os.path.join(config.main.deck_dir, deckname)) @@ -29,5 +37,3 @@ class DeckListHandler(OONIBHandler): 'description': d['description']}) f.close() self.write(json.dumps(deckList)) - -

1 0

[oonib/master] Return the descriptor as specified in oonib.md
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d31aa2658630b4dd9d58008c99ac5db09190e18d Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:01:02 2013 +0200 Return the descriptor as specified in oonib.md --- oonib/deck/api.py | 2 +- oonib/deck/handlers.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index db6fb66..726b48a 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{40})/yaml$", web.StaticFileHandler, {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index b84d1de..1b67a3f 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -14,9 +14,8 @@ class DeckDescHandler(OONIBHandler): f = open(os.path.join(config.main.deck_dir, bn)) a = {} deckDesc = yaml.safe_load(f) - a['id'] = deckID - a['name'] = deckDesc['name'] - a['description'] = deckDesc['description'] + for k in ['name', 'description', 'version', 'author', 'date']: + a[k] = deckDesc[k] self.write(json.dumps(a)) except IOError: log.err("Deck %s missing" % deckID)

1 0

[oonib/master] Use sha256 as per oonib.md spec
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d7a864a0df2723657d7ff5b02c7568042d49093f Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:29:04 2013 +0200 Use sha256 as per oonib.md spec --- oonib/deck/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index 726b48a..6fbc7f4 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -4,7 +4,7 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), - (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{40})/yaml$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), + (r"/deck/([a-z0-9]{64})/yaml$", web.StaticFileHandler, {"path": config.main.deck_dir}), ]

1 0

[oonib/master] Loosen regexp so that the descriptors are actually fetchable
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 7e2a526d88c20f038a5fabbe43062345b13448b9 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:11:07 2013 +0200 Loosen regexp so that the descriptors are actually fetchable --- oonib/inputs/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py index d74c23c..bfcf076 100644 --- a/oonib/inputs/api.py +++ b/oonib/inputs/api.py @@ -4,7 +4,7 @@ from oonib import config inputsAPI = [ (r"/inputs", handlers.InputsListHandler), - (r"/inputs/([a-z0-9]{40})", handlers.InputsDescHandler), - (r"/inputs/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + (r"/inputs/([a-f0-9]{64})", handlers.InputsDescHandler), + (r"/inputs/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": config.main.inputs_dir}), ]

1 0

[oonib/master] Implement input API as spec'd in oonib.md
by art＠torproject.org 11 Sep '13

11 Sep '13

commit b7fd4b9db379fc9bc28f22f6608c804e9f08e181 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:25:36 2013 +0200 Implement input API as spec'd in oonib.md --- oonib/input/handlers.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index 3ab0ea0..8db1482 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -1,28 +1,25 @@ import glob import json import os - import yaml from oonib.handlers import OONIBHandler - -from oonib import config +from oonib import config, log class InputDescHandler(OONIBHandler): def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec bn = os.path.basename(inputID) + ".desc" try: f = open(os.path.join(config.main.input_dir, bn)) a = {} inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] + for k in ['name', 'description', 'version', 'author', 'date']: + a[k] = inputDesc[k] self.write(json.dumps(a)) - except Exception: + except IOError: log.err("No Input Descriptor found for id %s" % inputID) + except Exception, e: + log.err("Invalid Input Descriptor found for id %s" % inputID) class InputListHandler(OONIBHandler): def get(self):

1 0

[oonib/master] Add Policy API
by art＠torproject.org 11 Sep '13

11 Sep '13

commit bf2a444878730722abad43b869a2bb7fe1194579 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Aug 19 00:00:17 2013 +0200 Add Policy API --- data/policy.yaml | 2 +- oonib/policy/api.py | 4 ---- oonib/policy/handlers.py | 8 ++++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data/policy.yaml b/data/policy.yaml index fecdf29..e0d6c08 100644 --- a/data/policy.yaml +++ b/data/policy.yaml @@ -1,4 +1,4 @@ -inputs: +input: - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} nettest: diff --git a/oonib/policy/api.py b/oonib/policy/api.py index 4cacea2..d6fdaad 100644 --- a/oonib/policy/api.py +++ b/oonib/policy/api.py @@ -1,10 +1,6 @@ from oonib.policy import handlers -#XXX: if policy is configured policyAPI = [ (r"/policy/nettest", handlers.NetTestPolicyHandler), - #XXX: add nettest handler - #(r"/policy/nettest/([a-z0-9]+)/py$", web.StaticFileHandler, {"path": - # config.main.nettest_dir}), (r"/policy/input", handlers.InputPolicyHandler), ] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 7544cf9..f72e746 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -10,11 +10,15 @@ class NetTestPolicyHandler(OONIBHandler): """ returns a list of accepted NetTests """ - pass + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.write(json.dumps(p['nettest'])) class InputPolicyHandler(OONIBHandler): def get(self): """ return list of input ids """ - pass + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.write(json.dumps(p['input']))

1 0

[oonib/master] Add Bouncer API
by art＠torproject.org 11 Sep '13

11 Sep '13

commit e03798beb5678386749022e88c4593a7546951e4 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Aug 19 01:09:32 2013 +0200 Add Bouncer API --- oonib/bouncer/handlers.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 90651b0..162f61d 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,4 +1,25 @@ +import json +import random +import yaml from oonib.handlers import OONIBHandler +from oonib import config + +#knownHelpers = {'helper-name': ['collector-address1', 'collector-address2', ... 'collector-addressn']} +knownHelpers = {} + +def updateKnownHelpers(): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in knownHelpers.keys(): + knownHelpers[helperName] = [] + + knownHelpers[helperName].append( + {'collector-name': collectorName, + 'helper-address': helperAddress + }) +updateKnownHelpers() class BouncerQueryHandler(OONIBHandler): def get(self): @@ -6,6 +27,14 @@ class BouncerQueryHandler(OONIBHandler): pass def post(self): - pass - # request.get? - #'collector': {'foo.onion': {'helper-name': 'helper-address'}, 'bar.onion': {'helper-name': 'helper-address'}} + helpers = json.loads(self.request.body)['test-helpers'] + a = {} + a['collector'] = {} + for helperName in helpers: + if helperName in knownHelpers.keys(): + chosen = random.choice(knownHelpers[helperName]) + collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] + if not collectorName in a['collector'].keys(): + a['collector'][collectorName] = {'test-helper': {}} + a['collector'][collectorName]['test-helper'][helperName] = helperAddress + self.write(json.dumps(a))

1 0

[oonib/master] Refactor the bouncer handler
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 0089a0eda583b7b14cda255ab10b80a137380a72 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 19 14:03:31 2013 +0200 Refactor the bouncer handler --- oonib/bouncer/handlers.py | 64 +++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 162f61d..bc81ddb 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -4,37 +4,51 @@ import yaml from oonib.handlers import OONIBHandler from oonib import config -#knownHelpers = {'helper-name': ['collector-address1', 'collector-address2', ... 'collector-addressn']} -knownHelpers = {} +class BouncerQueryHandler(OONIBHandler): + def initialize(self): + self.knownHelpers = {} + self.updateKnownHelpers() -def updateKnownHelpers(): - with open(config.main.bouncer_file) as f: - bouncerFile = yaml.safe_load(f) - for collectorName, helpers in bouncerFile['collector'].items(): - for helperName, helperAddress in helpers['test-helper'].items(): - if helperName not in knownHelpers.keys(): - knownHelpers[helperName] = [] - - knownHelpers[helperName].append( - {'collector-name': collectorName, - 'helper-address': helperAddress - }) -updateKnownHelpers() + def updateKnownHelpers(self): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in self.knownHelpers.keys(): + self.knownHelpers[helperName] = [] + + self.knownHelpers[helperName].append({ + 'collector-name': collectorName, + 'helper-address': helperAddress + }) -class BouncerQueryHandler(OONIBHandler): def get(self): #XXX unused pass def post(self): - helpers = json.loads(self.request.body)['test-helpers'] - a = {} - a['collector'] = {} + try: + query = json.loads(self.request.body) + except ValueError: + self.set_status(400) + self.write(json.dumps({'error': 'invalid-request'})) + return + + try: + helpers = query['test-helpers'] + except KeyError: + self.set_status(400) + self.write(json.dumps({'error': 'test-helpers-key-missing'})) + return + + response = {} + response['collector'] = {} for helperName in helpers: - if helperName in knownHelpers.keys(): - chosen = random.choice(knownHelpers[helperName]) + if helperName in self.knownHelpers.keys(): + chosen = random.choice(self.knownHelpers[helperName]) collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] - if not collectorName in a['collector'].keys(): - a['collector'][collectorName] = {'test-helper': {}} - a['collector'][collectorName]['test-helper'][helperName] = helperAddress - self.write(json.dumps(a)) + if not collectorName in response['collector'].keys(): + response['collector'][collectorName] = {'test-helper': {}} + response['collector'][collectorName]['test-helper'][helperName] = helperAddress + + self.write(response)

1 0