September 2013 - tor-commits - lists.torproject.org

[oonib/master] Normalize pluralization, use sha256 for input name
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 7d753791abef68c3cbbaf390b7ae92c01f1a56a8 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:11:05 2013 +0200 Normalize pluralization, use sha256 for input name Use the same pluralization semantics as the /deck api, and use sha256 hexdigest filename as specified in oonib.md --- oonib.conf.example | 2 +- oonib/api.py | 6 +++--- oonib/input/api.py | 10 ++++++++++ oonib/input/handlers.py | 41 +++++++++++++++++++++++++++++++++++++++++ oonib/inputs/api.py | 10 ---------- oonib/inputs/handlers.py | 41 ----------------------------------------- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 36df0e3..977b5dc 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,7 +1,7 @@ main: report_dir: data/reports/ archive_dir: data/archive/ - inputs_dir: data/inputs/ + input_dir: data/inputs/ deck_dir: data/decks/ policy_file: data/policy.yaml bouncer_file: data/bouncer.yaml diff --git a/oonib/api.py b/oonib/api.py index 775c4ad..384b949 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -2,7 +2,7 @@ from cyclone import web from oonib.deck.api import deckAPI from oonib.report.api import reportAPI -from oonib.inputs.api import inputsAPI +from oonib.input.api import inputAPI from oonib.policy.api import policyAPI from oonib.bouncer.api import bouncerAPI @@ -11,8 +11,8 @@ from oonib import config ooniBackendAPI = [] ooniBackendAPI += reportAPI -if config.main.inputs_dir: - ooniBackendAPI += inputsAPI +if config.main.input_dir: + ooniBackendAPI += inputAPI if config.main.deck_dir: ooniBackendAPI += deckAPI diff --git a/oonib/input/__init__.py b/oonib/input/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/input/api.py b/oonib/input/api.py new file mode 100644 index 0000000..88d85a2 --- /dev/null +++ b/oonib/input/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.input import handlers +from oonib import config + +inputAPI = [ + (r"/input", handlers.InputListHandler), + (r"/input/([a-f0-9]{64})", handlers.InputDescHandler), + (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), +] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py new file mode 100644 index 0000000..3ab0ea0 --- /dev/null +++ b/oonib/input/handlers.py @@ -0,0 +1,41 @@ +import glob +import json +import os + +import yaml + +from oonib.handlers import OONIBHandler + +from oonib import config + +class InputDescHandler(OONIBHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(OONIBHandler): + def get(self): + path = os.path.abspath(config.main.input_dir) + "/*.desc" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, inputname)) + d = yaml.safe_load(f) + inputList.append({ + 'id': inputname, + 'name': d['name'], + 'description': d['description'] + }) + f.close() + self.write(json.dumps(inputList)) diff --git a/oonib/inputs/__init__.py b/oonib/inputs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py deleted file mode 100644 index bfcf076..0000000 --- a/oonib/inputs/api.py +++ /dev/null @@ -1,10 +0,0 @@ -from cyclone import web -from oonib.inputs import handlers -from oonib import config - -inputsAPI = [ - (r"/inputs", handlers.InputsListHandler), - (r"/inputs/([a-f0-9]{64})", handlers.InputsDescHandler), - (r"/inputs/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.inputs_dir}), -] diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py deleted file mode 100644 index 373e275..0000000 --- a/oonib/inputs/handlers.py +++ /dev/null @@ -1,41 +0,0 @@ -import glob -import json -import os - -import yaml - -from oonib.handlers import OONIBHandler - -from oonib import config - -class InputsDescHandler(OONIBHandler): - def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec - bn = os.path.basename(inputID) + ".desc" - try: - f = open(os.path.join(config.main.inputs_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] - self.write(json.dumps(a)) - except Exception: - log.err("No Input Descriptor found for id %s" % inputID) - -class InputsListHandler(OONIBHandler): - def get(self): - path = os.path.abspath(config.main.inputs_dir) + "/*.desc" - inputnames = map(os.path.basename, glob.iglob(path)) - inputList = [] - for inputname in inputnames: - f = open(os.path.join(config.main.inputs_dir, inputname)) - d = yaml.safe_load(f) - inputList.append({ - 'id': inputname, - 'name': d['name'], - 'description': d['description'] - }) - f.close() - self.write(json.dumps(inputList))

1 0

[oonib/master] Add some basic inputs descriptors and add makefile to fetch them
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d3749779dcc7691e5e14e9b2b3d60cf923757e70 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:02:58 2013 +0200 Add some basic inputs descriptors and add makefile to fetch them --- ...36f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc | 2 ++ data/inputs/Makefile | 7 +++++++ ...8bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc | 2 ++ 3 files changed, 11 insertions(+) diff --git a/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc new file mode 100644 index 0000000..feecc5a --- /dev/null +++ b/data/inputs/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1.desc @@ -0,0 +1,2 @@ +{id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1, name: Alexa Top 1k domains, description: 'The Alexa list of most visited domains fetched from here: http://www.alexa.com/topsites'} + diff --git a/data/inputs/Makefile b/data/inputs/Makefile new file mode 100644 index 0000000..8e53144 --- /dev/null +++ b/data/inputs/Makefile @@ -0,0 +1,7 @@ +lists: + curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz + tar xzf input-pack.tar.gz + rm input-pack.tar.gz + mv input-pack/alexa-top-1k.txt 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1 + mv input-pack/alexa-top-1m.txt e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290 + rm -rf input-pack diff --git a/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc new file mode 100644 index 0000000..07030c6 --- /dev/null +++ b/data/inputs/e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290.desc @@ -0,0 +1,2 @@ +{id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290, name: Alexa Top 1m domains, description: 'The Alexa list of 1 million most visited domains fetched from here: http://www.alexa.com/topsites'} +

1 0

[oonib/master] Only consider descriptors
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 2d1a18e80be7c4cd860dad8fc510a524a0f0453b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:09:39 2013 +0200 Only consider descriptors --- oonib/inputs/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index cf056ba..373e275 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -26,7 +26,7 @@ class InputsDescHandler(OONIBHandler): class InputsListHandler(OONIBHandler): def get(self): - path = os.path.abspath(config.main.inputs_dir) + "/*" + path = os.path.abspath(config.main.inputs_dir) + "/*.desc" inputnames = map(os.path.basename, glob.iglob(path)) inputList = [] for inputname in inputnames:

1 0

[oonib/master] Fix missing imports
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 56e54b5f0e6168c6bd8139be2cc3ab6dc2cf6f75 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 22:51:10 2013 +0200 Fix missing imports --- oonib/deck/handlers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index edbe361..b84d1de 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,4 +1,11 @@ +import glob +import json +import os +import re +import yaml + from oonib.handlers import OONIBHandler +from oonib import config, log class DeckDescHandler(OONIBHandler): def get(self, deckID): @@ -21,6 +28,7 @@ class DeckListHandler(OONIBHandler): if not config.main.deck_dir: return path = os.path.abspath(config.main.deck_dir) + "/*" decknames = map(os.path.basename, glob.iglob(path)) + decknames = filter(lambda y: re.match("[a-z0-9]{40}", y), decknames) deckList = [] for deckname in decknames: f = open(os.path.join(config.main.deck_dir, deckname)) @@ -29,5 +37,3 @@ class DeckListHandler(OONIBHandler): 'description': d['description']}) f.close() self.write(json.dumps(deckList)) - -

1 0

[oonib/master] Return the descriptor as specified in oonib.md
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d31aa2658630b4dd9d58008c99ac5db09190e18d Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:01:02 2013 +0200 Return the descriptor as specified in oonib.md --- oonib/deck/api.py | 2 +- oonib/deck/handlers.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index db6fb66..726b48a 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -5,6 +5,6 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{40})/yaml$", web.StaticFileHandler, {"path": config.main.deck_dir}), ] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index b84d1de..1b67a3f 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -14,9 +14,8 @@ class DeckDescHandler(OONIBHandler): f = open(os.path.join(config.main.deck_dir, bn)) a = {} deckDesc = yaml.safe_load(f) - a['id'] = deckID - a['name'] = deckDesc['name'] - a['description'] = deckDesc['description'] + for k in ['name', 'description', 'version', 'author', 'date']: + a[k] = deckDesc[k] self.write(json.dumps(a)) except IOError: log.err("Deck %s missing" % deckID)

1 0

[oonib/master] Use sha256 as per oonib.md spec
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d7a864a0df2723657d7ff5b02c7568042d49093f Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:29:04 2013 +0200 Use sha256 as per oonib.md spec --- oonib/deck/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py index 726b48a..6fbc7f4 100644 --- a/oonib/deck/api.py +++ b/oonib/deck/api.py @@ -4,7 +4,7 @@ from oonib import config deckAPI = [ (r"/deck", handlers.DeckListHandler), - (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), - (r"/deck/([a-z0-9]{40})/yaml$", web.StaticFileHandler, {"path": + (r"/deck/([a-z0-9]{64})$", handlers.DeckDescHandler), + (r"/deck/([a-z0-9]{64})/yaml$", web.StaticFileHandler, {"path": config.main.deck_dir}), ]

1 0

[oonib/master] Loosen regexp so that the descriptors are actually fetchable
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 7e2a526d88c20f038a5fabbe43062345b13448b9 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 22:11:07 2013 +0200 Loosen regexp so that the descriptors are actually fetchable --- oonib/inputs/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py index d74c23c..bfcf076 100644 --- a/oonib/inputs/api.py +++ b/oonib/inputs/api.py @@ -4,7 +4,7 @@ from oonib import config inputsAPI = [ (r"/inputs", handlers.InputsListHandler), - (r"/inputs/([a-z0-9]{40})", handlers.InputsDescHandler), - (r"/inputs/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + (r"/inputs/([a-f0-9]{64})", handlers.InputsDescHandler), + (r"/inputs/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": config.main.inputs_dir}), ]

1 0

[oonib/master] Implement input API as spec'd in oonib.md
by art＠torproject.org 11 Sep '13

11 Sep '13

commit b7fd4b9db379fc9bc28f22f6608c804e9f08e181 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:25:36 2013 +0200 Implement input API as spec'd in oonib.md --- oonib/input/handlers.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py index 3ab0ea0..8db1482 100644 --- a/oonib/input/handlers.py +++ b/oonib/input/handlers.py @@ -1,28 +1,25 @@ import glob import json import os - import yaml from oonib.handlers import OONIBHandler - -from oonib import config +from oonib import config, log class InputDescHandler(OONIBHandler): def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec bn = os.path.basename(inputID) + ".desc" try: f = open(os.path.join(config.main.input_dir, bn)) a = {} inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] + for k in ['name', 'description', 'version', 'author', 'date']: + a[k] = inputDesc[k] self.write(json.dumps(a)) - except Exception: + except IOError: log.err("No Input Descriptor found for id %s" % inputID) + except Exception, e: + log.err("Invalid Input Descriptor found for id %s" % inputID) class InputListHandler(OONIBHandler): def get(self):

1 0

[oonib/master] Add Policy API
by art＠torproject.org 11 Sep '13

11 Sep '13

commit bf2a444878730722abad43b869a2bb7fe1194579 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Aug 19 00:00:17 2013 +0200 Add Policy API --- data/policy.yaml | 2 +- oonib/policy/api.py | 4 ---- oonib/policy/handlers.py | 8 ++++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data/policy.yaml b/data/policy.yaml index fecdf29..e0d6c08 100644 --- a/data/policy.yaml +++ b/data/policy.yaml @@ -1,4 +1,4 @@ -inputs: +input: - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} nettest: diff --git a/oonib/policy/api.py b/oonib/policy/api.py index 4cacea2..d6fdaad 100644 --- a/oonib/policy/api.py +++ b/oonib/policy/api.py @@ -1,10 +1,6 @@ from oonib.policy import handlers -#XXX: if policy is configured policyAPI = [ (r"/policy/nettest", handlers.NetTestPolicyHandler), - #XXX: add nettest handler - #(r"/policy/nettest/([a-z0-9]+)/py$", web.StaticFileHandler, {"path": - # config.main.nettest_dir}), (r"/policy/input", handlers.InputPolicyHandler), ] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 7544cf9..f72e746 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -10,11 +10,15 @@ class NetTestPolicyHandler(OONIBHandler): """ returns a list of accepted NetTests """ - pass + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.write(json.dumps(p['nettest'])) class InputPolicyHandler(OONIBHandler): def get(self): """ return list of input ids """ - pass + with open(config.main.policy_file) as f: + p = yaml.safe_load(f) + self.write(json.dumps(p['input']))

1 0

[oonib/master] Add Bouncer API
by art＠torproject.org 11 Sep '13

11 Sep '13

commit e03798beb5678386749022e88c4593a7546951e4 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Aug 19 01:09:32 2013 +0200 Add Bouncer API --- oonib/bouncer/handlers.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 90651b0..162f61d 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,4 +1,25 @@ +import json +import random +import yaml from oonib.handlers import OONIBHandler +from oonib import config + +#knownHelpers = {'helper-name': ['collector-address1', 'collector-address2', ... 'collector-addressn']} +knownHelpers = {} + +def updateKnownHelpers(): + with open(config.main.bouncer_file) as f: + bouncerFile = yaml.safe_load(f) + for collectorName, helpers in bouncerFile['collector'].items(): + for helperName, helperAddress in helpers['test-helper'].items(): + if helperName not in knownHelpers.keys(): + knownHelpers[helperName] = [] + + knownHelpers[helperName].append( + {'collector-name': collectorName, + 'helper-address': helperAddress + }) +updateKnownHelpers() class BouncerQueryHandler(OONIBHandler): def get(self): @@ -6,6 +27,14 @@ class BouncerQueryHandler(OONIBHandler): pass def post(self): - pass - # request.get? - #'collector': {'foo.onion': {'helper-name': 'helper-address'}, 'bar.onion': {'helper-name': 'helper-address'}} + helpers = json.loads(self.request.body)['test-helpers'] + a = {} + a['collector'] = {} + for helperName in helpers: + if helperName in knownHelpers.keys(): + chosen = random.choice(knownHelpers[helperName]) + collectorName, helperAddress = chosen['collector-name'], chosen['helper-address'] + if not collectorName in a['collector'].keys(): + a['collector'][collectorName] = {'test-helper': {}} + a['collector'][collectorName]['test-helper'][helperName] = helperAddress + self.write(json.dumps(a))

1 0