September 2013 - tor-commits - lists.torproject.org

[oonib/master] Merge pull request #11 from waaaaargh/master
by art＠torproject.org 11 Sep '13

11 Sep '13

commit daaef7623871feb66ef52439abc17e26bf7b7ef2 Merge: 6eb53ba 3d578ab Author: Arturo Filastò <hellais(a)gmail.com> Date: Fri Jun 21 08:21:59 2013 -0700 Merge pull request #11 from waaaaargh/master Repairs Vagrantfile Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[oonib/master] Add static file handler for decks and inputs
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 01a69dc3669b8fc9c42688d048466b96c1d5ccc2 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 1 16:23:51 2013 +0200 Add static file handler for decks and inputs --- oonib.conf.example | 2 ++ oonib/report/api.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/oonib.conf.example b/oonib.conf.example index b6924f1..1b51621 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,6 +1,8 @@ main: report_dir: Null archive_dir: Null + input_dir: Null + deck_dir: Null logfile: Null tor_datadir: Null database_uri: 'sqlite://oonib_test_db.db' diff --git a/oonib/report/api.py b/oonib/report/api.py index 7747a1e..c250e71 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -48,7 +48,9 @@ def parseUpdateReportRequest(request): reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), - (r"/pcap", file_collector.PCAPReportHandler) + (r"/pcap", file_collector.PCAPReportHandler), + (r"/deck/([a-z0-9]{40})$", web.StaticFileHandler, {"path": config.deck_dir}), + (r"/input/([a-z0-9]{40})$", web.StaticFileHandler, {"path": config.input_dir}), ] reportingBackend = web.Application(reportingBackendAPI, debug=True)

1 0

[oonib/master] Merge pull request #13 from aagbsn/fix/set_logfile_from_config
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 3f0f8e2b2c8fe3f2672c8e001cd80a6260d460f4 Merge: daaef76 ab1fed3 Author: Arturo Filastò <hellais(a)gmail.com> Date: Mon Jul 1 08:34:48 2013 -0700 Merge pull request #13 from aagbsn/fix/set_logfile_from_config Fixes a bug where logfile config option was unused. oonib/log.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)

1 0

[oonib/master] Use correct config path
by art＠torproject.org 11 Sep '13

11 Sep '13

commit b13875e947ecb4cf2931cd5da869017066a56f37 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 1 17:56:23 2013 +0200 Use correct config path --- oonib/report/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oonib/report/api.py b/oonib/report/api.py index c250e71..871836b 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -49,8 +49,10 @@ reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), (r"/pcap", file_collector.PCAPReportHandler), - (r"/deck/([a-z0-9]{40})$", web.StaticFileHandler, {"path": config.deck_dir}), - (r"/input/([a-z0-9]{40})$", web.StaticFileHandler, {"path": config.input_dir}), + (r"/deck/([a-z0-9]{40})$", web.StaticFileHandler, {"path": + config.main.deck_dir}), + (r"/input/([a-z0-9]{40})$", web.StaticFileHandler, {"path": + config.main.input_dir}), ] reportingBackend = web.Application(reportingBackendAPI, debug=True)

1 0

[oonib/master] Add Handlers that list Decks and Inputs
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a40c62e1f4e49fbf5949a328f12df9b4e629199a Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:30:38 2013 +0200 Add Handlers that list Decks and Inputs --- oonib/report/api.py | 2 ++ oonib/report/file_collector.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/oonib/report/api.py b/oonib/report/api.py index 871836b..8835784 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -49,6 +49,8 @@ reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), (r"/pcap", file_collector.PCAPReportHandler), + (r"/deck", file_collector.DeckListHandler), + (r"/input", file_collector.InputListHandler), (r"/deck/([a-z0-9]{40})$", web.StaticFileHandler, {"path": config.main.deck_dir}), (r"/input/([a-z0-9]{40})$", web.StaticFileHandler, {"path": diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index c8f0a63..57c4175 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -287,3 +287,17 @@ class PCAPReportHandler(web.RequestHandler): def post(self): pass + +class DeckListHandler(web.RequestHandler): + def get(self): + if not config.main.deck_dir: return + path = os.path.abspath(config.main.deck_dir) + "/*" + decknames = map(os.path.basename, glob.iglob(path)) + self.write(json.dumps(decknames)) + +class InputListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + self.write(json.dumps(decknames))

1 0

[oonib/master] Set Tor DataDir in startTor
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 31bfac043a400d218a812e7dc190ea804e9c6137 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:19:24 2013 +0200 Set Tor DataDir in startTor Tor's DataDir should be set before starting Tor so that descriptors are cached rather than downloaded each time. --- oonib/runner.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index a6b479f..edd1b6b 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -47,14 +47,6 @@ def setupCollector(tor_process_protocol): os.makedirs(tempfile.gettempdir()) _temp_dir = tempfile.mkdtemp() - if config.main.tor_datadir is None: - log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") - log.msg("Creating tmp directory in current directory for datadir.") - log.debug("Using %s" % _temp_dir) - datadir = _temp_dir - else: - datadir = config.main.tor_datadir - torconfig = TorConfig(tor_process_protocol.tor_protocol) public_port = 80 # XXX there is currently a bug in txtorcon that prevents data_dir from @@ -75,6 +67,14 @@ def startTor(): if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60 + if config.main.tor_datadir is None: + log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") + log.msg("Creating tmp directory in current directory for datadir.") + log.debug("Using %s" % _temp_dir) + datadir = _temp_dir + else: + datadir = config.main.tor_datadir + torconfig.DataDirectory = datadir torconfig.save() if config.main.tor_binary is not None: d = launch_tor(torconfig, reactor,

1 0

[oonib/master] Clean up imports
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 834361018e6b35fd8db6be943aca30ba439eaae2 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:29:37 2013 +0200 Clean up imports --- oonib/report/file_collector.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 490ed77..c8f0a63 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -1,23 +1,17 @@ +import glob +import json +import os import random +import re import string import time import yaml -import json -import re -import os - -from datetime import datetime - -from twisted.internet import fdesc, reactor from cyclone import web - -from oonib import randomStr -from oonib import otime - +from datetime import datetime +from oonib import randomStr, otime, config, log from oonib.report import MissingField, InvalidRequestField - -from oonib import config, log +from twisted.internet import fdesc, reactor def parseUpdateReportRequest(request): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$")

1 0

[oonib/master] Add Handlers and API for policy
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 5a349ee10d01b5cdb7c9eb256b58e78b310559ad Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 19:47:40 2013 +0200 Add Handlers and API for policy --- oonib/policy/api.py | 40 ++++++++++++++++++++++++ oonib/policy/handlers.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/oonib/policy/api.py b/oonib/policy/api.py new file mode 100644 index 0000000..577d988 --- /dev/null +++ b/oonib/policy/api.py @@ -0,0 +1,40 @@ +""" +/report + +/pcap + +This is the async pcap reporting system. It requires the client to have created +a report already, but can work independently from test progress. + +""" +import random +import string +import json +import re +import os + +from twisted.internet import reactor, defer + +from cyclone import web + +from oonib import otime +from oonib import randomStr + +from oonib import config +from oonib.policy import DeckListHandler, InputListHandler, NetTestListHandler + +#XXX: if policy is configured +policyAPI = [ + (r"/deck", DeckListHandler), + (r"/input", InputListHandler), + (r"/deck/([a-z0-9]{40})$", DeckDescHandler), + (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.deck_dir}), + (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), + (r"/input([a-z0-9]{40}$", InputDescHandler), + (r"/policy/nettest", NetTestListHandler), + (r"/policy/nettest/([a-z0-9]+/py$", web.StaticFileHandler, {"path": + config.main.nettest_dir}), + (r"/policy/input", InputListHandler +] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py new file mode 100644 index 0000000..567da2c --- /dev/null +++ b/oonib/policy/handlers.py @@ -0,0 +1,78 @@ +from cyclone import web +from oonib import config +import json +import os +import yaml + +class DeckDescHandler(web.RequestHandler): + def get(self, deckID): + bn = os.path.basename(deckID) + try: + f = open(os.path.join(config.main.deck_dir, bn)) + a = {} + deckDesc = yaml.safe_load(f) + a['id'] = deckID + a['name'] = deckDesc['name'] + a['description'] = deckDesc['description'] + self.write(json.dumps(a)) + except IOError: + log.err("Deck %s missing" % deckID) + except KeyError: + log.err("Deck %s missing required keys!" % deckID) + +class DeckListHandler(web.RequestHandler): + def get(self): + if not config.main.deck_dir: return + path = os.path.abspath(config.main.deck_dir) + "/*" + decknames = map(os.path.basename, glob.iglob(path)) + deckList = [] + for deckname in decknames: + f = open(os.path.join(config.main.deck_dir, deckname)) + d = yaml.safe_load(f) + deckList.append({'id': deckname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(deckList)) + +class InputDescHandler(web.RequestHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, deckname)) + d = yaml.safe_load(f) + inputList.append({'id': inputname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(inputList)) + +class NetTestListHandler(web.RequestHandler): + def get(self): + #XXX: returns a list of accepted NetTests + pass + +class HelperListHandler(web.RequestHandler): + def get(self): + #XXX: get the list of the running handlers + #'id' + #'description' + #'address' + pass

1 0

[oonib/master] Add Handlers and API for bouncer
by art＠torproject.org 11 Sep '13

11 Sep '13

commit bbe4e010381d1daf0710672f531c9fe99c9c93cf Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 19:48:06 2013 +0200 Add Handlers and API for bouncer --- oonib/bouncer/api.py | 8 ++++++++ oonib/bouncer/handlers.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/oonib/bouncer/api.py b/oonib/bouncer/api.py new file mode 100644 index 0000000..21f2be4 --- /dev/null +++ b/oonib/bouncer/api.py @@ -0,0 +1,8 @@ +from oonib.bouncer.handlers import * +#XXX: if bouncer is configured +bouncerAPI = [ + #return a collector and helper of the requested type + (r"/bouncer", BouncerQueryHandler), + #XXX: register a collector or helper + (r"/register", CollectorRegisterHandler), +] diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py new file mode 100644 index 0000000..74d828a --- /dev/null +++ b/oonib/bouncer/handlers.py @@ -0,0 +1,19 @@ +class BouncerQueryHandler(web.RequestHandler): + def get(self): + #XXX unused + pass + + def post(self): + pass + # request.get? + #'collector': {'foo.onion': {'helper-name': 'helper-address'}, 'bar.onion': {'helper-name': 'helper-address'}} + +class CollectorRegisterHandler(self): + def post(self): + #XXX: update the list of collectors to query + pass + + def get(self): + pass + #XXX unused +

1 0

[oonib/master] Organize handlers and api for each component
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a33028d8e4708e14da7b23e0b5c2c87c60650d5e Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 20:08:43 2013 +0200 Organize handlers and api for each component --- oonib/deck/api.py | 10 +++++++ oonib/deck/handlers.py | 31 +++++++++++++++++++++ oonib/input/api.py | 10 +++++++ oonib/input/handlers.py | 35 +++++++++++++++++++++++ oonib/policy/api.py | 40 ++++----------------------- oonib/policy/handlers.py | 69 ++-------------------------------------------- 6 files changed, 95 insertions(+), 100 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py new file mode 100644 index 0000000..db6fb66 --- /dev/null +++ b/oonib/deck/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.deck import handlers +from oonib import config + +deckAPI = [ + (r"/deck", handlers.DeckListHandler), + (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), + (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.deck_dir}), +] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py new file mode 100644 index 0000000..f37609a --- /dev/null +++ b/oonib/deck/handlers.py @@ -0,0 +1,31 @@ +class DeckDescHandler(web.RequestHandler): + def get(self, deckID): + bn = os.path.basename(deckID) + try: + f = open(os.path.join(config.main.deck_dir, bn)) + a = {} + deckDesc = yaml.safe_load(f) + a['id'] = deckID + a['name'] = deckDesc['name'] + a['description'] = deckDesc['description'] + self.write(json.dumps(a)) + except IOError: + log.err("Deck %s missing" % deckID) + except KeyError: + log.err("Deck %s missing required keys!" % deckID) + +class DeckListHandler(web.RequestHandler): + def get(self): + if not config.main.deck_dir: return + path = os.path.abspath(config.main.deck_dir) + "/*" + decknames = map(os.path.basename, glob.iglob(path)) + deckList = [] + for deckname in decknames: + f = open(os.path.join(config.main.deck_dir, deckname)) + d = yaml.safe_load(f) + deckList.append({'id': deckname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(deckList)) + + diff --git a/oonib/input/api.py b/oonib/input/api.py new file mode 100644 index 0000000..4c397ab --- /dev/null +++ b/oonib/input/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.input import handlers +from oonib import config + +inputAPI = [ + (r"/input", handlers.InputListHandler), + (r"/input/([a-z0-9]{40})", handlers.InputDescHandler), + (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), +] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py new file mode 100644 index 0000000..ed8008e --- /dev/null +++ b/oonib/input/handlers.py @@ -0,0 +1,35 @@ +from oonib import config +from cyclone import web +import json +import os +import yaml + +class InputDescHandler(web.RequestHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, deckname)) + d = yaml.safe_load(f) + inputList.append({'id': inputname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(inputList)) diff --git a/oonib/policy/api.py b/oonib/policy/api.py index 577d988..0f6f023 100644 --- a/oonib/policy/api.py +++ b/oonib/policy/api.py @@ -1,40 +1,12 @@ -""" -/report - -/pcap - -This is the async pcap reporting system. It requires the client to have created -a report already, but can work independently from test progress. - -""" -import random -import string -import json -import re -import os - -from twisted.internet import reactor, defer - from cyclone import web - -from oonib import otime -from oonib import randomStr - +from oonib.policy import handlers from oonib import config -from oonib.policy import DeckListHandler, InputListHandler, NetTestListHandler #XXX: if policy is configured policyAPI = [ - (r"/deck", DeckListHandler), - (r"/input", InputListHandler), - (r"/deck/([a-z0-9]{40})$", DeckDescHandler), - (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": - config.main.deck_dir}), - (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": - config.main.input_dir}), - (r"/input([a-z0-9]{40}$", InputDescHandler), - (r"/policy/nettest", NetTestListHandler), - (r"/policy/nettest/([a-z0-9]+/py$", web.StaticFileHandler, {"path": - config.main.nettest_dir}), - (r"/policy/input", InputListHandler + (r"/policy/nettest", handlers.NetTestPolicyHandler), + #XXX: add nettest handler + #(r"/policy/nettest/([a-z0-9]+)/py$", web.StaticFileHandler, {"path": + # config.main.nettest_dir}), + (r"/policy/input", handlers.InputPolicyHandler), ] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 567da2c..2b55fb7 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -4,75 +4,12 @@ import json import os import yaml -class DeckDescHandler(web.RequestHandler): - def get(self, deckID): - bn = os.path.basename(deckID) - try: - f = open(os.path.join(config.main.deck_dir, bn)) - a = {} - deckDesc = yaml.safe_load(f) - a['id'] = deckID - a['name'] = deckDesc['name'] - a['description'] = deckDesc['description'] - self.write(json.dumps(a)) - except IOError: - log.err("Deck %s missing" % deckID) - except KeyError: - log.err("Deck %s missing required keys!" % deckID) - -class DeckListHandler(web.RequestHandler): - def get(self): - if not config.main.deck_dir: return - path = os.path.abspath(config.main.deck_dir) + "/*" - decknames = map(os.path.basename, glob.iglob(path)) - deckList = [] - for deckname in decknames: - f = open(os.path.join(config.main.deck_dir, deckname)) - d = yaml.safe_load(f) - deckList.append({'id': deckname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(deckList)) - -class InputDescHandler(web.RequestHandler): - def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec - bn = os.path.basename(inputID) + ".desc" - try: - f = open(os.path.join(config.main.input_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] - self.write(json.dumps(a)) - except Exception: - log.err("No Input Descriptor found for id %s" % inputID) - -class InputListHandler(web.RequestHandler): - def get(self): - if not config.main.input_dir: return - path = os.path.abspath(config.main.input_dir) + "/*" - inputnames = map(os.path.basename, glob.iglob(path)) - inputList = [] - for inputname in inputnames: - f = open(os.path.join(config.main.input_dir, deckname)) - d = yaml.safe_load(f) - inputList.append({'id': inputname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(inputList)) - -class NetTestListHandler(web.RequestHandler): +class NetTestPolicyHandler(web.RequestHandler): def get(self): #XXX: returns a list of accepted NetTests pass -class HelperListHandler(web.RequestHandler): +class InputPolicyHandler(web.RequestHandler): def get(self): - #XXX: get the list of the running handlers - #'id' - #'description' - #'address' pass + #XXX return list of input ids

1 0