September 2013 - tor-commits - lists.torproject.org

[oonib/master] Add static file handler for decks and inputs
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 01a69dc3669b8fc9c42688d048466b96c1d5ccc2 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 1 16:23:51 2013 +0200 Add static file handler for decks and inputs --- oonib.conf.example | 2 ++ oonib/report/api.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/oonib.conf.example b/oonib.conf.example index b6924f1..1b51621 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,6 +1,8 @@ main: report_dir: Null archive_dir: Null + … [View More]

1 0

[oonib/master] Merge pull request #13 from aagbsn/fix/set_logfile_from_config
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 3f0f8e2b2c8fe3f2672c8e001cd80a6260d460f4 Merge: daaef76 ab1fed3 Author: Arturo Filastò <hellais(a)gmail.com> Date: Mon Jul 1 08:34:48 2013 -0700 Merge pull request #13 from aagbsn/fix/set_logfile_from_config Fixes a bug where logfile config option was unused. oonib/log.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)

1 0

[oonib/master] Use correct config path
by art＠torproject.org 11 Sep '13

11 Sep '13

commit b13875e947ecb4cf2931cd5da869017066a56f37 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 1 17:56:23 2013 +0200 Use correct config path --- oonib/report/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oonib/report/api.py b/oonib/report/api.py index c250e71..871836b 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -49,8 +49,10 @@ reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.… [View More]

1 0

[oonib/master] Add Handlers that list Decks and Inputs
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a40c62e1f4e49fbf5949a328f12df9b4e629199a Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:30:38 2013 +0200 Add Handlers that list Decks and Inputs --- oonib/report/api.py | 2 ++ oonib/report/file_collector.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/oonib/report/api.py b/oonib/report/api.py index 871836b..8835784 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -49,6 +49,8 @@ reportingBackendAPI = [ (r"/… [View More]

1 0

[oonib/master] Set Tor DataDir in startTor
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 31bfac043a400d218a812e7dc190ea804e9c6137 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:19:24 2013 +0200 Set Tor DataDir in startTor Tor's DataDir should be set before starting Tor so that descriptors are cached rather than downloaded each time. --- oonib/runner.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index a6b479f..edd1b6b 100644 --- a/oonib/runner.py +++ b/oonib/runner.… [View More]

1 0

[oonib/master] Clean up imports
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 834361018e6b35fd8db6be943aca30ba439eaae2 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jul 3 10:29:37 2013 +0200 Clean up imports --- oonib/report/file_collector.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 490ed77..c8f0a63 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -1,23 +1,17 @@ +import glob +import json +import os … [View More]

1 0

[oonib/master] Add Handlers and API for policy
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 5a349ee10d01b5cdb7c9eb256b58e78b310559ad Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 19:47:40 2013 +0200 Add Handlers and API for policy --- oonib/policy/api.py | 40 ++++++++++++++++++++++++ oonib/policy/handlers.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/oonib/policy/api.py b/oonib/policy/api.py new file mode 100644 index 0000000..577d988 --- /dev/null +++ b/oonib/policy/api.py @@ -0,0 +1,40 @@ +… [View More]""" +/report + +/pcap + +This is the async pcap reporting system. It requires the client to have created +a report already, but can work independently from test progress. + +""" +import random +import string +import json +import re +import os + +from twisted.internet import reactor, defer + +from cyclone import web + +from oonib import otime +from oonib import randomStr + +from oonib import config +from oonib.policy import DeckListHandler, InputListHandler, NetTestListHandler + +#XXX: if policy is configured +policyAPI = [ + (r"/deck", DeckListHandler), + (r"/input", InputListHandler), + (r"/deck/([a-z0-9]{40})$", DeckDescHandler), + (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.deck_dir}), + (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), + (r"/input([a-z0-9]{40}$", InputDescHandler), + (r"/policy/nettest", NetTestListHandler), + (r"/policy/nettest/([a-z0-9]+/py$", web.StaticFileHandler, {"path": + config.main.nettest_dir}), + (r"/policy/input", InputListHandler +] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py new file mode 100644 index 0000000..567da2c --- /dev/null +++ b/oonib/policy/handlers.py @@ -0,0 +1,78 @@ +from cyclone import web +from oonib import config +import json +import os +import yaml + +class DeckDescHandler(web.RequestHandler): + def get(self, deckID): + bn = os.path.basename(deckID) + try: + f = open(os.path.join(config.main.deck_dir, bn)) + a = {} + deckDesc = yaml.safe_load(f) + a['id'] = deckID + a['name'] = deckDesc['name'] + a['description'] = deckDesc['description'] + self.write(json.dumps(a)) + except IOError: + log.err("Deck %s missing" % deckID) + except KeyError: + log.err("Deck %s missing required keys!" % deckID) + +class DeckListHandler(web.RequestHandler): + def get(self): + if not config.main.deck_dir: return + path = os.path.abspath(config.main.deck_dir) + "/*" + decknames = map(os.path.basename, glob.iglob(path)) + deckList = [] + for deckname in decknames: + f = open(os.path.join(config.main.deck_dir, deckname)) + d = yaml.safe_load(f) + deckList.append({'id': deckname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(deckList)) + +class InputDescHandler(web.RequestHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, deckname)) + d = yaml.safe_load(f) + inputList.append({'id': inputname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(inputList)) + +class NetTestListHandler(web.RequestHandler): + def get(self): + #XXX: returns a list of accepted NetTests + pass + +class HelperListHandler(web.RequestHandler): + def get(self): + #XXX: get the list of the running handlers + #'id' + #'description' + #'address' + pass [View Less]

1 0

[oonib/master] Add Handlers and API for bouncer
by art＠torproject.org 11 Sep '13

11 Sep '13

commit bbe4e010381d1daf0710672f531c9fe99c9c93cf Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 19:48:06 2013 +0200 Add Handlers and API for bouncer --- oonib/bouncer/api.py | 8 ++++++++ oonib/bouncer/handlers.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/oonib/bouncer/api.py b/oonib/bouncer/api.py new file mode 100644 index 0000000..21f2be4 --- /dev/null +++ b/oonib/bouncer/api.py @@ -0,0 +1,8 @@ +from oonib.bouncer.handlers import * +#… [View More]

1 0

[oonib/master] Organize handlers and api for each component
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a33028d8e4708e14da7b23e0b5c2c87c60650d5e Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 20:08:43 2013 +0200 Organize handlers and api for each component --- oonib/deck/api.py | 10 +++++++ oonib/deck/handlers.py | 31 +++++++++++++++++++++ oonib/input/api.py | 10 +++++++ oonib/input/handlers.py | 35 +++++++++++++++++++++++ oonib/policy/api.py | 40 ++++----------------------- oonib/policy/handlers.py | 69 ++-----------------------------… [View More]--------------- 6 files changed, 95 insertions(+), 100 deletions(-) diff --git a/oonib/deck/api.py b/oonib/deck/api.py new file mode 100644 index 0000000..db6fb66 --- /dev/null +++ b/oonib/deck/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.deck import handlers +from oonib import config + +deckAPI = [ + (r"/deck", handlers.DeckListHandler), + (r"/deck/([a-z0-9]{40})$", handlers.DeckDescHandler), + (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.deck_dir}), +] diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py new file mode 100644 index 0000000..f37609a --- /dev/null +++ b/oonib/deck/handlers.py @@ -0,0 +1,31 @@ +class DeckDescHandler(web.RequestHandler): + def get(self, deckID): + bn = os.path.basename(deckID) + try: + f = open(os.path.join(config.main.deck_dir, bn)) + a = {} + deckDesc = yaml.safe_load(f) + a['id'] = deckID + a['name'] = deckDesc['name'] + a['description'] = deckDesc['description'] + self.write(json.dumps(a)) + except IOError: + log.err("Deck %s missing" % deckID) + except KeyError: + log.err("Deck %s missing required keys!" % deckID) + +class DeckListHandler(web.RequestHandler): + def get(self): + if not config.main.deck_dir: return + path = os.path.abspath(config.main.deck_dir) + "/*" + decknames = map(os.path.basename, glob.iglob(path)) + deckList = [] + for deckname in decknames: + f = open(os.path.join(config.main.deck_dir, deckname)) + d = yaml.safe_load(f) + deckList.append({'id': deckname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(deckList)) + + diff --git a/oonib/input/api.py b/oonib/input/api.py new file mode 100644 index 0000000..4c397ab --- /dev/null +++ b/oonib/input/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.input import handlers +from oonib import config + +inputAPI = [ + (r"/input", handlers.InputListHandler), + (r"/input/([a-z0-9]{40})", handlers.InputDescHandler), + (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), +] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py new file mode 100644 index 0000000..ed8008e --- /dev/null +++ b/oonib/input/handlers.py @@ -0,0 +1,35 @@ +from oonib import config +from cyclone import web +import json +import os +import yaml + +class InputDescHandler(web.RequestHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, deckname)) + d = yaml.safe_load(f) + inputList.append({'id': inputname,'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(inputList)) diff --git a/oonib/policy/api.py b/oonib/policy/api.py index 577d988..0f6f023 100644 --- a/oonib/policy/api.py +++ b/oonib/policy/api.py @@ -1,40 +1,12 @@ -""" -/report - -/pcap - -This is the async pcap reporting system. It requires the client to have created -a report already, but can work independently from test progress. - -""" -import random -import string -import json -import re -import os - -from twisted.internet import reactor, defer - from cyclone import web - -from oonib import otime -from oonib import randomStr - +from oonib.policy import handlers from oonib import config -from oonib.policy import DeckListHandler, InputListHandler, NetTestListHandler #XXX: if policy is configured policyAPI = [ - (r"/deck", DeckListHandler), - (r"/input", InputListHandler), - (r"/deck/([a-z0-9]{40})$", DeckDescHandler), - (r"/deck/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": - config.main.deck_dir}), - (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": - config.main.input_dir}), - (r"/input([a-z0-9]{40}$", InputDescHandler), - (r"/policy/nettest", NetTestListHandler), - (r"/policy/nettest/([a-z0-9]+/py$", web.StaticFileHandler, {"path": - config.main.nettest_dir}), - (r"/policy/input", InputListHandler + (r"/policy/nettest", handlers.NetTestPolicyHandler), + #XXX: add nettest handler + #(r"/policy/nettest/([a-z0-9]+)/py$", web.StaticFileHandler, {"path": + # config.main.nettest_dir}), + (r"/policy/input", handlers.InputPolicyHandler), ] diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 567da2c..2b55fb7 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -4,75 +4,12 @@ import json import os import yaml -class DeckDescHandler(web.RequestHandler): - def get(self, deckID): - bn = os.path.basename(deckID) - try: - f = open(os.path.join(config.main.deck_dir, bn)) - a = {} - deckDesc = yaml.safe_load(f) - a['id'] = deckID - a['name'] = deckDesc['name'] - a['description'] = deckDesc['description'] - self.write(json.dumps(a)) - except IOError: - log.err("Deck %s missing" % deckID) - except KeyError: - log.err("Deck %s missing required keys!" % deckID) - -class DeckListHandler(web.RequestHandler): - def get(self): - if not config.main.deck_dir: return - path = os.path.abspath(config.main.deck_dir) + "/*" - decknames = map(os.path.basename, glob.iglob(path)) - deckList = [] - for deckname in decknames: - f = open(os.path.join(config.main.deck_dir, deckname)) - d = yaml.safe_load(f) - deckList.append({'id': deckname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(deckList)) - -class InputDescHandler(web.RequestHandler): - def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec - bn = os.path.basename(inputID) + ".desc" - try: - f = open(os.path.join(config.main.input_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] - self.write(json.dumps(a)) - except Exception: - log.err("No Input Descriptor found for id %s" % inputID) - -class InputListHandler(web.RequestHandler): - def get(self): - if not config.main.input_dir: return - path = os.path.abspath(config.main.input_dir) + "/*" - inputnames = map(os.path.basename, glob.iglob(path)) - inputList = [] - for inputname in inputnames: - f = open(os.path.join(config.main.input_dir, deckname)) - d = yaml.safe_load(f) - inputList.append({'id': inputname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(inputList)) - -class NetTestListHandler(web.RequestHandler): +class NetTestPolicyHandler(web.RequestHandler): def get(self): #XXX: returns a list of accepted NetTests pass -class HelperListHandler(web.RequestHandler): +class InputPolicyHandler(web.RequestHandler): def get(self): - #XXX: get the list of the running handlers - #'id' - #'description' - #'address' pass + #XXX return list of input ids [View Less]

1 0

[oonib/master] Clean up report api and handlers
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 784e859ba1851e9221307b44b0543a7e0bc19426 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Jul 7 20:17:07 2013 +0200 Clean up report api and handlers --- oonib/__init__.py | 67 --------- oonib/report/__init__.py | 5 - oonib/report/api.py | 52 +------ oonib/report/file_collector.py | 303 ---------------------------------------- oonib/report/handlers.py | 292 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 296 insertions(+),… [View More] 423 deletions(-) diff --git a/oonib/__init__.py b/oonib/__init__.py deleted file mode 100644 index c374f53..0000000 --- a/oonib/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò, Isis Lovecruft -# :licence: see LICENSE for details -""" -In here we shall keep track of all variables and objects that should be -instantiated only once and be common to pieces of GLBackend code. -""" - -__version__ = '0.9.1' - -__all__ = ['Storage', 'randomStr'] - -import string -import random - -class Storage(dict): - """ - A Storage object is like a dictionary except `obj.foo` can be used - in addition to `obj['foo']`. - - >>> o = Storage(a=1) - >>> o.a - 1 - >>> o['a'] - 1 - >>> o.a = 2 - >>> o['a'] - 2 - >>> del o.a - >>> o.a - None - """ - def __getattr__(self, key): - try: - return self[key] - except KeyError, k: - return None - - def __setattr__(self, key, value): - self[key] = value - - def __delattr__(self, key): - try: - del self[key] - except KeyError, k: - raise AttributeError, k - - def __repr__(self): - return '<Storage ' + dict.__repr__(self) + '>' - - def __getstate__(self): - return dict(self) - - def __setstate__(self, value): - for (k, v) in value.items(): - self[k] = v - -def randomStr(length, num=True): - """ - Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) - string long length - """ - chars = string.ascii_lowercase + string.ascii_uppercase - if num: - chars += string.digits - return ''.join(random.choice(chars) for x in range(length)) diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py deleted file mode 100644 index fcbf220..0000000 --- a/oonib/report/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -class MissingField(Exception): - pass - -class InvalidRequestField(Exception): - pass diff --git a/oonib/report/api.py b/oonib/report/api.py index 8835784..c707291 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -7,54 +7,10 @@ This is the async pcap reporting system. It requires the client to have created a report already, but can work independently from test progress. """ -import random -import string -import json -import re -import os +from oonib.report import handlers -from twisted.internet import reactor, defer - -from cyclone import web - -from oonib import otime -from oonib import randomStr - -from oonib import config -from oonib.report import file_collector - -def parseUpdateReportRequest(request): - #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") - - # this is the regexp for the reports that include the timestamp - report_id_regexp = re.compile("[a-zA-Z0-9_-]+$") - - # XXX here we are actually parsing a json object that could be quite big. - # If we want this to scale properly we only want to look at the test_id - # field. - # We are also keeping in memory multiple copies of the same object. A lot - # of optimization can be done. - parsed_request = json.loads(request) - try: - report_id = parsed_request['report_id'] - except KeyError: - raise MissingField('report_id') - - if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') - - return parsed_request - -reportingBackendAPI = [ - (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), - (r"/report", file_collector.NewReportHandlerFile), +reportAPI = [ + (r"/report/([a-zA-Z0-9_\-]+)/close", handlers.CloseReportHandlerFile), + (r"/report", handlers.NewReportHandlerFile), (r"/pcap", file_collector.PCAPReportHandler), - (r"/deck", file_collector.DeckListHandler), - (r"/input", file_collector.InputListHandler), - (r"/deck/([a-z0-9]{40})$", web.StaticFileHandler, {"path": - config.main.deck_dir}), - (r"/input/([a-z0-9]{40})$", web.StaticFileHandler, {"path": - config.main.input_dir}), ] - -reportingBackend = web.Application(reportingBackendAPI, debug=True) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py deleted file mode 100644 index 57c4175..0000000 --- a/oonib/report/file_collector.py +++ /dev/null @@ -1,303 +0,0 @@ -import glob -import json -import os -import random -import re -import string -import time -import yaml - -from cyclone import web -from datetime import datetime -from oonib import randomStr, otime, config, log -from oonib.report import MissingField, InvalidRequestField -from twisted.internet import fdesc, reactor - -def parseUpdateReportRequest(request): - #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") - - # this is the regexp for the reports that include the timestamp - report_id_regexp = re.compile("[a-zA-Z0-9_\-]+$") - - # XXX here we are actually parsing a json object that could be quite big. - # If we want this to scale properly we only want to look at the test_id - # field. - # We are also keeping in memory multiple copies of the same object. A lot - # of optimization can be done. - parsed_request = json.loads(request) - try: - report_id = parsed_request['report_id'] - except KeyError: - raise MissingField('report_id') - - if not re.match(report_id_regexp, report_id): - raise InvalidRequestField('report_id') - - return parsed_request - -def parseNewReportRequest(request): - """ - Here we parse a new report request. - """ - version_string = re.compile("[0-9A-Za-z_\-\.]+$") - name = re.compile("[a-zA-Z0-9_\- ]+$") - probe_asn = re.compile("AS[0-9]+$") - - expected_request = { - 'software_name': name, - 'software_version': version_string, - 'test_name': name, - 'test_version': version_string, - 'probe_asn': probe_asn - } - - parsed_request = json.loads(request) - if not parsed_request['probe_asn']: - parsed_request['probe_asn'] = 'AS0' - - for k, regexp in expected_request.items(): - try: - value_to_check = parsed_request[k] - except KeyError: - raise MissingField(k) - - print "Matching %s with %s | %s" % (regexp, value_to_check, k) - if re.match(regexp, str(value_to_check)): - continue - else: - raise InvalidRequestField(k) - - return parsed_request - -class InvalidReportHeader(Exception): - pass - -class MissingReportHeaderKey(InvalidReportHeader): - pass - -def validate_report_header(report_header): - required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', - 'software_version', 'test_name', 'test_version'] - for key in required_keys: - if key not in report_header: - raise MissingReportHeaderKey(key) - - if report_header['probe_asn'] is None: - report_header['probe_asn'] = 'AS0' - - if not re.match('AS[0-9]+$', report_header['probe_asn']): - raise InvalidReportHeader('probe_asn') - - # If no country is known, set it to be ZZ (user assigned value in ISO 3166) - if report_header['probe_cc'] is None: - report_header['probe_cc'] = 'ZZ' - - if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): - raise InvalidReportHeader('probe_cc') - - if not re.match('[a-z_\-]+$', report_header['test_name']): - raise InvalidReportHeader('test_name') - - - if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): - raise InvalidReportHeader('test_version') - - return report_header - -def get_report_path(report_id): - return os.path.join(config.main.report_dir, report_id) - -def stale_check(report_id): - if (time.time() - config.reports[report_id]) > config.main.stale_time: - try: - close_report(report_id) - except ReportNotFound: - pass - -class NewReportHandlerFile(web.RequestHandler): - """ - Responsible for creating and updating reports by writing to flat file. - """ - - def post(self): - """ - Creates a new report with the input - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'probe_asn': 'XXX' - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - (not implemented, nor in client, nor in backend) - The idea behind these two fields is that it would be interesting to - also collect how the request was observed from the collectors point - of view. - - We use as a unique key the client_ip address and a time window. We - then need to tell the test_helper that is selected the client_ip - address and tell it to expect a connection from a probe in that time - window. - - Once the test_helper sees a connection from that client_ip it will - store for the testing session the data that it receives. - When the probe completes the report (or the time window is over) the - final report will include also the data collected from the - collectors view point. - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - # XXX here we should validate and sanitize the request - try: - report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField, e: - raise web.HTTPError(400, "Invalid Request Field %s" % e) - except MissingField, e: - raise web.HTTPError(400, "Missing Request Field %s" % e) - - print "Parsed this data %s" % report_data - software_name = report_data['software_name'] - software_version = report_data['software_version'] - test_name = report_data['test_name'] - test_version = report_data['test_version'] - probe_asn = report_data['probe_asn'] - content = yaml.safe_load(report_data['content']) - content['backend_version'] = config.backend_version - - try: - report_header = validate_report_header(content) - - except MissingReportHeaderKey, key: - raise web.HTTPError(406, "Missing report header key %s" % key) - - except InvalidReportHeader, key: - raise web.HTTPError(406, "Invalid report header %s" % key) - - report_header = yaml.dump(report_header) - content = "---\n" + report_header + '...\n' - - if not probe_asn: - probe_asn = "AS0" - - report_id = otime.timestamp() + '_' \ - + probe_asn + '_' \ - + randomStr(50) - - # The report filename contains the timestamp of the report plus a - # random nonce - report_filename = os.path.join(config.main.report_dir, report_id) - - response = {'backend_version': config.backend_version, - 'report_id': report_id - } - - config.reports[report_id] = time.time() - - reactor.callLater(config.main.stale_time, stale_check, report_id) - - self.writeToReport(report_filename, content) - - self.write(response) - - def writeToReport(self, report_filename, data): - with open(report_filename, 'w+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - - def put(self): - """ - Update an already existing report. - - { - 'report_id': 'XXX', - 'content': 'XXX' - } - """ - parsed_request = parseUpdateReportRequest(self.request.body) - - report_id = parsed_request['report_id'] - - log.debug("Got this request %s" % parsed_request) - report_filename = os.path.join(config.main.report_dir, - report_id) - - config.reports[report_id] = time.time() - reactor.callLater(config.main.stale_time, stale_check, report_id) - - self.updateReport(report_filename, parsed_request['content']) - - def updateReport(self, report_filename, data): - try: - with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - except IOError as e: - web.HTTPError(404, "Report not found") - -class ReportNotFound(Exception): - pass - -def close_report(report_id): - report_filename = get_report_path(report_id) - try: - with open(report_filename) as fd: - yaml_data = ''.join(fd.readline() for _ in range(12)) - report_details = yaml.safe_load(yaml_data) - except IOError: - raise ReportNotFound - - timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) - dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( - timestamp=timestamp, - **report_details) - - dst_path = os.path.join(config.main.archive_dir, - report_details['probe_cc']) - - if not os.path.isdir(dst_path): - os.mkdir(dst_path) - - dst_path = os.path.join(dst_path, dst_filename) - os.rename(report_filename, dst_path) - -class CloseReportHandlerFile(web.RequestHandler): - def get(self): - pass - - def post(self, report_id): - try: - close_report(report_id) - except ReportNotFound: - web.HTTPError(404, "Report not found") - -class PCAPReportHandler(web.RequestHandler): - def get(self): - pass - - def post(self): - pass - -class DeckListHandler(web.RequestHandler): - def get(self): - if not config.main.deck_dir: return - path = os.path.abspath(config.main.deck_dir) + "/*" - decknames = map(os.path.basename, glob.iglob(path)) - self.write(json.dumps(decknames)) - -class InputListHandler(web.RequestHandler): - def get(self): - if not config.main.input_dir: return - path = os.path.abspath(config.main.input_dir) + "/*" - inputnames = map(os.path.basename, glob.iglob(path)) - self.write(json.dumps(decknames)) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py new file mode 100644 index 0000000..c5e4a97 --- /dev/null +++ b/oonib/report/handlers.py @@ -0,0 +1,292 @@ +import json +import os +import re +import string +import time +import yaml + +from cyclone import web +from datetime import datetime +from oonib import randomStr, otime, config, log +from twisted.internet import fdesc, reactor + +class MissingField(Exception): + pass + +class InvalidRequestField(Exception): + pass + +def parseUpdateReportRequest(request): + #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") + + # this is the regexp for the reports that include the timestamp + report_id_regexp = re.compile("[a-zA-Z0-9_\-]+$") + + # XXX here we are actually parsing a json object that could be quite big. + # If we want this to scale properly we only want to look at the test_id + # field. + # We are also keeping in memory multiple copies of the same object. A lot + # of optimization can be done. + parsed_request = json.loads(request) + try: + report_id = parsed_request['report_id'] + except KeyError: + raise MissingField('report_id') + + if not re.match(report_id_regexp, report_id): + raise InvalidRequestField('report_id') + + return parsed_request + +def parseNewReportRequest(request): + """ + Here we parse a new report request. + """ + version_string = re.compile("[0-9A-Za-z_\-\.]+$") + name = re.compile("[a-zA-Z0-9_\- ]+$") + probe_asn = re.compile("AS[0-9]+$") + + expected_request = { + 'software_name': name, + 'software_version': version_string, + 'test_name': name, + 'test_version': version_string, + 'probe_asn': probe_asn + } + + parsed_request = json.loads(request) + if not parsed_request['probe_asn']: + parsed_request['probe_asn'] = 'AS0' + + for k, regexp in expected_request.items(): + try: + value_to_check = parsed_request[k] + except KeyError: + raise MissingField(k) + + print "Matching %s with %s | %s" % (regexp, value_to_check, k) + if re.match(regexp, str(value_to_check)): + continue + else: + raise InvalidRequestField(k) + + return parsed_request + +class InvalidReportHeader(Exception): + pass + +class MissingReportHeaderKey(InvalidReportHeader): + pass + +def validate_report_header(report_header): + required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', + 'software_version', 'test_name', 'test_version'] + for key in required_keys: + if key not in report_header: + raise MissingReportHeaderKey(key) + + if report_header['probe_asn'] is None: + report_header['probe_asn'] = 'AS0' + + if not re.match('AS[0-9]+$', report_header['probe_asn']): + raise InvalidReportHeader('probe_asn') + + # If no country is known, set it to be ZZ (user assigned value in ISO 3166) + if report_header['probe_cc'] is None: + report_header['probe_cc'] = 'ZZ' + + if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): + raise InvalidReportHeader('probe_cc') + + if not re.match('[a-z_\-]+$', report_header['test_name']): + raise InvalidReportHeader('test_name') + + + if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): + raise InvalidReportHeader('test_version') + + return report_header + +def get_report_path(report_id): + return os.path.join(config.main.report_dir, report_id) + +def stale_check(report_id): + if (time.time() - config.reports[report_id]) > config.main.stale_time: + try: + close_report(report_id) + except ReportNotFound: + pass + +class NewReportHandlerFile(web.RequestHandler): + """ + Responsible for creating and updating reports by writing to flat file. + """ + + def post(self): + """ + Creates a new report with the input + + * Request + + {'software_name': 'XXX', + 'software_version': 'XXX', + 'test_name': 'XXX', + 'test_version': 'XXX', + 'probe_asn': 'XXX' + 'content': 'XXX' + } + + Optional: + 'test_helper': 'XXX' + 'client_ip': 'XXX' + + (not implemented, nor in client, nor in backend) + The idea behind these two fields is that it would be interesting to + also collect how the request was observed from the collectors point + of view. + + We use as a unique key the client_ip address and a time window. We + then need to tell the test_helper that is selected the client_ip + address and tell it to expect a connection from a probe in that time + window. + + Once the test_helper sees a connection from that client_ip it will + store for the testing session the data that it receives. + When the probe completes the report (or the time window is over) the + final report will include also the data collected from the + collectors view point. + + * Response + + {'backend_version': 'XXX', 'report_id': 'XXX'} + + """ + # XXX here we should validate and sanitize the request + try: + report_data = parseNewReportRequest(self.request.body) + except InvalidRequestField, e: + raise web.HTTPError(400, "Invalid Request Field %s" % e) + except MissingField, e: + raise web.HTTPError(400, "Missing Request Field %s" % e) + + print "Parsed this data %s" % report_data + software_name = report_data['software_name'] + software_version = report_data['software_version'] + test_name = report_data['test_name'] + test_version = report_data['test_version'] + probe_asn = report_data['probe_asn'] + content = yaml.safe_load(report_data['content']) + content['backend_version'] = config.backend_version + + try: + report_header = validate_report_header(content) + + except MissingReportHeaderKey, key: + raise web.HTTPError(406, "Missing report header key %s" % key) + + except InvalidReportHeader, key: + raise web.HTTPError(406, "Invalid report header %s" % key) + + report_header = yaml.dump(report_header) + content = "---\n" + report_header + '...\n' + + if not probe_asn: + probe_asn = "AS0" + + report_id = otime.timestamp() + '_' \ + + probe_asn + '_' \ + + randomStr(50) + + # The report filename contains the timestamp of the report plus a + # random nonce + report_filename = os.path.join(config.main.report_dir, report_id) + + response = {'backend_version': config.backend_version, + 'report_id': report_id + } + + config.reports[report_id] = time.time() + + reactor.callLater(config.main.stale_time, stale_check, report_id) + + self.writeToReport(report_filename, content) + + self.write(response) + + def writeToReport(self, report_filename, data): + with open(report_filename, 'w+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + + def put(self): + """ + Update an already existing report. + + { + 'report_id': 'XXX', + 'content': 'XXX' + } + """ + parsed_request = parseUpdateReportRequest(self.request.body) + + report_id = parsed_request['report_id'] + + log.debug("Got this request %s" % parsed_request) + report_filename = os.path.join(config.main.report_dir, + report_id) + + config.reports[report_id] = time.time() + reactor.callLater(config.main.stale_time, stale_check, report_id) + + self.updateReport(report_filename, parsed_request['content']) + + def updateReport(self, report_filename, data): + try: + with open(report_filename, 'a+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + except IOError as e: + web.HTTPError(404, "Report not found") + +class ReportNotFound(Exception): + pass + +def close_report(report_id): + report_filename = get_report_path(report_id) + try: + with open(report_filename) as fd: + yaml_data = ''.join(fd.readline() for _ in range(12)) + report_details = yaml.safe_load(yaml_data) + except IOError: + raise ReportNotFound + + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + + dst_path = os.path.join(config.main.archive_dir, + report_details['probe_cc']) + + if not os.path.isdir(dst_path): + os.mkdir(dst_path) + + dst_path = os.path.join(dst_path, dst_filename) + os.rename(report_filename, dst_path) + +class CloseReportHandlerFile(web.RequestHandler): + def get(self): + pass + + def post(self, report_id): + try: + close_report(report_id) + except ReportNotFound: + web.HTTPError(404, "Report not found") + +class PCAPReportHandler(web.RequestHandler): + def get(self): + pass + + def post(self): + pass [View Less]

1 0