September 2013 - tor-commits - lists.torproject.org

r26348: {website} update the blog feed. (website/trunk/en)
by Andrew Lewman 11 Sep '13

11 Sep '13

Author: phobos Date: 2013-09-11 11:06:56 +0000 (Wed, 11 Sep 2013) New Revision: 26348 Modified: website/trunk/en/index.wml Log: update the blog feed. Modified: website/trunk/en/index.wml =================================================================== --- website/trunk/en/index.wml 2013-09-10 16:32:47 UTC (rev 26347) +++ website/trunk/en/index.wml 2013-09-11 11:06:56 UTC (rev 26348) @@ -157,26 +157,26 @@ <table> <tr> <td> - <div class="calendar"><span class="month">Sep</span><br><span class="day">05</span></div> - <p>How to handle millions of new Tor clients. Read the <a href="https://blog.torproject.org/blog/how-to-handle-millions-new-tor-clients">details</a>.</p> + <div class="calendar"><span class="month">Sep</span><br><span class="day">11</span></div> + <p>Tor, NSA, and QUICK ANT Speculation. Read <a href="https://blog.torproject.org/blog/tor-nsa-and-quick-ant-speculation">our thoughts</a>.</p> </td> </tr> <tr> <td> - <div class="calendar"><span class="month">Sep</span><br><span class="day">04</span></div> - <p><a href="https://blog.torproject.org/blog/tor-weekly-news-%E2%80%94-september-4th-20…">Tor Weekly News</a> as of 04 September.</p> + <div class="calendar"><span class="month">Sep</span><br><span class="day">11</span></div> + <p>Learn about the <a href="https://blog.torproject.org/blog/lifecycle-of-a-new-relay">lifecycle of a new relay</a>.</p> </td> </tr> <tr> <td> - <div class="calendar"><span class="month">Aug</span><br><span class="day">20</span></div> - <p>Deterministic Builds Part One: Cyberwar and Global Compromise. Read the <a href="https://blog.torproject.org/blog/deterministic-builds-part-one-cyberwar-and…">blog post</a>.</p> + <div class="calendar"><span class="month">Sep</span><br><span class="day">11</span></div> + <p>New Tor 0.2.4.17-rc packages. Learn <a href="https://blog.torproject.org/blog/new-tor-02417-rc-packages">what's new</a>.</p> </td> </tr> <tr> <td> - <div class="calendar"><span class="month">Aug</span><br><span class="day">19</span></div> - <p>Transparency, openness, and our 2012 financial docs. <a href="https://blog.torproject.org/blog/transparency-openness-and-our-2012-financi…">Read more</a>.</p> + <div class="calendar"><span class="month">Sep</span><br><span class="day">05</span></div> + <p>How to handle millions of new Tor clients. Read the <a href="https://blog.torproject.org/blog/how-to-handle-millions-new-tor-clients">details</a>.</p> </td> </tr> </table>

1 0

[oonib/master] Add support for updating a report in a more RESTful manner
by art＠torproject.org 11 Sep '13

11 Sep '13

commit d220907d31acae6c0a332a1fe00e9071a3a15083 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 18:05:45 2013 +0200 Add support for updating a report in a more RESTful manner --- oonib/report/api.py | 1 + oonib/report/file_collector.py | 75 +++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/oonib/report/api.py b/oonib/report/api.py index 7747a1e..e294b17 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -47,6 +47,7 @@ def parseUpdateReportRequest(request): reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), + (r"/report/([a-zA-Z0-9_\-]+)", file_collector.UpdateReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), (r"/pcap", file_collector.PCAPReportHandler) ] diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 490ed77..ae6e5d6 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -120,7 +120,46 @@ def stale_check(report_id): except ReportNotFound: pass -class NewReportHandlerFile(web.RequestHandler): +class UpdateReportHandlerFile(web.RequestHandler): + def writeToReport(self, report_filename, data): + with open(report_filename, 'w+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + + def put(self, report_id=None): + """ + Update an already existing report. + + { + 'report_id': 'XXX', + 'content': 'XXX' + } + """ + if not report_id: + parsed_request = parseUpdateReportRequest(self.request.body) + report_id = parsed_request['report_id'] + else: + parsed_request = json.loads(self.request.body) + + log.debug("Got this request %s" % parsed_request) + report_filename = os.path.join(config.main.report_dir, + report_id) + + config.reports[report_id] = time.time() + reactor.callLater(config.main.stale_time, stale_check, report_id) + + self.updateReport(report_filename, parsed_request['content']) + + def updateReport(self, report_filename, data): + try: + with open(report_filename, 'a+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + except IOError as e: + web.HTTPError(404, "Report not found") + + +class NewReportHandlerFile(UpdateReportHandlerFile): """ Responsible for creating and updating reports by writing to flat file. """ @@ -216,40 +255,6 @@ class NewReportHandlerFile(web.RequestHandler): self.write(response) - def writeToReport(self, report_filename, data): - with open(report_filename, 'w+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - - def put(self): - """ - Update an already existing report. - - { - 'report_id': 'XXX', - 'content': 'XXX' - } - """ - parsed_request = parseUpdateReportRequest(self.request.body) - - report_id = parsed_request['report_id'] - - log.debug("Got this request %s" % parsed_request) - report_filename = os.path.join(config.main.report_dir, - report_id) - - config.reports[report_id] = time.time() - reactor.callLater(config.main.stale_time, stale_check, report_id) - - self.updateReport(report_filename, parsed_request['content']) - - def updateReport(self, report_filename, data): - try: - with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - except IOError as e: - web.HTTPError(404, "Report not found") class ReportNotFound(Exception): pass

1 0

[oonib/master] Get the thing to actually start.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 683bbc695af313e1eba36bd3d9698eec17fca8cb Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 21:38:24 2013 +0200 Get the thing to actually start. Misc bugfixes and retested the requirements.txt --- oonib/__init__.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ oonib/api.py | 19 ++++---------- oonib/bouncer/handlers.py | 4 ++- oonib/deck/handlers.py | 2 +- oonib/handlers.py | 7 +++++ oonib/inputs/api.py | 2 +- oonib/inputs/handlers.py | 2 +- oonib/policy/handlers.py | 2 +- oonib/report/handlers.py | 4 +-- oonib/runner.py | 14 ++++------ requirements.txt | 20 +------------- 11 files changed, 89 insertions(+), 50 deletions(-) diff --git a/oonib/__init__.py b/oonib/__init__.py new file mode 100644 index 0000000..54d3006 --- /dev/null +++ b/oonib/__init__.py @@ -0,0 +1,63 @@ +""" +In here we shall keep track of all variables and objects that should be +instantiated only once and be common to pieces of GLBackend code. +""" + +__version__ = '0.9.1' + +__all__ = ['Storage', 'randomStr'] + +import string +import random + +class Storage(dict): + """ + A Storage object is like a dictionary except `obj.foo` can be used + in addition to `obj['foo']`. + + >>> o = Storage(a=1) + >>> o.a + 1 + >>> o['a'] + 1 + >>> o.a = 2 + >>> o['a'] + 2 + >>> del o.a + >>> o.a + None + """ + def __getattr__(self, key): + try: + return self[key] + except KeyError, k: + return None + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + try: + del self[key] + except KeyError, k: + raise AttributeError, k + + def __repr__(self): + return '<Storage ' + dict.__repr__(self) + '>' + + def __getstate__(self): + return dict(self) + + def __setstate__(self, value): + for (k, v) in value.items(): + self[k] = v + +def randomStr(length, num=True): + """ + Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) + string long length + """ + chars = string.ascii_lowercase + string.ascii_uppercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) diff --git a/oonib/api.py b/oonib/api.py index ce682fc..48c2e1e 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -1,31 +1,22 @@ -from cyclone import web - from oonib.deck.api import deckAPI +from oonib.report.api import reportAPI from oonib.inputs.api import inputsAPI from oonib.policy.api import policyAPI from oonib.bouncer.api import bouncerAPI from oonib import config -class OONIBHandler(web.RequestHandler): - pass - -class OONIBError(web.HTTPError): - pass - oonibAPI = [] oonibAPI += reportAPI -if config.inputs_dir: +if config.main.inputs_dir: oonibAPI += inputsAPI -if config.deck_dir: +if config.main.deck_dir: oonibAPI += deckAPI -if config.policy_file: +if config.main.policy_file: oonibAPI += policyAPI -if config.bouncer_file: +if config.main.bouncer_file: oonibAPI += bouncerAPI - - diff --git a/oonib/bouncer/__init__.py b/oonib/bouncer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index f2249dd..90651b0 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -1,4 +1,6 @@ -class BouncerQueryHandler(web.RequestHandler): +from oonib.handlers import OONIBHandler + +class BouncerQueryHandler(OONIBHandler): def get(self): #XXX unused pass diff --git a/oonib/deck/__init__.py b/oonib/deck/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 97b390a..edbe361 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,4 +1,4 @@ -from oonib.api import OONIBHandler +from oonib.handlers import OONIBHandler class DeckDescHandler(OONIBHandler): def get(self, deckID): diff --git a/oonib/handlers.py b/oonib/handlers.py new file mode 100644 index 0000000..62496ce --- /dev/null +++ b/oonib/handlers.py @@ -0,0 +1,7 @@ +from cyclone import web + +class OONIBHandler(web.RequestHandler): + pass + +class OONIBError(web.HTTPError): + pass diff --git a/oonib/inputs/__init__.py b/oonib/inputs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py index 53342ae..d74c23c 100644 --- a/oonib/inputs/api.py +++ b/oonib/inputs/api.py @@ -1,5 +1,5 @@ from cyclone import web -from oonib.input import handlers +from oonib.inputs import handlers from oonib import config inputsAPI = [ diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index 5391881..a1ed80d 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -1,4 +1,4 @@ -from oonib.api import OONIBHandler +from oonib.handlers import OONIBHandler from oonib import config diff --git a/oonib/policy/__init__.py b/oonib/policy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index ee6f089..7544cf9 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,4 +1,4 @@ -from oonib.api import OONIBHandler +from oonib.handlers import OONIBHandler from oonib import config import json diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 681d7d1..05949e6 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -3,7 +3,7 @@ import string import time import yaml -from oonib.api import OONIBHandler +from oonib.handlers import OONIBHandler from datetime import datetime from oonib import randomStr, otime, config, log @@ -12,8 +12,6 @@ from twisted.internet import fdesc, reactor class MissingField(Exception): pass -from oonib import randomStr -from oonib import otime class InvalidRequestField(Exception): pass diff --git a/oonib/runner.py b/oonib/runner.py index 3bb064e..10f2bef 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -21,10 +21,6 @@ from txtorcon import TCPHiddenServiceEndpoint, TorConfig from txtorcon import launch_tor from oonib.report.api import reportAPI -from oonib.deck.api import deckAPI -from oonib.inputs.api import inputsAPI -from oonib.policy.api import policyAPI -from oonib.bouncer.api import bouncerAPI from oonib.api import oonibAPI from oonib import oonibackend @@ -47,11 +43,6 @@ def setupCollector(tor_process_protocol): print("Exposed collector Tor hidden service on httpo://%s" % port.onion_uri) - tempfile.tempdir = os.path.join(_repo_dir, 'tmp') - if not os.path.isdir(tempfile.gettempdir()): - os.makedirs(tempfile.gettempdir()) - _temp_dir = tempfile.mkdtemp() - torconfig = TorConfig(tor_process_protocol.tor_protocol) public_port = 80 # XXX there is currently a bug in txtorcon that prevents data_dir from @@ -70,6 +61,11 @@ def startTor(): def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) + tempfile.tempdir = os.path.join(_repo_dir, 'tmp') + if not os.path.isdir(tempfile.gettempdir()): + os.makedirs(tempfile.gettempdir()) + _temp_dir = tempfile.mkdtemp() + torconfig = TorConfig() torconfig.SocksPort = config.main.socks_port if config.main.tor2webmode: diff --git a/requirements.txt b/requirements.txt index 08fd114..d7921fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,4 @@ --i https://pypi.python.org/packages -## -## The above line replaces pip's default package index, and this line is -## removed from the .requirements.travis file, due to its incompatibility -## with the '--use-mirrors' option that TravisCI recommends. -## -## The URLs in this file which have been commented out are due to a design -## problem in pip-1.3.0, where it ships with a default package index of -## 'https://pypi.python.org/simple', which though *it* was over SSL as -## promised, it still crawled the links on a module's PyPI page, which contains -## arbitrary links, such as project homepages, created by the PyPI maintainer -## of the module. see -## -## https://github.com/TheTorProject/ooni-backend/pull/1#discussion_r4084881 for -## a detailed description of the problem, and why these links may still be -## useful until pip is updated again. -## -PyYAML>=3.10 -#https://pypi.python.org/packages/source/P/PyYAML/PyYAML-3.10.tar.gz#md5=74c94a383886519e9e7b3dd1ee540247#egg=PyYAML +PyYaml Twisted>=12.2.0 #https://pypi.python.org/packages/source/T/Twisted/Twisted-13.0.0.tar.bz2#md5=68afff4e1efd3757d934e39f70c99f57#egg=Twisted cyclone>=1.1

1 0

[oonib/master] Merge branch 'master' of github.com:TheTorProject/ooni-backend
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 02b6747405a89e05d91336431f1bd9c1939f342d Merge: d220907 3f0f8e2 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 18:06:04 2013 +0200 Merge branch 'master' of github.com:TheTorProject/ooni-backend * 'master' of github.com:TheTorProject/ooni-backend: Fixes a bug where logfile config option was unused. adds libsqlite3-dev installation to Vagrantfile Vagrantfile | 2 +- oonib/log.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-)

1 0

[oonib/master] Add skeleton of the various new directories for bouncer/policy/inputs/decks
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a9b6dd9f9a8f8a51cdedecbb17351802934b10df Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 19:50:05 2013 +0200 Add skeleton of the various new directories for bouncer/policy/inputs/decks --- data/bouncer.yaml | 3 +++ data/policy.yaml | 15 +++++++++++++++ oonib.conf.example | 12 ++++++------ oonib/api.py | 31 +++++++++++++++++++++++++++++++ oonib/deck/handlers.py | 6 +++--- oonib/inputs/handlers.py | 15 ++++++++++----- oonib/policy/handlers.py | 15 ++++++++++----- oonib/report/handlers.py | 21 +++++++++++---------- oonib/runner.py | 18 ++---------------- 9 files changed, 91 insertions(+), 45 deletions(-) diff --git a/data/bouncer.yaml b/data/bouncer.yaml new file mode 100644 index 0000000..ba54898 --- /dev/null +++ b/data/bouncer.yaml @@ -0,0 +1,3 @@ +collector: + httpo://nkvphnp3p6agi5qq.onion: + test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'} diff --git a/data/policy.yaml b/data/policy.yaml new file mode 100644 index 0000000..fecdf29 --- /dev/null +++ b/data/policy.yaml @@ -0,0 +1,15 @@ +inputs: +- {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1} +- {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290} +nettest: +- {name: dnsconsistency, version: 0.5} +- {name: http_requests, version: 0.2.3} +- {name: tcpconnect, version: 0.1} +- {name: captiveportal, version: 0.2} +- {name: daphne, version: 0.1} +- {name: dnsspoof, version: 0.2} +- {name: http_body, version: 0.0.1} +- {name: http_header_field_manipulation, version: 0.1.3} +- {name: http_host, version: 0.2.3} +- {name: http_invalid_request_line, version: 0.1.4} +- {name: traceroute, version: 0.1.1} diff --git a/oonib.conf.example b/oonib.conf.example index 60e177f..a1bba84 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,10 +1,10 @@ main: - report_dir: Null - archive_dir: Null - inputs_dir: Null - deck_dir: Null - policy_file: Null - bouncer_file: Null + report_dir: data/reports/ + archive_dir: data/archive/ + inputs_dir: data/inputs/ + deck_dir: data/decks/ + policy_file: data/policy.yaml + bouncer_file: data/bouncer.yaml logfile: Null tor_datadir: Null diff --git a/oonib/api.py b/oonib/api.py new file mode 100644 index 0000000..ce682fc --- /dev/null +++ b/oonib/api.py @@ -0,0 +1,31 @@ +from cyclone import web + +from oonib.deck.api import deckAPI +from oonib.inputs.api import inputsAPI +from oonib.policy.api import policyAPI +from oonib.bouncer.api import bouncerAPI + +from oonib import config + +class OONIBHandler(web.RequestHandler): + pass + +class OONIBError(web.HTTPError): + pass + +oonibAPI = [] +oonibAPI += reportAPI + +if config.inputs_dir: + oonibAPI += inputsAPI + +if config.deck_dir: + oonibAPI += deckAPI + +if config.policy_file: + oonibAPI += policyAPI + +if config.bouncer_file: + oonibAPI += bouncerAPI + + diff --git a/oonib/data/.gitignore b/oonib/data/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index 511b6b1..97b390a 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,6 +1,6 @@ -from cyclone import web +from oonib.api import OONIBHandler -class DeckDescHandler(web.RequestHandler): +class DeckDescHandler(OONIBHandler): def get(self, deckID): bn = os.path.basename(deckID) try: @@ -16,7 +16,7 @@ class DeckDescHandler(web.RequestHandler): except KeyError: log.err("Deck %s missing required keys!" % deckID) -class DeckListHandler(web.RequestHandler): +class DeckListHandler(OONIBHandler): def get(self): if not config.main.deck_dir: return path = os.path.abspath(config.main.deck_dir) + "/*" diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index 722a757..5391881 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -1,10 +1,12 @@ +from oonib.api import OONIBHandler + from oonib import config -from cyclone import web + import json import os import yaml -class InputsDescHandler(web.RequestHandler): +class InputsDescHandler(OONIBHandler): def get(self, inputID): #XXX return the input descriptor # see oonib.md in ooni-spec @@ -20,7 +22,7 @@ class InputsDescHandler(web.RequestHandler): except Exception: log.err("No Input Descriptor found for id %s" % inputID) -class InputsListHandler(web.RequestHandler): +class InputsListHandler(OONIBHandler): def get(self): if not config.main.input_dir: return path = os.path.abspath(config.main.input_dir) + "/*" @@ -29,7 +31,10 @@ class InputsListHandler(web.RequestHandler): for inputname in inputnames: f = open(os.path.join(config.main.input_dir, deckname)) d = yaml.safe_load(f) - inputList.append({'id': inputname, 'name': d['name'], - 'description': d['description']}) + inputList.append({ + 'id': inputname, + 'name': d['name'], + 'description': d['description'] + }) f.close() self.write(json.dumps(inputList)) diff --git a/oonib/policy/handlers.py b/oonib/policy/handlers.py index 2b55fb7..ee6f089 100644 --- a/oonib/policy/handlers.py +++ b/oonib/policy/handlers.py @@ -1,15 +1,20 @@ -from cyclone import web +from oonib.api import OONIBHandler + from oonib import config import json import os import yaml -class NetTestPolicyHandler(web.RequestHandler): +class NetTestPolicyHandler(OONIBHandler): def get(self): - #XXX: returns a list of accepted NetTests + """ + returns a list of accepted NetTests + """ pass -class InputPolicyHandler(web.RequestHandler): +class InputPolicyHandler(OONIBHandler): def get(self): + """ + return list of input ids + """ pass - #XXX return list of input ids diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index c5e4a97..b79898c 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -5,7 +5,8 @@ import string import time import yaml -from cyclone import web +from oonib.api import OONIBHandler + from datetime import datetime from oonib import randomStr, otime, config, log from twisted.internet import fdesc, reactor @@ -117,7 +118,7 @@ def stale_check(report_id): except ReportNotFound: pass -class NewReportHandlerFile(web.RequestHandler): +class NewReportHandlerFile(OONIBHandler): """ Responsible for creating and updating reports by writing to flat file. """ @@ -165,9 +166,9 @@ class NewReportHandlerFile(web.RequestHandler): try: report_data = parseNewReportRequest(self.request.body) except InvalidRequestField, e: - raise web.HTTPError(400, "Invalid Request Field %s" % e) + raise OONIBError(400, "Invalid Request Field %s" % e) except MissingField, e: - raise web.HTTPError(400, "Missing Request Field %s" % e) + raise OONIBError(400, "Missing Request Field %s" % e) print "Parsed this data %s" % report_data software_name = report_data['software_name'] @@ -182,10 +183,10 @@ class NewReportHandlerFile(web.RequestHandler): report_header = validate_report_header(content) except MissingReportHeaderKey, key: - raise web.HTTPError(406, "Missing report header key %s" % key) + raise OONIBError(406, "Missing report header key %s" % key) except InvalidReportHeader, key: - raise web.HTTPError(406, "Invalid report header %s" % key) + raise OONIBError(406, "Invalid report header %s" % key) report_header = yaml.dump(report_header) content = "---\n" + report_header + '...\n' @@ -246,7 +247,7 @@ class NewReportHandlerFile(web.RequestHandler): fdesc.setNonBlocking(fd.fileno()) fdesc.writeToFD(fd.fileno(), data) except IOError as e: - web.HTTPError(404, "Report not found") + OONIBError(404, "Report not found") class ReportNotFound(Exception): pass @@ -274,7 +275,7 @@ def close_report(report_id): dst_path = os.path.join(dst_path, dst_filename) os.rename(report_filename, dst_path) -class CloseReportHandlerFile(web.RequestHandler): +class CloseReportHandlerFile(OONIBHandler): def get(self): pass @@ -282,9 +283,9 @@ class CloseReportHandlerFile(web.RequestHandler): try: close_report(report_id) except ReportNotFound: - web.HTTPError(404, "Report not found") + OONIBError(404, "Report not found") -class PCAPReportHandler(web.RequestHandler): +class PCAPReportHandler(OONIBHandler): def get(self): pass diff --git a/oonib/runner.py b/oonib/runner.py index 8e89f32..3bb064e 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -25,6 +25,7 @@ from oonib.deck.api import deckAPI from oonib.inputs.api import inputsAPI from oonib.policy.api import policyAPI from oonib.bouncer.api import bouncerAPI +from oonib.api import oonibAPI from oonib import oonibackend from oonib import config @@ -61,22 +62,7 @@ def setupCollector(tor_process_protocol): #XXX: also set up a separate keyed hidden service for collectors to push their status to, if the bouncer is enabled hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, data_dir=datadir) - enabledAPIs = [] - enabledAPIs += reportAPI - - if config.inputs_dir: - enabledAPIs += inputsAPI - - if config.deck_dir: - enabledAPIs += deckAPI - - if config.policy_file: - enabledAPIs += policyAPI - - if config.bouncer_file: - enabledAPIs += bouncerAPI - - hidden_service = hs_endpoint.listen(enabledAPIs) + hidden_service = hs_endpoint.listen(oonibAPI) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed)

1 0

[oonib/master] Merge branch 'master' into feature/add_deck_api
by art＠torproject.org 11 Sep '13

11 Sep '13

commit b0a56e5113f35a4a9b8a7f07ac3a3de2b07e0c9a Merge: a9b6dd9 02b6747 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 19:59:26 2013 +0200 Merge branch 'master' into feature/add_deck_api * master: Add support for updating a report in a more RESTful manner Conflicts: oonib/report/api.py oonib/report/handlers.py oonib/report/handlers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --cc oonib/report/handlers.py index b79898c,0000000..681d7d1 mode 100644,000000..100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@@ -1,293 -1,0 +1,293 @@@ - import json - import os - import re ++import random +import string +import time +import yaml + +from oonib.api import OONIBHandler + +from datetime import datetime +from oonib import randomStr, otime, config, log +from twisted.internet import fdesc, reactor + +class MissingField(Exception): + pass + ++from oonib import randomStr ++from oonib import otime +class InvalidRequestField(Exception): + pass + +def parseUpdateReportRequest(request): + #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") + + # this is the regexp for the reports that include the timestamp + report_id_regexp = re.compile("[a-zA-Z0-9_\-]+$") + + # XXX here we are actually parsing a json object that could be quite big. + # If we want this to scale properly we only want to look at the test_id + # field. + # We are also keeping in memory multiple copies of the same object. A lot + # of optimization can be done. + parsed_request = json.loads(request) + try: + report_id = parsed_request['report_id'] + except KeyError: + raise MissingField('report_id') + + if not re.match(report_id_regexp, report_id): + raise InvalidRequestField('report_id') + + return parsed_request + +def parseNewReportRequest(request): + """ + Here we parse a new report request. + """ + version_string = re.compile("[0-9A-Za-z_\-\.]+$") + name = re.compile("[a-zA-Z0-9_\- ]+$") + probe_asn = re.compile("AS[0-9]+$") + + expected_request = { + 'software_name': name, + 'software_version': version_string, + 'test_name': name, + 'test_version': version_string, + 'probe_asn': probe_asn + } + + parsed_request = json.loads(request) + if not parsed_request['probe_asn']: + parsed_request['probe_asn'] = 'AS0' + + for k, regexp in expected_request.items(): + try: + value_to_check = parsed_request[k] + except KeyError: + raise MissingField(k) + + print "Matching %s with %s | %s" % (regexp, value_to_check, k) + if re.match(regexp, str(value_to_check)): + continue + else: + raise InvalidRequestField(k) + + return parsed_request + +class InvalidReportHeader(Exception): + pass + +class MissingReportHeaderKey(InvalidReportHeader): + pass + +def validate_report_header(report_header): + required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', + 'software_version', 'test_name', 'test_version'] + for key in required_keys: + if key not in report_header: + raise MissingReportHeaderKey(key) + + if report_header['probe_asn'] is None: + report_header['probe_asn'] = 'AS0' + + if not re.match('AS[0-9]+$', report_header['probe_asn']): + raise InvalidReportHeader('probe_asn') + + # If no country is known, set it to be ZZ (user assigned value in ISO 3166) + if report_header['probe_cc'] is None: + report_header['probe_cc'] = 'ZZ' + + if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): + raise InvalidReportHeader('probe_cc') + + if not re.match('[a-z_\-]+$', report_header['test_name']): + raise InvalidReportHeader('test_name') + + + if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): + raise InvalidReportHeader('test_version') + + return report_header + +def get_report_path(report_id): + return os.path.join(config.main.report_dir, report_id) + +def stale_check(report_id): + if (time.time() - config.reports[report_id]) > config.main.stale_time: + try: + close_report(report_id) + except ReportNotFound: + pass + +class NewReportHandlerFile(OONIBHandler): + """ + Responsible for creating and updating reports by writing to flat file. + """ + + def post(self): + """ + Creates a new report with the input + + * Request + + {'software_name': 'XXX', + 'software_version': 'XXX', + 'test_name': 'XXX', + 'test_version': 'XXX', + 'probe_asn': 'XXX' + 'content': 'XXX' + } + + Optional: + 'test_helper': 'XXX' + 'client_ip': 'XXX' + + (not implemented, nor in client, nor in backend) + The idea behind these two fields is that it would be interesting to + also collect how the request was observed from the collectors point + of view. + + We use as a unique key the client_ip address and a time window. We + then need to tell the test_helper that is selected the client_ip + address and tell it to expect a connection from a probe in that time + window. + + Once the test_helper sees a connection from that client_ip it will + store for the testing session the data that it receives. + When the probe completes the report (or the time window is over) the + final report will include also the data collected from the + collectors view point. + + * Response + + {'backend_version': 'XXX', 'report_id': 'XXX'} + + """ + # XXX here we should validate and sanitize the request + try: + report_data = parseNewReportRequest(self.request.body) + except InvalidRequestField, e: + raise OONIBError(400, "Invalid Request Field %s" % e) + except MissingField, e: + raise OONIBError(400, "Missing Request Field %s" % e) + + print "Parsed this data %s" % report_data + software_name = report_data['software_name'] + software_version = report_data['software_version'] + test_name = report_data['test_name'] + test_version = report_data['test_version'] + probe_asn = report_data['probe_asn'] + content = yaml.safe_load(report_data['content']) + content['backend_version'] = config.backend_version + + try: + report_header = validate_report_header(content) + + except MissingReportHeaderKey, key: + raise OONIBError(406, "Missing report header key %s" % key) + + except InvalidReportHeader, key: + raise OONIBError(406, "Invalid report header %s" % key) + + report_header = yaml.dump(report_header) + content = "---\n" + report_header + '...\n' + + if not probe_asn: + probe_asn = "AS0" + + report_id = otime.timestamp() + '_' \ + + probe_asn + '_' \ + + randomStr(50) + + # The report filename contains the timestamp of the report plus a + # random nonce + report_filename = os.path.join(config.main.report_dir, report_id) + + response = {'backend_version': config.backend_version, + 'report_id': report_id + } + + config.reports[report_id] = time.time() + + reactor.callLater(config.main.stale_time, stale_check, report_id) + + self.writeToReport(report_filename, content) + + self.write(response) + + def writeToReport(self, report_filename, data): + with open(report_filename, 'w+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + + def put(self): + """ + Update an already existing report. + + { + 'report_id': 'XXX', + 'content': 'XXX' + } + """ + parsed_request = parseUpdateReportRequest(self.request.body) + + report_id = parsed_request['report_id'] + + log.debug("Got this request %s" % parsed_request) + report_filename = os.path.join(config.main.report_dir, + report_id) + + config.reports[report_id] = time.time() + reactor.callLater(config.main.stale_time, stale_check, report_id) + + self.updateReport(report_filename, parsed_request['content']) + + def updateReport(self, report_filename, data): + try: + with open(report_filename, 'a+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) + except IOError as e: + OONIBError(404, "Report not found") + +class ReportNotFound(Exception): + pass + +def close_report(report_id): + report_filename = get_report_path(report_id) + try: + with open(report_filename) as fd: + yaml_data = ''.join(fd.readline() for _ in range(12)) + report_details = yaml.safe_load(yaml_data) + except IOError: + raise ReportNotFound + + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + + dst_path = os.path.join(config.main.archive_dir, + report_details['probe_cc']) + + if not os.path.isdir(dst_path): + os.mkdir(dst_path) + + dst_path = os.path.join(dst_path, dst_filename) + os.rename(report_filename, dst_path) + +class CloseReportHandlerFile(OONIBHandler): + def get(self): + pass + + def post(self, report_id): + try: + close_report(report_id) + except ReportNotFound: + OONIBError(404, "Report not found") + +class PCAPReportHandler(OONIBHandler): + def get(self): + pass + + def post(self): + pass

1 0

[oonib/master] Do some refactoring and cleaning up of the various handlers.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit a4f09650c55aec62b0de0e319b3336547b5f2a90 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 19:17:14 2013 +0200 Do some refactoring and cleaning up of the various handlers. @aagbsn you should take a look at this. --- oonib.conf.example | 5 ++++- oonib/bouncer/api.py | 6 ++---- oonib/bouncer/handlers.py | 10 ---------- oonib/deck/handlers.py | 2 ++ oonib/input/api.py | 10 ---------- oonib/input/handlers.py | 35 ----------------------------------- oonib/inputs/api.py | 10 ++++++++++ oonib/inputs/handlers.py | 35 +++++++++++++++++++++++++++++++++++ oonib/policy/api.py | 2 -- oonib/report/api.py | 11 +---------- oonib/runner.py | 22 ++++++++++++++++++---- oonib/testhelpers/dns_helpers.py | 1 - 12 files changed, 72 insertions(+), 77 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 1b51621..60e177f 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,8 +1,11 @@ main: report_dir: Null archive_dir: Null - input_dir: Null + inputs_dir: Null deck_dir: Null + policy_file: Null + bouncer_file: Null + logfile: Null tor_datadir: Null database_uri: 'sqlite://oonib_test_db.db' diff --git a/oonib/bouncer/api.py b/oonib/bouncer/api.py index 21f2be4..42f53e1 100644 --- a/oonib/bouncer/api.py +++ b/oonib/bouncer/api.py @@ -1,8 +1,6 @@ -from oonib.bouncer.handlers import * +from oonib.bouncer import handlers #XXX: if bouncer is configured bouncerAPI = [ #return a collector and helper of the requested type - (r"/bouncer", BouncerQueryHandler), - #XXX: register a collector or helper - (r"/register", CollectorRegisterHandler), + (r"/bouncer", handlers.BouncerQueryHandler), ] diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index 74d828a..f2249dd 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -7,13 +7,3 @@ class BouncerQueryHandler(web.RequestHandler): pass # request.get? #'collector': {'foo.onion': {'helper-name': 'helper-address'}, 'bar.onion': {'helper-name': 'helper-address'}} - -class CollectorRegisterHandler(self): - def post(self): - #XXX: update the list of collectors to query - pass - - def get(self): - pass - #XXX unused - diff --git a/oonib/deck/handlers.py b/oonib/deck/handlers.py index f37609a..511b6b1 100644 --- a/oonib/deck/handlers.py +++ b/oonib/deck/handlers.py @@ -1,3 +1,5 @@ +from cyclone import web + class DeckDescHandler(web.RequestHandler): def get(self, deckID): bn = os.path.basename(deckID) diff --git a/oonib/input/api.py b/oonib/input/api.py deleted file mode 100644 index 4c397ab..0000000 --- a/oonib/input/api.py +++ /dev/null @@ -1,10 +0,0 @@ -from cyclone import web -from oonib.input import handlers -from oonib import config - -inputAPI = [ - (r"/input", handlers.InputListHandler), - (r"/input/([a-z0-9]{40})", handlers.InputDescHandler), - (r"/input/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": - config.main.input_dir}), -] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py deleted file mode 100644 index ed8008e..0000000 --- a/oonib/input/handlers.py +++ /dev/null @@ -1,35 +0,0 @@ -from oonib import config -from cyclone import web -import json -import os -import yaml - -class InputDescHandler(web.RequestHandler): - def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec - bn = os.path.basename(inputID) + ".desc" - try: - f = open(os.path.join(config.main.input_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] - self.write(json.dumps(a)) - except Exception: - log.err("No Input Descriptor found for id %s" % inputID) - -class InputListHandler(web.RequestHandler): - def get(self): - if not config.main.input_dir: return - path = os.path.abspath(config.main.input_dir) + "/*" - inputnames = map(os.path.basename, glob.iglob(path)) - inputList = [] - for inputname in inputnames: - f = open(os.path.join(config.main.input_dir, deckname)) - d = yaml.safe_load(f) - inputList.append({'id': inputname,'name': d['name'], - 'description': d['description']}) - f.close() - self.write(json.dumps(inputList)) diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py new file mode 100644 index 0000000..53342ae --- /dev/null +++ b/oonib/inputs/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.input import handlers +from oonib import config + +inputsAPI = [ + (r"/inputs", handlers.InputsListHandler), + (r"/inputs/([a-z0-9]{40})", handlers.InputsDescHandler), + (r"/inputs/([a-z0-9]{40})/file$", web.StaticFileHandler, {"path": + config.main.inputs_dir}), +] diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py new file mode 100644 index 0000000..722a757 --- /dev/null +++ b/oonib/inputs/handlers.py @@ -0,0 +1,35 @@ +from oonib import config +from cyclone import web +import json +import os +import yaml + +class InputsDescHandler(web.RequestHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputsListHandler(web.RequestHandler): + def get(self): + if not config.main.input_dir: return + path = os.path.abspath(config.main.input_dir) + "/*" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, deckname)) + d = yaml.safe_load(f) + inputList.append({'id': inputname, 'name': d['name'], + 'description': d['description']}) + f.close() + self.write(json.dumps(inputList)) diff --git a/oonib/policy/api.py b/oonib/policy/api.py index 0f6f023..4cacea2 100644 --- a/oonib/policy/api.py +++ b/oonib/policy/api.py @@ -1,6 +1,4 @@ -from cyclone import web from oonib.policy import handlers -from oonib import config #XXX: if policy is configured policyAPI = [ diff --git a/oonib/report/api.py b/oonib/report/api.py index c707291..e8a4c0c 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -1,16 +1,7 @@ -""" -/report - -/pcap - -This is the async pcap reporting system. It requires the client to have created -a report already, but can work independently from test progress. - -""" from oonib.report import handlers reportAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", handlers.CloseReportHandlerFile), (r"/report", handlers.NewReportHandlerFile), - (r"/pcap", file_collector.PCAPReportHandler), + (r"/pcap", handlers.PCAPReportHandler), ] diff --git a/oonib/runner.py b/oonib/runner.py index cfc0563..8e89f32 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -22,8 +22,9 @@ from txtorcon import launch_tor from oonib.report.api import reportAPI from oonib.deck.api import deckAPI -from oonib.input.api import inputAPI +from oonib.inputs.api import inputsAPI from oonib.policy.api import policyAPI +from oonib.bouncer.api import bouncerAPI from oonib import oonibackend from oonib import config @@ -60,9 +61,22 @@ def setupCollector(tor_process_protocol): #XXX: also set up a separate keyed hidden service for collectors to push their status to, if the bouncer is enabled hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, data_dir=datadir) - EnabledAPIs = [] - EnabledAPIs.append(reportAPI) - hidden_service = hs_endpoint.listen(EnabledAPIs) + enabledAPIs = [] + enabledAPIs += reportAPI + + if config.inputs_dir: + enabledAPIs += inputsAPI + + if config.deck_dir: + enabledAPIs += deckAPI + + if config.policy_file: + enabledAPIs += policyAPI + + if config.bouncer_file: + enabledAPIs += bouncerAPI + + hidden_service = hs_endpoint.listen(enabledAPIs) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed) diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py index cb4ff9f..dc9c772 100644 --- a/oonib/testhelpers/dns_helpers.py +++ b/oonib/testhelpers/dns_helpers.py @@ -12,5 +12,4 @@ class DNSTestHelper(server.DNSServerFactory): caches = caches, clients = [resolver], verbose = verbose) def handleQuery(self, message, protocol, address): - print message, protocol, address server.DNSServerFactory.handleQuery(self, message, protocol, address)

1 0

[oonib/master] Add support for exposing OONIB not over Tor (useful for debugging)
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 86aa9ff669b567c5677ba1174c6c07bb9dcc6cc2 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 16 21:52:31 2013 +0200 Add support for exposing OONIB not over Tor (useful for debugging) Test the input list API. --- oonib.conf.example | 2 ++ oonib/api.py | 18 ++++++++++++------ oonib/inputs/handlers.py | 17 +++++++++-------- oonib/runner.py | 14 ++++++++------ 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index a1bba84..36df0e3 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -28,6 +28,8 @@ main: debug: false stale_time: 3600 + tor_hidden_service: true + helpers: http_return_request: port: 57001 diff --git a/oonib/api.py b/oonib/api.py index 48c2e1e..775c4ad 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -1,3 +1,5 @@ +from cyclone import web + from oonib.deck.api import deckAPI from oonib.report.api import reportAPI from oonib.inputs.api import inputsAPI @@ -6,17 +8,21 @@ from oonib.bouncer.api import bouncerAPI from oonib import config -oonibAPI = [] -oonibAPI += reportAPI +ooniBackendAPI = [] +ooniBackendAPI += reportAPI if config.main.inputs_dir: - oonibAPI += inputsAPI + ooniBackendAPI += inputsAPI if config.main.deck_dir: - oonibAPI += deckAPI + ooniBackendAPI += deckAPI if config.main.policy_file: - oonibAPI += policyAPI + ooniBackendAPI += policyAPI if config.main.bouncer_file: - oonibAPI += bouncerAPI + ooniBackendAPI += bouncerAPI + +print ooniBackendAPI + +ooniBackend = web.Application(ooniBackendAPI, debug=True) diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py index a1ed80d..cf056ba 100644 --- a/oonib/inputs/handlers.py +++ b/oonib/inputs/handlers.py @@ -1,18 +1,20 @@ -from oonib.handlers import OONIBHandler - -from oonib import config - +import glob import json import os + import yaml +from oonib.handlers import OONIBHandler + +from oonib import config + class InputsDescHandler(OONIBHandler): def get(self, inputID): #XXX return the input descriptor # see oonib.md in ooni-spec bn = os.path.basename(inputID) + ".desc" try: - f = open(os.path.join(config.main.input_dir, bn)) + f = open(os.path.join(config.main.inputs_dir, bn)) a = {} inputDesc = yaml.safe_load(f) a['id'] = inputID @@ -24,12 +26,11 @@ class InputsDescHandler(OONIBHandler): class InputsListHandler(OONIBHandler): def get(self): - if not config.main.input_dir: return - path = os.path.abspath(config.main.input_dir) + "/*" + path = os.path.abspath(config.main.inputs_dir) + "/*" inputnames = map(os.path.basename, glob.iglob(path)) inputList = [] for inputname in inputnames: - f = open(os.path.join(config.main.input_dir, deckname)) + f = open(os.path.join(config.main.inputs_dir, inputname)) d = yaml.safe_load(f) inputList.append({ 'id': inputname, diff --git a/oonib/runner.py b/oonib/runner.py index 10f2bef..294f414 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -21,7 +21,7 @@ from txtorcon import TCPHiddenServiceEndpoint, TorConfig from txtorcon import launch_tor from oonib.report.api import reportAPI -from oonib.api import oonibAPI +from oonib.api import ooniBackend from oonib import oonibackend from oonib import config @@ -37,7 +37,7 @@ def txSetupFailed(failure): log.err("Setup failed") log.exception(failure) -def setupCollector(tor_process_protocol): +def setupCollector(tor_process_protocol, datadir): def setup_complete(port): #XXX: drop some other noise about what API are available on this machine print("Exposed collector Tor hidden service on httpo://%s" @@ -53,7 +53,7 @@ def setupCollector(tor_process_protocol): #XXX: also set up a separate keyed hidden service for collectors to push their status to, if the bouncer is enabled hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, data_dir=datadir) - hidden_service = hs_endpoint.listen(oonibAPI) + hidden_service = hs_endpoint.listen(ooniBackend) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed) @@ -86,10 +86,9 @@ def startTor(): progress_updates=updates) else: d = launch_tor(torconfig, reactor, progress_updates=updates) - d.addCallback(setupCollector) + d.addCallback(setupCollector, datadir) d.addErrback(txSetupFailed) - if platformType == "win32": from twisted.scripts._twistw import WindowsApplicationRunner @@ -108,7 +107,10 @@ else: self.startApplication(self.application) # This is our addition. The rest is taken from # twisted/scripts/_twistd_unix.py 12.2.0 - startTor() + if config.main.tor_hidden_service: + startTor() + else: + reactor.listenTCP(8888, ooniBackend, interface="127.0.0.1") self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if os.path.exists(tempfile.gettempdir()):

1 0

[oonib/master] Add NetTest API
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 45a5ddb9eb7388976c5c7e09b5ca553681b6b46f Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:41:41 2013 +0200 Add NetTest API --- oonib/nettest/api.py | 10 ++++++++++ oonib/nettest/handlers.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/oonib/nettest/api.py b/oonib/nettest/api.py new file mode 100644 index 0000000..758b31d --- /dev/null +++ b/oonib/nettest/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.nettest import handlers +from oonib import config + +nettestAPI = [ + (r"/nettest", handlers.NetTestListHandler), + (r"/nettest/([a-f0-9]{64})", handlers.NetTestDescHandler), + (r"/nettest/([a-f0-9]{64})/py$", web.StaticFileHandler, {"path": + config.main.nettest_dir}), +] diff --git a/oonib/nettest/handlers.py b/oonib/nettest/handlers.py new file mode 100644 index 0000000..17d5590 --- /dev/null +++ b/oonib/nettest/handlers.py @@ -0,0 +1,25 @@ +import glob +import json +import os +import yaml + +from oonib.handlers import OONIBHandler +from oonib import config, log + +class NetTestDescHandler(OONIBHandler): + def get(self, netTestID): + bn = os.path.basename(netTestID) + ".desc" + try: + f = open(os.path.join(config.main.nettest_dir, bn)) + a = {} + netTestDesc = yaml.safe_load(f) + for k in ['name', 'description', 'version', 'author', 'date']: + a[k] = netTestDesc[k] + self.write(json.dumps(a)) + except IOError: + log.err("No NetTest Descriptor found for id %s" % netTestID) + except Exception, e: + log.err("Invalid NetTest Descriptor found for id %s" % netTestID) + + +

1 0

[oonib/master] Normalize pluralization, use sha256 for input name
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 7d753791abef68c3cbbaf390b7ae92c01f1a56a8 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 18 23:11:05 2013 +0200 Normalize pluralization, use sha256 for input name Use the same pluralization semantics as the /deck api, and use sha256 hexdigest filename as specified in oonib.md --- oonib.conf.example | 2 +- oonib/api.py | 6 +++--- oonib/input/api.py | 10 ++++++++++ oonib/input/handlers.py | 41 +++++++++++++++++++++++++++++++++++++++++ oonib/inputs/api.py | 10 ---------- oonib/inputs/handlers.py | 41 ----------------------------------------- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 36df0e3..977b5dc 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -1,7 +1,7 @@ main: report_dir: data/reports/ archive_dir: data/archive/ - inputs_dir: data/inputs/ + input_dir: data/inputs/ deck_dir: data/decks/ policy_file: data/policy.yaml bouncer_file: data/bouncer.yaml diff --git a/oonib/api.py b/oonib/api.py index 775c4ad..384b949 100644 --- a/oonib/api.py +++ b/oonib/api.py @@ -2,7 +2,7 @@ from cyclone import web from oonib.deck.api import deckAPI from oonib.report.api import reportAPI -from oonib.inputs.api import inputsAPI +from oonib.input.api import inputAPI from oonib.policy.api import policyAPI from oonib.bouncer.api import bouncerAPI @@ -11,8 +11,8 @@ from oonib import config ooniBackendAPI = [] ooniBackendAPI += reportAPI -if config.main.inputs_dir: - ooniBackendAPI += inputsAPI +if config.main.input_dir: + ooniBackendAPI += inputAPI if config.main.deck_dir: ooniBackendAPI += deckAPI diff --git a/oonib/input/__init__.py b/oonib/input/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/input/api.py b/oonib/input/api.py new file mode 100644 index 0000000..88d85a2 --- /dev/null +++ b/oonib/input/api.py @@ -0,0 +1,10 @@ +from cyclone import web +from oonib.input import handlers +from oonib import config + +inputAPI = [ + (r"/input", handlers.InputListHandler), + (r"/input/([a-f0-9]{64})", handlers.InputDescHandler), + (r"/input/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": + config.main.input_dir}), +] diff --git a/oonib/input/handlers.py b/oonib/input/handlers.py new file mode 100644 index 0000000..3ab0ea0 --- /dev/null +++ b/oonib/input/handlers.py @@ -0,0 +1,41 @@ +import glob +import json +import os + +import yaml + +from oonib.handlers import OONIBHandler + +from oonib import config + +class InputDescHandler(OONIBHandler): + def get(self, inputID): + #XXX return the input descriptor + # see oonib.md in ooni-spec + bn = os.path.basename(inputID) + ".desc" + try: + f = open(os.path.join(config.main.input_dir, bn)) + a = {} + inputDesc = yaml.safe_load(f) + a['id'] = inputID + a['name'] = inputDesc['name'] + a['description'] = inputDesc['description'] + self.write(json.dumps(a)) + except Exception: + log.err("No Input Descriptor found for id %s" % inputID) + +class InputListHandler(OONIBHandler): + def get(self): + path = os.path.abspath(config.main.input_dir) + "/*.desc" + inputnames = map(os.path.basename, glob.iglob(path)) + inputList = [] + for inputname in inputnames: + f = open(os.path.join(config.main.input_dir, inputname)) + d = yaml.safe_load(f) + inputList.append({ + 'id': inputname, + 'name': d['name'], + 'description': d['description'] + }) + f.close() + self.write(json.dumps(inputList)) diff --git a/oonib/inputs/__init__.py b/oonib/inputs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/inputs/api.py b/oonib/inputs/api.py deleted file mode 100644 index bfcf076..0000000 --- a/oonib/inputs/api.py +++ /dev/null @@ -1,10 +0,0 @@ -from cyclone import web -from oonib.inputs import handlers -from oonib import config - -inputsAPI = [ - (r"/inputs", handlers.InputsListHandler), - (r"/inputs/([a-f0-9]{64})", handlers.InputsDescHandler), - (r"/inputs/([a-f0-9]{64})/file$", web.StaticFileHandler, {"path": - config.main.inputs_dir}), -] diff --git a/oonib/inputs/handlers.py b/oonib/inputs/handlers.py deleted file mode 100644 index 373e275..0000000 --- a/oonib/inputs/handlers.py +++ /dev/null @@ -1,41 +0,0 @@ -import glob -import json -import os - -import yaml - -from oonib.handlers import OONIBHandler - -from oonib import config - -class InputsDescHandler(OONIBHandler): - def get(self, inputID): - #XXX return the input descriptor - # see oonib.md in ooni-spec - bn = os.path.basename(inputID) + ".desc" - try: - f = open(os.path.join(config.main.inputs_dir, bn)) - a = {} - inputDesc = yaml.safe_load(f) - a['id'] = inputID - a['name'] = inputDesc['name'] - a['description'] = inputDesc['description'] - self.write(json.dumps(a)) - except Exception: - log.err("No Input Descriptor found for id %s" % inputID) - -class InputsListHandler(OONIBHandler): - def get(self): - path = os.path.abspath(config.main.inputs_dir) + "/*.desc" - inputnames = map(os.path.basename, glob.iglob(path)) - inputList = [] - for inputname in inputnames: - f = open(os.path.join(config.main.inputs_dir, inputname)) - d = yaml.safe_load(f) - inputList.append({ - 'id': inputname, - 'name': d['name'], - 'description': d['description'] - }) - f.close() - self.write(json.dumps(inputList))

1 0