September 2013 - tor-commits - lists.torproject.org

r26343: {website} bump beta TBBs and other packages for tor 0.2.4.17-rc (website/trunk/include)
by Erinn Clark 06 Sep '13

06 Sep '13

Author: erinn Date: 2013-09-06 15:59:52 +0000 (Fri, 06 Sep 2013) New Revision: 26343 Modified: website/trunk/include/versions.wmi Log: bump beta TBBs and other packages for tor 0.2.4.17-rc Modified: website/trunk/include/versions.wmi =================================================================== --- website/trunk/include/versions.wmi 2013-09-05 13:57:40 UTC (rev 26342) +++ website/trunk/include/versions.wmi 2013-09-06 15:59:52 UTC (rev 26343) @@ -2,26 +2,26 @@ <define-tag version-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-win32-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-win32-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-win32-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-win32-bundle-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-win32-bundle-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-win32-bundle-alpha whitespace=delete>0.2.4.17-rc</define-tag> -<define-tag version-win32-bridge-bundle-alpha whitespace=delete>0.2.4.16-rc-<version-vidalia-stable></define-tag> -<define-tag version-win32-relay-bundle-alpha whitespace=delete>0.2.4.16-rc-<version-vidalia-stable></define-tag> -<define-tag version-win32-exit-bundle-alpha whitespace=delete>0.2.4.16-rc-<version-vidalia-stable></define-tag> +<define-tag version-win32-bridge-bundle-alpha whitespace=delete>0.2.4.17-rc-<version-vidalia-stable></define-tag> +<define-tag version-win32-relay-bundle-alpha whitespace=delete>0.2.4.17-rc-<version-vidalia-stable></define-tag> +<define-tag version-win32-exit-bundle-alpha whitespace=delete>0.2.4.17-rc-<version-vidalia-stable></define-tag> <define-tag version-osx-x86-bundle-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-osx-x86-bundle-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-osx-x86-bundle-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-osx-ppc-bundle-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-osx-ppc-bundle-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-osx-ppc-bundle-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-osx-x86-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-osx-x86-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-osx-x86-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-osx-ppc-stable whitespace=delete>0.2.3.25</define-tag> -<define-tag version-osx-ppc-alpha whitespace=delete>0.2.4.16-rc</define-tag> +<define-tag version-osx-ppc-alpha whitespace=delete>0.2.4.17-rc</define-tag> <define-tag version-torbrowserbundle whitespace=delete>2.3.25-12</define-tag> <define-tag version-torbrowser-tor whitespace=delete>0.2.3.25</define-tag> @@ -32,7 +32,7 @@ <define-tag version-torbrowser-otr whitespace=delete>3.2</define-tag> <define-tag version-torbrowser-vidalia whitespace=delete>0.2.21</define-tag> <define-tag version-torimbrowserbundle whitespace=delete>1.3.21</define-tag> -<define-tag version-torbrowserbundlealpha whitespace=delete>2.4.16-beta-1</define-tag> +<define-tag version-torbrowserbundlealpha whitespace=delete>2.4.17-beta-1</define-tag> <define-tag version-torobfsbundlealpha whitespace=delete>2.4.16-beta-1-pt1</define-tag> <define-tag version-torbrowserbundlelinux32 whitespace=delete>2.3.25-12</define-tag> @@ -42,8 +42,8 @@ <define-tag version-gnu-torbrowser-firefox whitespace=delete>17.0.8esr</define-tag> <define-tag version-gnu-torbrowser-torbutton whitespace=delete>1.5.2</define-tag> <define-tag version-gnu-torbrowser-vidalia whitespace=delete>0.2.21</define-tag> -<define-tag version-torbrowserbundlelinux32alpha whitespace=delete>2.4.16-beta-1</define-tag> -<define-tag version-torbrowserbundlelinux64alpha whitespace=delete>2.4.16-beta-1</define-tag> +<define-tag version-torbrowserbundlelinux32alpha whitespace=delete>2.4.17-beta-1</define-tag> +<define-tag version-torbrowserbundlelinux64alpha whitespace=delete>2.4.17-beta-1</define-tag> <define-tag version-torobfsbundlelinux32alpha whitespace=delete>2.4.16-beta-1-pt1</define-tag> <define-tag version-torobfsbundlelinux64alpha whitespace=delete>2.4.16-beta-1-pt1</define-tag> @@ -53,8 +53,8 @@ <define-tag version-osx-torbrowser-firefox whitespace=delete>17.0.8esr</define-tag> <define-tag version-osx-torbrowser-torbutton whitespace=delete>1.5.2</define-tag> <define-tag version-osx-torbrowser-vidalia whitespace=delete>0.2.21</define-tag> -<define-tag version-torbrowserbundleosx32alpha whitespace=delete>2.4.16-beta-1</define-tag> -<define-tag version-torbrowserbundleosx64alpha whitespace=delete>2.4.16-beta-1</define-tag> +<define-tag version-torbrowserbundleosx32alpha whitespace=delete>2.4.17-beta-1</define-tag> +<define-tag version-torbrowserbundleosx64alpha whitespace=delete>2.4.17-beta-1</define-tag> <define-tag version-torobfsbundleosx32alpha whitespace=delete>2.4.16-beta-1-pt1</define-tag> <define-tag version-torobfsbundleosx64alpha whitespace=delete>2.4.16-beta-1-pt1</define-tag>

1 0

[translation/tails-greeter_completed] Update translations for tails-greeter_completed
by translation＠torproject.org 05 Sep '13

05 Sep '13

commit d81885b2ddc2d67b9df98c0d3a96dc15376744c4 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Sep 5 22:16:26 2013 +0000 Update translations for tails-greeter_completed --- de/de.po | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/de/de.po b/de/de.po index 9a2d2b9..aadbb32 100644 --- a/de/de.po +++ b/de/de.po @@ -7,6 +7,7 @@ # mo <moritz(a)torservers.net>, 2013 # Cooligan <ppt23(a)lkj.hopto.org>, 2012 # Sacro <Scion(a)T-Online.de>, 2012 +# Victor1975 <VKerley75(a)gmail.com>, 2013 # djcrackhome <wallcityserver(a)gmail.com>, 2012 # Wasilis <inactive+Wasilis(a)transifex.com>, 2013 msgid "" @@ -14,8 +15,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2013-05-12 19:29+0200\n" -"PO-Revision-Date: 2013-08-25 20:00+0000\n" -"Last-Translator: mo <moritz(a)torservers.net>\n" +"PO-Revision-Date: 2013-09-05 22:10+0000\n" +"Last-Translator: Victor1975 <VKerley75(a)gmail.com>\n" "Language-Team: German (http://www.transifex.com/projects/p/torproject/language/de/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -49,7 +50,7 @@ msgstr "Nein" #: ../glade/persistencewindow.glade.h:7 msgid "Passphrase:" -msgstr "Passphrase:" +msgstr "Passwort:" #: ../glade/persistencewindow.glade.h:8 msgid "Read-Only?"

1 0

[translation/tails-greeter] Update translations for tails-greeter
by translation＠torproject.org 05 Sep '13

05 Sep '13

commit 0438cf4249c1e0f811f45b6455b1499824a542c1 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Sep 5 22:16:23 2013 +0000 Update translations for tails-greeter --- de/de.po | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/de/de.po b/de/de.po index 9a2d2b9..aadbb32 100644 --- a/de/de.po +++ b/de/de.po @@ -7,6 +7,7 @@ # mo <moritz(a)torservers.net>, 2013 # Cooligan <ppt23(a)lkj.hopto.org>, 2012 # Sacro <Scion(a)T-Online.de>, 2012 +# Victor1975 <VKerley75(a)gmail.com>, 2013 # djcrackhome <wallcityserver(a)gmail.com>, 2012 # Wasilis <inactive+Wasilis(a)transifex.com>, 2013 msgid "" @@ -14,8 +15,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2013-05-12 19:29+0200\n" -"PO-Revision-Date: 2013-08-25 20:00+0000\n" -"Last-Translator: mo <moritz(a)torservers.net>\n" +"PO-Revision-Date: 2013-09-05 22:10+0000\n" +"Last-Translator: Victor1975 <VKerley75(a)gmail.com>\n" "Language-Team: German (http://www.transifex.com/projects/p/torproject/language/de/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -49,7 +50,7 @@ msgstr "Nein" #: ../glade/persistencewindow.glade.h:7 msgid "Passphrase:" -msgstr "Passphrase:" +msgstr "Passwort:" #: ../glade/persistencewindow.glade.h:8 msgid "Read-Only?"

1 0

[translation/tails-persistence-setup_completed] Update translations for tails-persistence-setup_completed
by translation＠torproject.org 05 Sep '13

05 Sep '13

commit ddf3c243669273adf2d7bbb9da7c9778e1afbb64 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Sep 5 22:16:20 2013 +0000 Update translations for tails-persistence-setup_completed --- de/de.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/de/de.po b/de/de.po index 03645df..a903fe6 100644 --- a/de/de.po +++ b/de/de.po @@ -12,7 +12,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2013-08-01 16:49+0200\n" -"PO-Revision-Date: 2013-09-01 17:30+0000\n" +"PO-Revision-Date: 2013-09-05 22:10+0000\n" "Last-Translator: MarioBaier26 <mario.baier26(a)gmx.de>\n" "Language-Team: German (http://www.transifex.com/projects/p/torproject/language/de/)\n" "MIME-Version: 1.0\n"

1 0

[translation/tails-persistence-setup] Update translations for tails-persistence-setup
by translation＠torproject.org 05 Sep '13

05 Sep '13

commit 6063b80f4ed2974dc3843ccc210c7e16b83be5ee Author: Translation commit bot <translation(a)torproject.org> Date: Thu Sep 5 22:16:17 2013 +0000 Update translations for tails-persistence-setup --- de/de.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/de/de.po b/de/de.po index 03645df..a903fe6 100644 --- a/de/de.po +++ b/de/de.po @@ -12,7 +12,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2013-08-01 16:49+0200\n" -"PO-Revision-Date: 2013-09-01 17:30+0000\n" +"PO-Revision-Date: 2013-09-05 22:10+0000\n" "Last-Translator: MarioBaier26 <mario.baier26(a)gmx.de>\n" "Language-Team: German (http://www.transifex.com/projects/p/torproject/language/de/)\n" "MIME-Version: 1.0\n"

1 0

[ooni-probe/master] Don't reference deck_hash if it was not supplied
by art＠torproject.org 05 Sep '13

05 Sep '13

commit f94db8679b6a8cb77bc9b25dd48f823f3418783f Author: aagbsn <aagbsn(a)extc.org> Date: Thu Sep 5 10:58:35 2013 -0400 Don't reference deck_hash if it was not supplied --- ooni/deck.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 3b0716c..36d88c6 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -77,9 +77,10 @@ class Deck(InputFile): cached_deck_dir = os.path.join(config.advanced.data_dir, 'decks') - cache_path = os.path.join(cached_deck_dir, deck_hash) - self.cached_file = cache_path - self.cached_descriptor = cache_path + '.desc' + if deck_hash: + cache_path = os.path.join(cached_deck_dir, deck_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' if deckFile: self.loadDeck(deckFile)

1 0

[ooni-probe/master] Implement more thorough end 2 end tests for oonibackend
by art＠torproject.org 05 Sep '13

05 Sep '13

commit 7e6885926987b191d0eb0145240e334cd5347e84 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 17:16:02 2013 +0200 Implement more thorough end 2 end tests for oonibackend --- ooni/tests/test_oonibclient.py | 42 ++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 0f22786..aef48b7 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -12,12 +12,14 @@ from ooni.oonibclient import OONIBClient data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir + input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' +deck_id = 'd4ae40ecfb3c1b943748cce503ab8233efce7823f3e391058fc0f87829c644ed' class TestOONIBClient(unittest.TestCase): def setUp(self): host = '127.0.0.1' - port = 8888 + port = 8889 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, port)) @@ -26,6 +28,7 @@ class TestOONIBClient(unittest.TestCase): except: pass os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) + os.mkdir(os.path.join(data_dir, 'decks')) except Exception as ex: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) @@ -53,16 +56,22 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_get_deck_list(self): - input_list = yield self.oonibclient.getInputList() - self.assertTrue(isinstance(input_list, list)) + deck_list = yield self.oonibclient.getDeckList() + self.assertTrue(isinstance(deck_list, list)) + @defer.inlineCallbacks def test_get_deck_descriptor(self): - pass + deck_descriptor = yield self.oonibclient.getDeck(deck_id) + for key in ['name', 'description', + 'version', 'author', 'date', 'id']: + self.assertTrue(hasattr(deck_descriptor, key)) + @defer.inlineCallbacks def test_download_deck(self): - pass + yield self.oonibclient.downloadDeck(deck_id) def test_lookup_invalid_helpers(self): + self.oonibclient.address = 'http://127.0.0.1:8888' return self.failUnlessFailure( self.oonibclient.lookupTestHelpers([ 'sdadsadsa', 'dns' @@ -70,17 +79,26 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_lookup_test_helpers(self): + self.oonibclient.address = 'http://127.0.0.1:8888' required_helpers = [u'http-return-json-headers', u'dns'] helpers = yield self.oonibclient.lookupTestHelpers(required_helpers) self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) @defer.inlineCallbacks - def test_get_nettest_list(self): - input_list = yield self.oonibclient.getInputList() - self.assertTrue(isinstance(input_list, list)) + def test_invalid_requests(self): + + @defer.inlineCallbacks + def all_requests(path): + for mthd in ['GET', 'POST', 'PUT', 'OPTION']: + try: + yield self.oonibclient.queryBackend(mthd, path) + except: + pass - def test_get_nettest_descriptor(self): - pass + for path in ['/policy/input', '/policy/nettest', + '/input', '/input/'+'a'*64, '/fooo']: + yield all_requests(path) - def test_download_nettest(self): - pass + for path in ['/bouncer']: + self.oonibclient.address = 'http://127.0.0.1:8888' + yield all_requests(path)

1 0

[ooni-probe/master] Update ooniprobe to reflect changes in the backend specification
by art＠torproject.org 05 Sep '13

05 Sep '13

commit b80a5e2e23483d5c55749341730a743c6930e5af Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 13:37:29 2013 +0200 Update ooniprobe to reflect changes in the backend specification We now also support running test decks that have test helpers that can't reside on the same machine. --- ooni/deck.py | 39 ++++++++++++++++++++++----------------- ooni/errors.py | 11 ++++++++++- ooni/oonibclient.py | 17 ++++++++++++----- ooni/tests/test_oonibclient.py | 11 ++++++----- 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index a8aee39..23f4cb1 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -41,29 +41,34 @@ class Deck(object): def setup(self): """ fetch and verify inputs for all NetTests in the deck """ if self.bouncer: + log.msg("Looking up test helpers...") + yield self.lookupTestHelpers() for net_test_loader in self.netTestLoaders: + log.msg("Fetching required net test inputs...") yield self.fetchAndVerifyNetTestInput(net_test_loader) - yield self.lookupTestHelper(net_test_loader) - yield self.lookupTestCollector(net_test_loader) @defer.inlineCallbacks - def lookupTestHelper(self, net_test_loader): + def lookupTestHelpers(self): oonibclient = OONIBClient(self.bouncer) - for th in net_test_loader.requiredTestHelpers: - # {'name':'', 'option':'', 'test_class':''} - response = yield oonibclient.lookupTestHelper(th['name']) - th['test_class'].localOptions[th['option']] = response['test-helper'] - #XXX: collector is only set once! - if 'collector' in response.keys(): - net_test_loader.collector = response['collector'].encode('utf-8') + required_test_helpers = [] + for net_test_loader in self.netTestLoaders: + for th in net_test_loader.requiredTestHelpers: + # {'name':'', 'option':'', 'test_class':''} + required_test_helpers.append(th['name']) - @defer.inlineCallbacks - def lookupTestCollector(self, net_test_loader): - oonibclient = OONIBClient(self.bouncer) - if net_test_loader.collector is None: - response = yield oonibclient.lookupTestCollector(net_test_loader.testName) - if 'collector' in response.keys(): - net_test_loader.collector = response['collector'].encode('utf-8') + response = yield oonibclient.lookupTestHelpers(required_test_helpers) + + for net_test_loader in self.netTestLoaders: + log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name']) + if not net_test_loader.requiredTestHelpers: + log.msg("Using the default collector: %s" % response['default']['collector']) + net_test_loader.collector = response['default']['collector'] + + for th in net_test_loader.requiredTestHelpers: + test_helper = response[th['name']] + log.msg("Using this helper: %s" % test_helper) + th['test_class'].localOptions[th['option']] = test_helper['address'] + net_test_loader.collector = test_helper['collector'].encode('utf-8') @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): diff --git a/ooni/errors.py b/ooni/errors.py index 200491e..b29b778 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -165,7 +165,10 @@ class NoMoreReporters(Exception): class TorNotRunning(Exception): pass -class OONIBReportError(Exception): +class OONIBError(Exception): + pass + +class OONIBReportError(OONIBError): pass class OONIBReportUpdateError(OONIBReportError): @@ -185,3 +188,9 @@ class CouldNotFindTestHelper(Exception): class CouldNotFindTestCollector(Exception): pass + +def get_error(error_key): + if error_key == 'test-helpers-key-missing': + return CouldNotFindTestHelper + else: + return OONIBError diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 227c775..9dada5f 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -121,9 +121,16 @@ class OONIBClient(object): bodyProducer = None if query: bodyProducer = StringProducer(json.dumps(query)) - + def genReceiver(finished, content_length): - return BodyReceiver(finished, content_length, json.loads) + def process_response(s): + response = json.loads(s) + if 'error' in response: + print "Got this backend error message %s" % response + log.err("Got this backend error message %s" % response) + raise e.get_error(response['error']) + return response + return BodyReceiver(finished, content_length, process_response) return self._request(method, urn, genReceiver, bodyProducer) @@ -197,15 +204,15 @@ class OONIBClient(object): defer.returnValue(test_collector) @defer.inlineCallbacks - def lookupTestHelper(self, test_helper_name): + def lookupTestHelpers(self, test_helper_names): try: test_helper = yield self.queryBackend('POST', '/bouncer', - query={'test-helper': test_helper_name}) + query={'test-helpers': test_helper_names}) except Exception: raise e.CouldNotFindTestHelper - if not test_helpers: + if not test_helper: raise e.CouldNotFindTestHelper defer.returnValue(test_helper) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 1a1c4bc..0f22786 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -64,14 +64,15 @@ class TestOONIBClient(unittest.TestCase): def test_lookup_invalid_helpers(self): return self.failUnlessFailure( - self.oonibclient.lookupTestHelpers( - ['dns', 'http-return-json-headers', 'sdadsadsa'] - ), e.CouldNotFindTestHelper) + self.oonibclient.lookupTestHelpers([ + 'sdadsadsa', 'dns' + ]), e.CouldNotFindTestHelper) @defer.inlineCallbacks def test_lookup_test_helpers(self): - helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers']) - self.assertTrue(len(helpers) == 1) + required_helpers = [u'http-return-json-headers', u'dns'] + helpers = yield self.oonibclient.lookupTestHelpers(required_helpers) + self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default'])) @defer.inlineCallbacks def test_get_nettest_list(self):

1 0

[ooni-probe/master] bump to 1.0.0-rc1
by art＠torproject.org 05 Sep '13

05 Sep '13

commit be1fa4b7f0372ca35526fafbae06dd706f969ec5 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 30 18:01:10 2013 +0200 bump to 1.0.0-rc1 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 16fa9d7..2e7160b 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Arturo Filastò" -__version__ = "0.0.12" +__version__ = "1.0.0-rc1" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter',

1 0

[ooni-probe/master] Add support for looking up decks and deck descriptors in oonibclient
by art＠torproject.org 05 Sep '13

05 Sep '13

commit 0b62769ebed74e62882b06ee4579e78799df3d64 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 17:09:05 2013 +0200 Add support for looking up decks and deck descriptors in oonibclient --- ooni/deck.py | 70 ++++++++++++++++++++++++++++++++++-- ooni/oonibclient.py | 99 +++++++++++++++++++++++---------------------------- 2 files changed, 112 insertions(+), 57 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 23f4cb1..3b0716c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -5,20 +5,82 @@ from ooni.settings import config from ooni.utils import log from ooni.utils.txagentwithsocks import Agent from ooni import errors as e -from ooni.oonibclient import OONIBClient from twisted.internet import reactor, defer import os import re import yaml +import json +from hashlib import sha256 -class Deck(object): - def __init__(self, deckFile=None): +class InputFile(object): + def __init__(self, input_hash): + self.id = input_hash + cached_input_dir = os.path.join(config.advanced.data_dir, + 'inputs') + cache_path = os.path.join(cached_input_dir, input_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + + @property + def descriptorCached(self): + if os.path.exists(self.cached_descriptor): + with open(self.cached_descriptor) as f: + descriptor = json.load(f) + self.load(descriptor) + return True + return False + + @property + def fileCached(self): + if os.path.exists(self.cached_file): + try: + self.verify() + except AssertionError: + log.err("The input %s failed validation. Going to consider it not cached." % self.id) + return False + return True + return False + + def save(self): + with open(self.cached_descriptor, 'w+') as f: + json.dump({ + 'name': self.name, + 'id': self.id, + 'version': self.version, + 'author': self.author, + 'date': self.date, + 'description': self.description + }, f) + + def load(self, descriptor): + self.name = descriptor['name'] + self.version = descriptor['version'] + self.author = descriptor['author'] + self.date = descriptor['date'] + self.description = descriptor['description'] + + def verify(self): + digest = os.path.basename(self.cached_file) + with open(self.cached_file) as f: + file_hash = sha256(f.read()) + assert file_hash.hexdigest() == digest + +class Deck(InputFile): + def __init__(self, deck_hash=None, deckFile=None): + self.id = deck_hash self.bouncer = None self.netTestLoaders = [] self.inputs = [] self.testHelpers = {} + + cached_deck_dir = os.path.join(config.advanced.data_dir, + 'decks') + cache_path = os.path.join(cached_deck_dir, deck_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + if deckFile: self.loadDeck(deckFile) def loadDeck(self, deckFile): @@ -49,6 +111,7 @@ class Deck(object): @defer.inlineCallbacks def lookupTestHelpers(self): + from ooni.oonibclient import OONIBClient oonibclient = OONIBClient(self.bouncer) required_test_helpers = [] for net_test_loader in self.netTestLoaders: @@ -73,6 +136,7 @@ class Deck(object): @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ + from ooni.oonibclient import OONIBClient log.debug("Fetching and verifying inputs") for i in net_test_loader.inputFiles: if 'url' in i: diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9dada5f..24dc374 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -7,64 +7,12 @@ from twisted.internet import defer, reactor from ooni.utils.txagentwithsocks import Agent +from ooni.deck import Deck, InputFile from ooni import errors as e from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader -class InputFile(object): - def __init__(self, input_hash): - self.id = input_hash - cached_input_dir = os.path.join(config.advanced.data_dir, - 'inputs') - cache_path = os.path.join(cached_input_dir, input_hash) - self.cached_file = cache_path - self.cached_descriptor = cache_path + '.desc' - - @property - def descriptorCached(self): - if os.path.exists(self.cached_descriptor): - with open(self.cached_descriptor) as f: - descriptor = json.load(f) - self.load(descriptor) - return True - return False - - @property - def fileCached(self): - if os.path.exists(self.cached_file): - try: - self.verify() - except AssertionError: - log.err("The input %s failed validation. Going to consider it not cached." % self.id) - return False - return True - return False - - def save(self): - with open(self.cached_descriptor, 'w+') as f: - json.dump({ - 'name': self.name, - 'id': self.id, - 'version': self.version, - 'author': self.author, - 'date': self.date, - 'description': self.description - }, f) - - def load(self, descriptor): - self.name = descriptor['name'] - self.version = descriptor['version'] - self.author = descriptor['author'] - self.date = descriptor['date'] - self.description = descriptor['description'] - - def verify(self): - digest = os.path.basename(self.cached_file) - with open(self.cached_file) as f: - file_hash = sha256(f.read()) - assert file_hash.hexdigest() == digest - class Collector(object): def __init__(self, address): self.address = address @@ -193,6 +141,50 @@ class OONIBClient(object): def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest') + def getDeckList(self): + return self.queryBackend('GET', '/deck') + + def getDeck(self, deck_hash): + deck = Deck(deck_hash) + if deck.descriptorCached: + return defer.succeed(deck) + else: + d = self.queryBackend('GET', '/deck/' + deck_hash) + + @d.addCallback + def cb(descriptor): + deck.load(descriptor) + deck.save() + return deck + + @d.addErrback + def err(err): + log.err("Failed to get descriptor for deck %s" % deck_hash) + print err + log.exception(err) + + return d + + def downloadDeck(self, deck_hash): + deck = Deck(deck_hash) + if deck.fileCached: + return defer.succeed(deck) + else: + d = self.download('/deck/'+deck_hash+'/file', deck.cached_file) + + @d.addCallback + def cb(res): + deck.verify() + return deck + + @d.addErrback + def err(err): + log.err("Failed to download the deck %s" % deck_hash) + print err + log.exception(err) + + return d + @defer.inlineCallbacks def lookupTestCollector(self, test_name): try: @@ -216,4 +208,3 @@ class OONIBClient(object): raise e.CouldNotFindTestHelper defer.returnValue(test_helper) -

1 0