February 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Dynamically generate the deck hash path.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit baed9a2481f591f8139920217d32e02dffff934f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 30 19:58:39 2014 +0100 Dynamically generate the deck hash path. --- ooni/deck.py | 4 ++-- ooni/tests/test_deck.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 8212c3d..1b5fd72 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -17,7 +17,7 @@ from hashlib import sha256 class InputFile(object): def __init__(self, input_hash, base_path=config.inputs_directory): self.id = input_hash - cache_path = os.path.join(base_path, input_hash) + cache_path = os.path.join(os.path.abspath(base_path), input_hash) self.cached_file = cache_path self.cached_descriptor = cache_path + '.desc' @@ -93,7 +93,7 @@ class Deck(InputFile): self.oonibclient = OONIBClient(self.bouncer) - self.decksDirectory = decks_directory + self.decksDirectory = os.path.abspath(decks_directory) self.deckHash = deck_hash if deckFile: self.loadDeck(deckFile) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 3a330c0..7c7bc35 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -41,7 +41,7 @@ class BaseTestCase(unittest.TestCase): subargs: [] test_file: %s/dummy_test.py testdeck: null - """ % self.cwd +""" % self.cwd class TestInputFile(BaseTestCase): def test_file_cached(self): @@ -93,8 +93,8 @@ class MockOONIBClient(object): class TestDeck(BaseTestCase): def setUp(self): super(TestDeck, self).setUp() - self.deck_file = os.path.join(self.cwd, - '4145fd06158ce16e45469fc6b41c3d79a93b958925484ffa93c205d9f3b2e30a') + deck_hash = sha256(self.dummy_deck_content).hexdigest() + self.deck_file = os.path.join(self.cwd, deck_hash) with open(self.deck_file, 'w+') as f: f.write(self.dummy_deck_content) with open(os.path.join(self.cwd, 'dummy_test.py'), 'w+') as f:

1 0

[ooni-probe/master] Install dev requirements before running trial
by art＠torproject.org 01 Mar '14

01 Mar '14

commit c241a94c1509482fb84079f60ed74ada2929179b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 15:03:58 2014 +0100 Install dev requirements before running trial --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index da48f1a..3514d79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: - pip install pyrex - pip install coveralls - python setup.py install + - pip install -r requirements-dev.txt # command to run tests, e.g. python setup.py test script: - coverage run $(which trial) ooni

1 0

[ooni-probe/master] Fix bug inside of oonibclient spotted in unittesting.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 95d67f74773b83e31a9da903b89581f2075d115d Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 13:51:43 2014 +0100 Fix bug inside of oonibclient spotted in unittesting. --- ooni/oonibclient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index be5804f..84e12b5 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -103,6 +103,9 @@ class OONIBClient(object): def genReceiver(finished, content_length): def process_response(s): + # If empty string then don't parse it. + if not s: + return try: response = json.loads(s) except ValueError:

1 0

[ooni-probe/master] Compute the base path inside of the unittest
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 6902a93c0f9b8f832cbf48b30d2cd5e672d4fd7e Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 30 20:02:29 2014 +0100 Compute the base path inside of the unittest --- ooni/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index cafb00f..a374232 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -1,9 +1,9 @@ import os import unittest from ooni.utils import pushFilenameStack -basefilename = os.path.abspath('dummyfile') class TestUtils(unittest.TestCase): def test_pushFilenameStack(self): + basefilename = os.path.join(os.getcwd(), 'dummyfile') f = open(basefilename, "w+") f.write("0\n") f.close()

1 0

[ooni-probe/master] Remove lie from director.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 4a1c482fb65d7ee053097833a86c14cfcf4105f7 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 16:54:35 2014 +0100 Remove lie from director. --- ooni/director.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 3f538ca..f3d646a 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -271,15 +271,13 @@ class Director(object): log.debug("We now have the following circuits: ") for circuit in state.circuits.values(): log.debug(" * %s" % circuit) - + socks_port = yield state.protocol.get_conf("SocksPort") control_port = yield state.protocol.get_conf("ControlPort") config.tor.socks_port = int(socks_port.values()[0]) config.tor.control_port = int(control_port.values()[0]) - log.msg("Obtained our IP address from a Tor Relay %s" % config.probe_ip) - def setup_failed(failure): log.exception(failure) raise errors.UnableToStartTor

1 0

[ooni-probe/master] Fix a couple of bugs inside of oonibclient.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 90d2c9e40916a9f8f3c5c9476eb6d00f92d3656e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 13:40:02 2014 +0100 Fix a couple of bugs inside of oonibclient. --- ooni/oonibclient.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 60771b6..be5804f 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -47,10 +47,10 @@ class OONIBClient(object): def __init__(self, address): self.address = address - def _request(self, method, urn, genReceiver, bodyProducer=None): + address = self.address if self.address.startswith('httpo://'): - self.address = self.address.replace('httpo://', 'http://') + address = self.address.replace('httpo://', 'http://') agent = TrueHeadersSOCKS5Agent(reactor, proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', config.tor.socks_port)) @@ -59,18 +59,16 @@ class OONIBClient(object): log.err("HTTPS based bouncers are currently not supported.") raise e.InvalidOONIBBouncerAddress - elif address.startswith('http://'): + elif self.address.startswith('http://'): log.msg("Warning using unencrypted collector") - self.address = address - self.agent = Agent(reactor) + agent = Agent(reactor) - def _request(self, method, urn, genReceiver, bodyProducer=None): attempts = 0 finished = defer.Deferred() def perform_request(attempts): - uri = self.address + urn + uri = address + urn headers = {} d = agent.request(method, uri, bodyProducer=bodyProducer)

1 0

[ooni-probe/master] Merge branch 'master' into feature/improve_unittesting
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 0a921df80dace9756b9e5043611c0867e536e5c1 Merge: 6902a93 6d5cf42 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 30 20:04:40 2014 +0100 Merge branch 'master' into feature/improve_unittesting * master: Add tests for the oonib reporting API Fix the unittests for the oonibclient. Conflicts: ooni/oonibclient.py ooni/oonibclient.py | 12 +++++++++++- ooni/tests/test_nettest.py | 1 - ooni/tests/test_oonibclient.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --cc ooni/oonibclient.py index 6eff112,e257ef8..60771b6 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@@ -54,10 -52,15 +55,16 @@@ class OONIBClient(object) proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', config.tor.socks_port)) - elif address.startswith('https://'): + elif self.address.startswith('https://'): log.err("HTTPS based bouncers are currently not supported.") + raise e.InvalidOONIBBouncerAddress + elif address.startswith('http://'): + log.msg("Warning using unencrypted collector") + self.address = address + self.agent = Agent(reactor) + + def _request(self, method, urn, genReceiver, bodyProducer=None): attempts = 0 finished = defer.Deferred() diff --cc ooni/tests/test_nettest.py index d965735,d965735..8833d27 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@@ -16,7 -16,7 +16,6 @@@ from ooni.managers import TaskManage from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce from ooni.tests.mocks import MockNetTest, MockDirector, MockReporter from ooni.tests.mocks import MockMeasurementManager --defer.setDebugging(True) net_test_string = """ from twisted.python import usage

1 0

[ooni-probe/master] Add development requirements.txt
by art＠torproject.org 01 Mar '14

01 Mar '14

commit a5460e66f470d24f108f73a191e20d58b6e3f020 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 15:03:21 2014 +0100 Add development requirements.txt --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..932a895 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +mock

1 0

[ooni-probe/master] Add unittests for OONIB reporter
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 4c4ea8b014bed5fae89094a82e81d643a836b76d Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 15:03:03 2014 +0100 Add unittests for OONIB reporter --- ooni/reporter.py | 2 +- ooni/tests/test_reporter.py | 62 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 545997d..7db24aa 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -367,7 +367,7 @@ class OONIBReporter(OReporter): log.err("Failed to parse collector response %s" % backend_response) log.exception(e) raise errors.OONIBReportCreationError - + if response.code == 406: # XXX make this more strict log.err("The specified input or nettests cannot be submitted to this collector.") diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 0a4d240..23c1190 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -1,24 +1,43 @@ import yaml +import json import time +from mock import MagicMock +from twisted.internet import defer from twisted.trial import unittest -from ooni.reporter import YAMLReporter +from ooni.utils.net import StringProducer +from ooni import errors as e +from ooni.reporter import YAMLReporter, OONIBReporter class MockTest(object): _start_time = time.time() report = {'report_content': 'ham'} input = 'spam' +test_details = { + 'test_name': 'spam', + 'test_version': '1.0', + 'software_name': 'spam', + 'software_version': '1.0', + 'input_hashes': [], + 'probe_asn': 'AS0' +} + +oonib_new_report_message = { + 'report_id': "2014-01-29T202038Z_AS0_"+"A"*50, + 'backend_version': "1.0" +} + +oonib_generic_error_message = { + 'error': 'generic-error' +} + class TestYAMLReporter(unittest.TestCase): def setUp(self): pass def test_write_report(self): - test_details = { - 'test_name': 'spam', - 'test_version': '1.0' - } test = MockTest() y_reporter = YAMLReporter(test_details) @@ -37,3 +56,36 @@ class TestYAMLReporter(unittest.TestCase): 'test_name', 'test_started', \ 'test_runtime']) +class TestOONIBReporter(unittest.TestCase): + + def setUp(self): + self.mock_response = {} + self.collector_address = 'http://example.com' + + self.oonib_reporter = OONIBReporter(test_details, self.collector_address) + self.oonib_reporter.agent = MagicMock() + self.mock_agent_response = MagicMock() + def deliverBody(body_receiver): + body_receiver.dataReceived(json.dumps(self.mock_response)) + body_receiver.connectionLost(None) + self.mock_agent_response.deliverBody = deliverBody + self.oonib_reporter.agent.request.return_value = defer.succeed(self.mock_agent_response) + + @defer.inlineCallbacks + def test_create_report(self): + self.mock_response = oonib_new_report_message + yield self.oonib_reporter.createReport() + assert self.oonib_reporter.reportID == oonib_new_report_message['report_id'] + + @defer.inlineCallbacks + def test_create_report_failure(self): + self.mock_response = oonib_generic_error_message + self.mock_agent_response.code = 406 + yield self.assertFailure(self.oonib_reporter.createReport(), e.OONIBReportCreationError) + + @defer.inlineCallbacks + def test_write_report_entry(self): + req = {'content': 'something'} + yield self.oonib_reporter.writeReportEntry(req) + assert self.oonib_reporter.agent.request.called +

1 0

[ooni-probe/master] Make the geoip related tests make a less stronger assertion.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 9d44242c98099a532b89dcd9738a71c7c680235c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 20 12:21:40 2014 +0000 Make the geoip related tests make a less stronger assertion. --- ooni/nettest.py | 2 +- ooni/tests/test_geoip.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index aa8a21d..943aeba 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -245,7 +245,7 @@ class NetTestLoader(object): 'ip': '127.0.0.1' } - if config.probe_ip.address and (config.privacy.includeip or \ + if config.probe_ip and config.probe_ip.address and (config.privacy.includeip or \ config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity): diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index ac18a2e..6ae31f3 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -11,8 +11,8 @@ class TestGeoIP(unittest.TestCase): def test_ip_to_location(self): location = geoip.IPToLocation('8.8.8.8') - assert location['countrycode'] == 'US' - assert location['asn'] == 'AS15169' + assert 'countrycode' in location + assert 'asn' in location assert 'city' in location @defer.inlineCallbacks

1 0