- tor-commits - lists.torproject.org

[ooni-probe/master] Re-create the command inside the test method.
by art＠torproject.org 29 Apr '16

29 Apr '16

commit d368c895b9fda39d3422ea78e567876b1b1294dc Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 14:41:45 2016 +0100 Re-create the command inside the test method. If it's not done inside the test method when it fails and is re-scheduled it will not re-run the setUp and hence it will have already been deleted. --- ooni/nettests/third_party/psiphon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 7c1462b..575002b 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -58,6 +58,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): getenv('HOME'), 'psiphon-circumvention-system/pyclient/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) + def createCommand(self): # psi_client.py can not be run directly because the paths in the # code are relative, so it'll fail to execute from this test x = """ @@ -68,7 +69,7 @@ connect(False) f.write(x) f.close() self.command = [sys.executable, f.name] - log.debug('command: %s' % ''.join(self.command)) + log.debug('command: %s' % ' '.join(self.command)) def handleRead(self, stdout, stderr): if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: @@ -81,6 +82,7 @@ connect(False) def test_psiphon(self): log.debug('PsiphonTest.test_psiphon') + self.createCommand() self.report['bootstrapped_success'] = None self.report['request_success'] = None

1 0

[ooni-probe/master] Implement measurement_start_time and test_start_time in client
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 58db73a6d8c7d5296262cfa8ea2ce6e987ef724e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 17 15:49:08 2016 +0100 Implement measurement_start_time and test_start_time in client --- ooni/nettest.py | 11 ++++++----- ooni/otime.py | 3 +++ ooni/reporter.py | 4 +++- ooni/tasks.py | 5 ++--- ooni/tests/test_director.py | 6 ++---- ooni/tests/test_reporter.py | 2 +- ooni/tests/test_utils.py | 10 +++++----- ooni/utils/__init__.py | 6 +++--- 8 files changed, 25 insertions(+), 22 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9bdc6d9..9c5e171 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -197,8 +197,7 @@ class NetTestLoader(object): return input_files - @property - def testDetails(self): + def setTestDetails(self): from ooni import __version__ as software_version input_file_hashes = [] @@ -206,8 +205,8 @@ class NetTestLoader(object): input_file_hashes.append(input_file['hash']) options = sanitize_options(self.options) - test_details = { - 'start_time': otime.epochToUTC(time.time()), + self.testDetails = { + 'test_start_time': otime.UTCTimestampNow(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], 'probe_ip': config.probe_ip.geodata['ip'], @@ -223,7 +222,6 @@ class NetTestLoader(object): 'annotations': self.annotations, 'data_format_version': '0.2.0' } - return test_details def _parseNetTestOptions(self, klass): """ @@ -351,6 +349,9 @@ class NetTestLoader(object): if options: klass.localOptions = options + # XXX this class all needs to be refactored and this is kind of a + # hack. + self.setTestDetails() test_instance = klass() if test_instance.requiresRoot and not hasRawSocketPermission(): diff --git a/ooni/otime.py b/ooni/otime.py index 1406fbf..37ada02 100644 --- a/ooni/otime.py +++ b/ooni/otime.py @@ -85,3 +85,6 @@ def epochToTimestamp(seconds): def epochToUTC(seconds): return float(datetime.utcfromtimestamp(seconds).strftime("%s")) + +def UTCTimestampNow(): + return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") diff --git a/ooni/reporter.py b/ooni/reporter.py index d9d147c..d772ae9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -267,6 +267,7 @@ class OONIBReporter(OReporter): 'input': entry.testInstance.report.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.testInstance.report.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.testInstance.report.pop('test_runtime', None), 'test_keys': entry.testInstance.report } @@ -275,6 +276,7 @@ class OONIBReporter(OReporter): 'input': entry.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } @@ -352,7 +354,7 @@ class OONIBReporter(OReporter): 'probe_cc': self.testDetails['probe_cc'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], - 'start_time': self.testDetails['start_time'], + 'test_start_time': self.testDetails['test_start_time'], 'input_hashes': self.testDetails['input_hashes'], 'data_format_version': self.testDetails['data_format_version'], 'format': 'json' diff --git a/ooni/tasks.py b/ooni/tasks.py index 3f331a2..b5a15e0 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -116,9 +116,8 @@ class Measurement(TaskWithTimeout): if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'test_start_time' not in self.testInstance.report.keys(): - start_time = otime.epochToUTC(self.testInstance._start_time) - self.testInstance.report['test_start_time'] = start_time + if 'measurement_start_time' not in self.testInstance.report.keys(): + self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() self.testInstance.setUp() diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 38e8481..61d504c 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -1,5 +1,3 @@ -import time - from mock import patch, MagicMock from ooni.settings import config @@ -60,7 +58,7 @@ class TestStartSniffing(unittest.TestCase): self.director = Director() self.testDetails = { 'test_name': 'foo', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' } # Each NetTestCase has a name attribute @@ -85,7 +83,7 @@ class TestStartSniffing(unittest.TestCase): self.testDetails = { 'test_name': 'bar', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' } with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: sniffer = mock_scapy_sniffer.return_value diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 4b99584..5adb4af 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -26,7 +26,7 @@ test_details = { 'input_hashes': [], 'probe_asn': 'AS0', 'probe_cc': 'ZZ', - 'start_time': time.time(), + 'test_start_time': '2016-01-01 22:33:11', 'data_format_version': '0.2.0' } diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index e2e448a..855eb19 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -8,7 +8,7 @@ class TestUtils(unittest.TestCase): def setUp(self): self.test_details = { 'test_name': 'foo', - 'start_time': 441763200 + 'test_start_time': '2016-01-01 01:22:22' } self.extension = 'ext' self.prefix = 'prefix' @@ -46,19 +46,19 @@ class TestUtils(unittest.TestCase): def test_generate_filename(self): filename = generate_filename(self.test_details) - self.assertEqual(filename, 'foo-1984-01-01T000000Z') + self.assertEqual(filename, 'foo-2016-01-01T012222Z') def test_generate_filename_with_extension(self): filename = generate_filename(self.test_details, extension=self.extension) - self.assertEqual(filename, 'foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext') def test_generate_filename_with_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z') def test_generate_filename_with_extension_and_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext') def test_generate_filename_with_filename(self): filename = generate_filename(self.test_details, filename=self.filename) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5e9616b..78a1c7b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -3,12 +3,12 @@ import string import random import glob import os +from datetime import datetime import gzip from base64 import b64encode from zipfile import ZipFile -from ooni import otime from ooni import errors @@ -122,8 +122,8 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): extension. """ if filename is None: - test_name, start_time = testDetails['test_name'], testDetails['start_time'] - start_time = otime.epochToTimestamp(start_time) + test_name, start_time = testDetails['test_name'], testDetails['test_start_time'] + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H%M%SZ") suffix = "%s-%s" % (test_name, start_time) basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix final_filename = '%s.%s' % (basename, extension) if extension is not None else basename

1 0

[ooni-probe/master] Fix bug in parsing of options for Lantern
by art＠torproject.org 29 Apr '16

29 Apr '16

commit bcc2be41fb5b3a0ce45e5dce81718f677fbf0af5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Mar 16 20:10:12 2016 +0100 Fix bug in parsing of options for Lantern * Add check to assert if lantern is installed in test requirements --- ooni/nettests/third_party/lantern.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooni/nettests/third_party/lantern.py b/ooni/nettests/third_party/lantern.py index da19566..5018ccb 100644 --- a/ooni/nettests/third_party/lantern.py +++ b/ooni/nettests/third_party/lantern.py @@ -10,6 +10,9 @@ from ooni.templates.process import ProcessTest, ProcessDirector from ooni.utils import log, net from ooni.errors import handleAllFailures +class LanternNotInstalled(Exception): + pass + class UsageOptions(usage.Options): optParameters = [ ['url', 'u', net.GOOGLE_HUMANS[0], @@ -35,6 +38,11 @@ class LanternTest(ProcessTest): author = "Aaron Gibson" version = "0.1.0" timeout = 120 + usageOptions = UsageOptions + + def requirements(self): + if not distutils.spawn.find_executable("lantern"): + raise LanternNotInstalled('lantern is not installed') def setUp(self): self.report['body'] = None

1 0

[ooni-probe/master] Remove dead/unnecessary code from otime
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 7528c07975edbd45776086ccaf329e0aed923d35 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Mar 22 17:33:12 2016 +0100 Remove dead/unnecessary code from otime * Minor naming changes to functions --- ooni/nettest.py | 2 +- ooni/otime.py | 80 +++--------------------------------------------- ooni/tasks.py | 2 +- ooni/tests/test_otime.py | 14 --------- ooni/utils/log.py | 2 +- 5 files changed, 8 insertions(+), 92 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9c5e171..6cd9f57 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -206,7 +206,7 @@ class NetTestLoader(object): options = sanitize_options(self.options) self.testDetails = { - 'test_start_time': otime.UTCTimestampNow(), + 'test_start_time': otime.timestampNowLongUTC(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], 'probe_ip': config.probe_ip.geodata['ip'], diff --git a/ooni/otime.py b/ooni/otime.py index 37ada02..ffa95d0 100644 --- a/ooni/otime.py +++ b/ooni/otime.py @@ -1,17 +1,4 @@ -from datetime import datetime, timedelta, tzinfo - -class UTC(tzinfo): - """UTC""" - ZERO = timedelta(0) - - def utcoffset(self, dt): - return self.ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return self.ZERO +from datetime import datetime def prettyDateNow(): """ @@ -19,72 +6,15 @@ def prettyDateNow(): """ return datetime.now().ctime() -def utcPrettyDateNow(): +def prettyDateNowUTC(): """ Returns a good looking string for utc time. """ return datetime.utcnow().ctime() -class InvalidTimestampFormat(Exception): - pass - -def fromTimestamp(s): - """ - Converts a string that is output from the timestamp function back to a - datetime object - - Args: - s (str): a ISO8601 formatted string. - ex. 1912-06-23T101234Z" - - Note: we currently only support parsing strings that are generated from the - timestamp function and have no intention in supporting the full standard. - """ - try: - date_part, time_part = s.split('T') - hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6] - year, month, day = date_part.split('-') - except: - raise InvalidTimestampFormat(s) - - return datetime(int(year), int(month), int(day), int(hours), int(minutes), - int(seconds)) - -def timestamp(t=None): +def timestampNowLongUTC(): """ - The timestamp for ooni reports follows ISO 8601 in - UTC time format. - We do not inlcude ':' and include seconds. - - Example: - - if the current date is "10:12:34 AM, June 23 1912" (datetime(1912, 6, - 23, 10, 12, 34)) - - the timestamp will be: - - "1912-06-23T101234Z" - - Args: - t (datetime): a datetime object representing the - time to be represented (*MUST* be expressed - in UTC). - - If not specified will default to the current time - in UTC. + Returns a timestamp in the format of %Y-%m-%d %H:%M:%S in Universal Time + Coordinates. """ - if not t: - t = datetime.utcnow() - ISO8601 = "%Y-%m-%dT%H%M%SZ" - return t.strftime(ISO8601) - - -def epochToTimestamp(seconds): - return timestamp(datetime.fromtimestamp(seconds, UTC())) - - -def epochToUTC(seconds): - return float(datetime.utcfromtimestamp(seconds).strftime("%s")) - -def UTCTimestampNow(): return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") diff --git a/ooni/tasks.py b/ooni/tasks.py index 381902a..ac128d6 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -140,7 +140,7 @@ class Measurement(TaskWithTimeout): def run(self): if 'measurement_start_time' not in self.testInstance.report.keys(): - self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow() + self.testInstance.report['measurement_start_time'] = otime.timestampNowLongUTC() return self.netTestMethod() diff --git a/ooni/tests/test_otime.py b/ooni/tests/test_otime.py deleted file mode 100644 index f3997eb..0000000 --- a/ooni/tests/test_otime.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest -from datetime import datetime -from ooni import otime - -test_date = datetime(2002, 6, 26, 22, 45, 49) - - -class TestOtime(unittest.TestCase): - def test_timestamp(self): - self.assertEqual(otime.timestamp(test_date), "2002-06-26T224549Z") - - def test_fromTimestamp(self): - time_stamp = otime.timestamp(test_date) - self.assertEqual(test_date, otime.fromTimestamp(time_stamp)) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index beba78f..d3036f0 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -64,7 +64,7 @@ class OONILogger(object): txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), - otime.utcPrettyDateNow())) + otime.prettyDateNowUTC())) self.fileObserver = txlog.FileLogObserver(daily_logfile) self.stdoutObserver = LogWithNoPrefix(sys.stdout)

1 0

[ooni-probe/master] Implement unittest to test for bug spotted by @bassosimone
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 48435739f1bdd6631065a8ef41e5c51605955b2a Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Mar 21 16:54:18 2016 +0100 Implement unittest to test for bug spotted by @bassosimone * Fix bug in serializing of JSON --- ooni/reporter.py | 2 +- ooni/tests/test_reporter.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index d772ae9..2ee8468 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -276,7 +276,7 @@ class OONIBReporter(OReporter): 'input': entry.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.pop('test_start_time', None), - 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), + 'measurement_start_time': entry.pop('measurement_start_time', None), 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 5adb4af..94192ee 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -32,7 +32,8 @@ test_details = { oonib_new_report_message = { 'report_id': "20140129T202038Z_AS0_" + "A" * 50, - 'backend_version': "1.0" + 'backend_version': "1.0", + 'supported_formats': ["yaml", "json"] } oonib_generic_error_message = {

1 0

[ooni-probe/master] Add check for input file in http_host test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 3683957390dca24ff0d29ae40b7ecea06cec29d1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 24 13:25:54 2016 +0100 Add check for input file in http_host test --- ooni/nettests/manipulation/http_host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index d0387cb..a38e3cc 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -21,7 +21,7 @@ class UsageOptions(usage.Options): optParameters = [['backend', 'b', None, 'URL of the test backend to use. Should be \ listening on port 80 and be a \ - HTTPReturnJSONHeadersHelper'], + HTTPReturnJSONHeadersHelper (ex. http://1.1.1.1)'], ['content', 'c', None, 'The file to read \ from containing the content of a block page']] @@ -50,7 +50,7 @@ class HTTPHost(httpt.HTTPTest): 'List of hostnames to test for censorship'] requiredTestHelpers = {'backend': 'http-return-json-headers'} - requiredOptions = ['backend'] + requiredOptions = ['backend', 'file'] requiresTor = False requiresRoot = False

1 0

[ooni-probe/master] Fix serialisation of SOA queries and cast set to list
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 07749637dd9d4d8b4c5f9ff9137a1dd9d2cafbd8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 24 13:37:41 2016 +0100 Fix serialisation of SOA queries and cast set to list --- ooni/nettests/manipulation/captiveportal.py | 2 +- ooni/templates/dnst.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index f700973..b1cbba7 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -426,7 +426,7 @@ class CaptivePortal(httpt.HTTPTest, dnst.DNSTest): gmatch, google_dns_result = yield self.compare_random_hostnames(3, 10) ret = { 'result': gmatch, - 'addresses': google_dns_result + 'addresses': list(google_dns_result) } if gmatch: diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 9ee7508..b96fdec 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -72,8 +72,8 @@ def representAnswer(answer): } if answer_type is 'SOA': represented_answer['ttl'] = answer.payload.ttl - represented_answer['hostname'] = answer.payload.mname - represented_answer['responsible_name'] = answer.payload.rname + represented_answer['hostname'] = answer.payload.mname.name + represented_answer['responsible_name'] = answer.payload.rname.name represented_answer['serial_number'] = answer.payload.serial represented_answer['refresh_interval'] = answer.payload.refresh represented_answer['retry_interval'] = answer.payload.retry

1 0

[ooni-probe/master] Move daphne to the experimental directory
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 03a820cbb53d1c9de114b0760ce58da0fdbce8df Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 24 13:40:27 2016 +0100 Move daphne to the experimental directory In it's current incarnation it's very hard to use and we should consider it experimental --- ooni/nettests/experimental/daphne.py | 121 +++++++++++++++++++++++++++++++++++ ooni/nettests/manipulation/daphne.py | 121 ----------------------------------- 2 files changed, 121 insertions(+), 121 deletions(-) diff --git a/ooni/nettests/experimental/daphne.py b/ooni/nettests/experimental/daphne.py new file mode 100644 index 0000000..72b5c8f --- /dev/null +++ b/ooni/nettests/experimental/daphne.py @@ -0,0 +1,121 @@ +# -*- encoding: utf-8 -*- +from twisted.python import usage +from twisted.internet import protocol, endpoints, reactor + +from ooni import nettest +from ooni.kit import daphn3 +from ooni.utils import log + +class Daphn3ClientProtocol(daphn3.Daphn3Protocol): + def nextStep(self): + log.debug("Moving on to next step in the state walk") + self.current_data_received = 0 + if self.current_step >= (len(self.steps) - 1): + log.msg("Reached the end of the state machine") + log.msg("Censorship fingerprint bisected!") + step_idx, mutation_idx = self.factory.mutation + log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) + #self.transport.loseConnection() + if self.report: + self.report['mutation_idx'] = mutation_idx + self.report['step_idx'] = step_idx + self.d.callback(None) + return + else: + self.current_step += 1 + if self._current_step_role() == self.role: + # We need to send more data because we are again responsible for + # doing so. + self.sendPayload() + + +class Daphn3ClientFactory(protocol.ClientFactory): + protocol = daphn3.Daphn3Protocol + mutation = [0,0] + steps = None + + def buildProtocol(self, addr): + p = self.protocol() + p.steps = self.steps + p.factory = self + return p + + def startedConnecting(self, connector): + log.msg("Started connecting %s" % connector) + + def clientConnectionFailed(self, reason, connector): + log.err("We failed connecting the the OONIB") + log.err("Cannot perform test. Perhaps it got blocked?") + log.err("Please report this to tor-assistants(a)torproject.org") + + def clientConnectionLost(self, reason, connector): + log.err("Daphn3 client connection lost") + print reason + +class daphn3Args(usage.Options): + optParameters = [ + ['host', 'h', '127.0.0.1', 'Target Hostname'], + ['port', 'p', 57003, 'Target port number']] + + optFlags = [['pcap', 'c', 'Specify that the input file is a pcap file'], + ['yaml', 'y', 'Specify that the input file is a YAML file (default)']] + +class daphn3Test(nettest.NetTestCase): + + name = "Daphn3" + description = "Bisects the censors fingerprint by mutating the given input packets." + usageOptions = daphn3Args + inputFile = ['file', 'f', None, + 'Specify the pcap or YAML file to be used as input to the test'] + + #requiredOptions = ['file'] + requiresRoot = False + requiresTor = False + steps = None + + def inputProcessor(self, filename): + """ + step_idx is the step in the packet exchange + ex. + [.X.] are packets sent by a client or a server + + client: [.1.] [.3.] [.4.] + server: [.2.] [.5.] + + mutation_idx: is the sub index of the packet as in the byte of the + packet at the step_idx that is to be mutated + + """ + if self.localOptions['pcap']: + daphn3Steps = daphn3.read_pcap(filename) + else: + daphn3Steps = daphn3.read_yaml(filename) + log.debug("Loaded these steps %s" % daphn3Steps) + yield daphn3Steps + + def test_daphn3(self): + host = self.localOptions['host'] + port = int(self.localOptions['port']) + + def failure(failure): + log.msg("Failed to connect") + self.report['censored'] = True + self.report['mutation'] = 0 + raise Exception("Error in connection, perhaps the backend is censored") + return + + def success(protocol): + log.msg("Successfully connected") + protocol.sendPayload() + return protocol.d + + log.msg("Connecting to %s:%s" % (host, port)) + endpoint = endpoints.TCP4ClientEndpoint(reactor, host, port) + daphn3_factory = Daphn3ClientFactory() + daphn3_factory.steps = self.input + daphn3_factory.report = self.report + d = endpoint.connect(daphn3_factory) + d.addErrback(failure) + d.addCallback(success) + return d + diff --git a/ooni/nettests/manipulation/daphne.py b/ooni/nettests/manipulation/daphne.py deleted file mode 100644 index 72b5c8f..0000000 --- a/ooni/nettests/manipulation/daphne.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- encoding: utf-8 -*- -from twisted.python import usage -from twisted.internet import protocol, endpoints, reactor - -from ooni import nettest -from ooni.kit import daphn3 -from ooni.utils import log - -class Daphn3ClientProtocol(daphn3.Daphn3Protocol): - def nextStep(self): - log.debug("Moving on to next step in the state walk") - self.current_data_received = 0 - if self.current_step >= (len(self.steps) - 1): - log.msg("Reached the end of the state machine") - log.msg("Censorship fingerprint bisected!") - step_idx, mutation_idx = self.factory.mutation - log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) - #self.transport.loseConnection() - if self.report: - self.report['mutation_idx'] = mutation_idx - self.report['step_idx'] = step_idx - self.d.callback(None) - return - else: - self.current_step += 1 - if self._current_step_role() == self.role: - # We need to send more data because we are again responsible for - # doing so. - self.sendPayload() - - -class Daphn3ClientFactory(protocol.ClientFactory): - protocol = daphn3.Daphn3Protocol - mutation = [0,0] - steps = None - - def buildProtocol(self, addr): - p = self.protocol() - p.steps = self.steps - p.factory = self - return p - - def startedConnecting(self, connector): - log.msg("Started connecting %s" % connector) - - def clientConnectionFailed(self, reason, connector): - log.err("We failed connecting the the OONIB") - log.err("Cannot perform test. Perhaps it got blocked?") - log.err("Please report this to tor-assistants(a)torproject.org") - - def clientConnectionLost(self, reason, connector): - log.err("Daphn3 client connection lost") - print reason - -class daphn3Args(usage.Options): - optParameters = [ - ['host', 'h', '127.0.0.1', 'Target Hostname'], - ['port', 'p', 57003, 'Target port number']] - - optFlags = [['pcap', 'c', 'Specify that the input file is a pcap file'], - ['yaml', 'y', 'Specify that the input file is a YAML file (default)']] - -class daphn3Test(nettest.NetTestCase): - - name = "Daphn3" - description = "Bisects the censors fingerprint by mutating the given input packets." - usageOptions = daphn3Args - inputFile = ['file', 'f', None, - 'Specify the pcap or YAML file to be used as input to the test'] - - #requiredOptions = ['file'] - requiresRoot = False - requiresTor = False - steps = None - - def inputProcessor(self, filename): - """ - step_idx is the step in the packet exchange - ex. - [.X.] are packets sent by a client or a server - - client: [.1.] [.3.] [.4.] - server: [.2.] [.5.] - - mutation_idx: is the sub index of the packet as in the byte of the - packet at the step_idx that is to be mutated - - """ - if self.localOptions['pcap']: - daphn3Steps = daphn3.read_pcap(filename) - else: - daphn3Steps = daphn3.read_yaml(filename) - log.debug("Loaded these steps %s" % daphn3Steps) - yield daphn3Steps - - def test_daphn3(self): - host = self.localOptions['host'] - port = int(self.localOptions['port']) - - def failure(failure): - log.msg("Failed to connect") - self.report['censored'] = True - self.report['mutation'] = 0 - raise Exception("Error in connection, perhaps the backend is censored") - return - - def success(protocol): - log.msg("Successfully connected") - protocol.sendPayload() - return protocol.d - - log.msg("Connecting to %s:%s" % (host, port)) - endpoint = endpoints.TCP4ClientEndpoint(reactor, host, port) - daphn3_factory = Daphn3ClientFactory() - daphn3_factory.steps = self.input - daphn3_factory.report = self.report - d = endpoint.connect(daphn3_factory) - d.addErrback(failure) - d.addCallback(success) - return d -

1 0

[ooni-probe/master] Add unittests for the DNS test template and representing of answers
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c0d2cc0d4fb087035ae57d2730f82b121b427eca Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 24 16:42:31 2016 +0100 Add unittests for the DNS test template and representing of answers --- ooni/tests/test_templates.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index cbd378c..bf05e56 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -1,9 +1,11 @@ -from ooni.templates import httpt +from ooni.templates import httpt, dnst +from twisted.names import dns from twisted.internet.error import DNSLookupError -from twisted.internet import reactor, defer +from twisted.internet import reactor, defer, base from twisted.trial import unittest +base.DelayedCall.debug = True class TestHTTPT(unittest.TestCase): def setUp(self): @@ -41,3 +43,41 @@ class TestHTTPT(unittest.TestCase): http_test._setUp() yield self.assertFailure(http_test.doRequest('http://invaliddomain/'), DNSLookupError) assert http_test.report['requests'][0]['failure'] == 'dns_lookup_error' + + +class TestDNST(unittest.TestCase): + def test_represent_answer_a(self): + a_record = dns.RRHeader(payload=dns.Record_A(address="1.1.1.1"), + type=dns.A) + self.assertEqual(dnst.representAnswer(a_record), + {'ipv4': '1.1.1.1', 'answer_type': 'A'}) + + def test_represent_answer_ptr(self): + ptr_record = dns.RRHeader(payload=dns.Record_PTR(name="example.com"), + type=dns.PTR) + self.assertEqual(dnst.representAnswer(ptr_record), + {'hostname': 'example.com', 'answer_type': 'PTR'}) + + def test_represent_answer_soa(self): + ptr_record = dns.RRHeader(payload=dns.Record_SOA(mname='example.com', + rname='foo.example.com'), + type=dns.SOA) + represented_answer = {} + represented_answer['ttl'] = None + represented_answer['answer_type'] = 'SOA' + represented_answer['hostname'] = 'example.com' + represented_answer['responsible_name'] = 'foo.example.com' + represented_answer['serial_number'] = 0 + represented_answer['refresh_interval'] = 0 + represented_answer['retry_interval'] = 0 + represented_answer['minimum_ttl'] = 0 + represented_answer['expiration_limit'] = 0 + self.assertEqual(dnst.representAnswer(ptr_record), + represented_answer) + + @defer.inlineCallbacks + def test_perform_a_lookup(self): + dns_test = dnst.DNSTest() + dns_test._setUp() + result = yield dns_test.performALookup('example.com', dns_server=('8.8.8.8', 53)) + self.assertEqual(result, ['93.184.216.34'])

1 0

[ooni-probe/master] Add checks for input file and url in http_requests test
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 510adaf6c473fb289aca6973b662f0a6cb2a53c9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Mar 24 13:50:53 2016 +0100 Add checks for input file and url in http_requests test --- ooni/nettests/blocking/http_requests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index e00b4c6..e2c9671 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -11,6 +11,8 @@ from ooni.utils.net import userAgents from ooni.templates import httpt from ooni.errors import failureToString +class MissingInput(Exception): + pass class UsageOptions(usage.Options): optParameters = [ @@ -50,6 +52,12 @@ class HTTPRequestsTest(httpt.HTTPTest): control_body_length = None experiment_body_length = None + def requirements(self): + if not self.localOptions['url'] and \ + not self.localOptions['file']: + raise MissingInput("You did not specify either a URL with -u " + "or an input file with -f") + def setUp(self): """ Check for inputs.

1 0