commit 58db73a6d8c7d5296262cfa8ea2ce6e987ef724e Author: Arturo Filastò arturo@filasto.net Date: Thu Mar 17 15:49:08 2016 +0100

Implement measurement_start_time and test_start_time in client --- ooni/nettest.py | 11 ++++++----- ooni/otime.py | 3 +++ ooni/reporter.py | 4 +++- ooni/tasks.py | 5 ++--- ooni/tests/test_director.py | 6 ++---- ooni/tests/test_reporter.py | 2 +- ooni/tests/test_utils.py | 10 +++++----- ooni/utils/__init__.py | 6 +++--- 8 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py index 9bdc6d9..9c5e171 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -197,8 +197,7 @@ class NetTestLoader(object):

return input_files

- @property - def testDetails(self): + def setTestDetails(self): from ooni import __version__ as software_version

input_file_hashes = [] @@ -206,8 +205,8 @@ class NetTestLoader(object): input_file_hashes.append(input_file['hash'])

options = sanitize_options(self.options) - test_details = { - 'start_time': otime.epochToUTC(time.time()), + self.testDetails = { + 'test_start_time': otime.UTCTimestampNow(), 'probe_asn': config.probe_ip.geodata['asn'], 'probe_cc': config.probe_ip.geodata['countrycode'], 'probe_ip': config.probe_ip.geodata['ip'], @@ -223,7 +222,6 @@ class NetTestLoader(object): 'annotations': self.annotations, 'data_format_version': '0.2.0' } - return test_details

def _parseNetTestOptions(self, klass): """ @@ -351,6 +349,9 @@ class NetTestLoader(object):

if options: klass.localOptions = options + # XXX this class all needs to be refactored and this is kind of a + # hack. + self.setTestDetails()

test_instance = klass() if test_instance.requiresRoot and not hasRawSocketPermission(): diff --git a/ooni/otime.py b/ooni/otime.py index 1406fbf..37ada02 100644 --- a/ooni/otime.py +++ b/ooni/otime.py @@ -85,3 +85,6 @@ def epochToTimestamp(seconds):

def epochToUTC(seconds): return float(datetime.utcfromtimestamp(seconds).strftime("%s")) + +def UTCTimestampNow(): + return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") diff --git a/ooni/reporter.py b/ooni/reporter.py index d9d147c..d772ae9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -267,6 +267,7 @@ class OONIBReporter(OReporter): 'input': entry.testInstance.report.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.testInstance.report.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.testInstance.report.pop('test_runtime', None), 'test_keys': entry.testInstance.report } @@ -275,6 +276,7 @@ class OONIBReporter(OReporter): 'input': entry.pop('input', None), 'id': str(uuid.uuid4()), 'test_start_time': entry.pop('test_start_time', None), + 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), 'test_runtime': entry.pop('test_runtime', None), 'test_keys': entry } @@ -352,7 +354,7 @@ class OONIBReporter(OReporter): 'probe_cc': self.testDetails['probe_cc'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], - 'start_time': self.testDetails['start_time'], + 'test_start_time': self.testDetails['test_start_time'], 'input_hashes': self.testDetails['input_hashes'], 'data_format_version': self.testDetails['data_format_version'], 'format': 'json' diff --git a/ooni/tasks.py b/ooni/tasks.py index 3f331a2..b5a15e0 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -116,9 +116,8 @@ class Measurement(TaskWithTimeout):

if 'input' not in self.testInstance.report.keys(): self.testInstance.report['input'] = test_input - if 'test_start_time' not in self.testInstance.report.keys(): - start_time = otime.epochToUTC(self.testInstance._start_time) - self.testInstance.report['test_start_time'] = start_time + if 'measurement_start_time' not in self.testInstance.report.keys(): + self.testInstance.report['measurement_start_time'] = otime.UTCTimestampNow()

self.testInstance.setUp()

diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 38e8481..61d504c 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -1,5 +1,3 @@ -import time - from mock import patch, MagicMock

from ooni.settings import config @@ -60,7 +58,7 @@ class TestStartSniffing(unittest.TestCase): self.director = Director() self.testDetails = { 'test_name': 'foo', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' }

# Each NetTestCase has a name attribute @@ -85,7 +83,7 @@ class TestStartSniffing(unittest.TestCase):

self.testDetails = { 'test_name': 'bar', - 'start_time': time.time() + 'test_start_time': '2016-01-01 12:34:56' } with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: sniffer = mock_scapy_sniffer.return_value diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index 4b99584..5adb4af 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -26,7 +26,7 @@ test_details = { 'input_hashes': [], 'probe_asn': 'AS0', 'probe_cc': 'ZZ', - 'start_time': time.time(), + 'test_start_time': '2016-01-01 22:33:11', 'data_format_version': '0.2.0' }

diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index e2e448a..855eb19 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -8,7 +8,7 @@ class TestUtils(unittest.TestCase): def setUp(self): self.test_details = { 'test_name': 'foo', - 'start_time': 441763200 + 'test_start_time': '2016-01-01 01:22:22' } self.extension = 'ext' self.prefix = 'prefix' @@ -46,19 +46,19 @@ class TestUtils(unittest.TestCase):

def test_generate_filename(self): filename = generate_filename(self.test_details) - self.assertEqual(filename, 'foo-1984-01-01T000000Z') + self.assertEqual(filename, 'foo-2016-01-01T012222Z')

def test_generate_filename_with_extension(self): filename = generate_filename(self.test_details, extension=self.extension) - self.assertEqual(filename, 'foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext')

def test_generate_filename_with_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z')

def test_generate_filename_with_extension_and_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension) - self.assertEqual(filename, 'prefix-foo-1984-01-01T000000Z.ext') + self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext')

def test_generate_filename_with_filename(self): filename = generate_filename(self.test_details, filename=self.filename) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5e9616b..78a1c7b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -3,12 +3,12 @@ import string import random import glob import os +from datetime import datetime

import gzip from base64 import b64encode from zipfile import ZipFile

-from ooni import otime from ooni import errors

@@ -122,8 +122,8 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): extension. """ if filename is None: - test_name, start_time = testDetails['test_name'], testDetails['start_time'] - start_time = otime.epochToTimestamp(start_time) + test_name, start_time = testDetails['test_name'], testDetails['test_start_time'] + start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H%M%SZ") suffix = "%s-%s" % (test_name, start_time) basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix final_filename = '%s.%s' % (basename, extension) if extension is not None else basename