commit fb477a0982247974bc5f5979239ed015a17ce3ec Author: Arturo Filastò art@fuffa.org Date: Thu Dec 6 14:42:38 2012 +0100
Change the timestamp format to use ISO8601. * Write unittests for timestamp generation functions * Add documentation on how reports will be stored in the backend --- HACKING | 6 ++-- docs/source/reports.rst | 43 ++++++++++++++++++++++ ooni/config.py | 2 +- ooni/kit/__init__.py | 4 +- ooni/otime.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++ ooni/reporter.py | 14 +++++-- ooni/utils/log.py | 2 +- ooni/utils/otime.py | 51 -------------------------- oonib/report/api.py | 3 +- tests/test_otime.py | 15 ++++++++ 10 files changed, 169 insertions(+), 63 deletions(-)
diff --git a/HACKING b/HACKING index 66e6d9e..f39f1e5 100644 --- a/HACKING +++ b/HACKING @@ -75,6 +75,9 @@ Code Structure Handles running ooni.nettests as well as ooni.plugoo.tests.OONITests.
+- ooni/otime.py + Generation of timestamps, time conversions and all the rest + - ooni/kit/ In here go utilities that can be used by tests.
@@ -102,9 +105,6 @@ Code Structure Utilities for working with Tor. XXX this code should be removed and merged into txtorcon.
-- ooni/utils/otime.py - Generation of timestamps, time conversions and all the rest - - ooni/utils/txscapy.py Tools for making scapy work well with twisted.
diff --git a/docs/source/reports.rst b/docs/source/reports.rst new file mode 100644 index 0000000..cf5e008 --- /dev/null +++ b/docs/source/reports.rst @@ -0,0 +1,43 @@ +Reports +======= + +The reports collected by ooniprobe are stored on +https://ooni.torproject.org/reports/%60reportFormatVersion%60/%60CC%60/ + +Where `CC` is the two letter country code as specified by +http://en.wikipedia.org/wiki/ISO_3166-2. + +For example the reports for Italy (`CC` is `it`) of the `reportVersion` 0.1 may +be found in: + +:: + https://ooni.torproject.org/reports/0.1/IT/ + + +This directory shall contain the various reports for the test using the +following convention: + +`ISO8601`-AS`probeASNumber`.yamloo + +If a collision is detected then an int (starting with 1) will get appended to +the test. + +For example a report that was created on the first of January 2012 at Noon (UTC +time) sharp from MIT (AS3) will be stored here: + +:: + + https://ooni.torproject.org/reports/0.1/US/2012-01-01T120000Z-AS3.yamloo + + +Report format version changelog +=============================== + +In here shall go details about he major changes to the reporting format. + +version 0.1 +----------- + +Initial format version. + + diff --git a/ooni/config.py b/ooni/config.py index bc255be..b82936f 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -8,7 +8,7 @@ import yaml
from twisted.internet import reactor, threads, defer
-from ooni.utils import otime +from ooni import otime from ooni.utils import Storage
reports = Storage() diff --git a/ooni/kit/__init__.py b/ooni/kit/__init__.py index 9a24fae..85d2e3c 100644 --- a/ooni/kit/__init__.py +++ b/ooni/kit/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['domclass'] -from . import domclass +#__all__ = ['domclass'] +#from . import domclass diff --git a/ooni/otime.py b/ooni/otime.py new file mode 100644 index 0000000..0af4ec8 --- /dev/null +++ b/ooni/otime.py @@ -0,0 +1,92 @@ +""" +Here is the location for all time and date related utility functions. +""" +import time +from datetime import datetime + +def utcDateNow(): + """ + Returns the datetime object of the current UTC time. + """ + return datetime.utcnow() + +def utcTimeNow(): + """ + Returns seconds since epoch in UTC time, it's of type float. + """ + return time.mktime(time.gmtime()) + +def dateToTime(date): + """ + Takes as input a datetime object and outputs the seconds since epoch. + """ + return time.mktime(date.timetuple()) + +def prettyDateNow(): + """ + Returns a good looking string for the local time. + """ + return datetime.now().ctime() + +def utcPrettyDateNow(): + """ + Returns a good looking string for utc time. + """ + return datetime.utcnow().ctime() + +def timeToPrettyDate(time_val): + return time.ctime(time_val) + +class InvalidTimestampFormat(Exception): + pass + +def fromTimestamp(s): + """ + Converts a string that is output from the timestamp function back to a + datetime object + + Args: + s (str): a ISO8601 formatted string. + ex. 1912-06-23T101234Z" + + Note: we currently only support parsing strings that are generated from the + timestamp function and have no intention in supporting the full standard. + """ + try: + date_part, time_part = s.split('T') + hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6] + year, month, day = date_part.split('-') + except: + raise InvalidTimestampFormat(s) + + return datetime(int(year), int(month), int(day), int(hours), int(minutes), + int(seconds)) + +def timestamp(t=None): + """ + The timestamp for ooni reports follows ISO 8601 in + UTC time format. + We do not inlcude ':' and include seconds. + + Example: + + if the current date is "10:12:34 AM, June 23 1912" (datetime(1912, 6, + 23, 10, 12, 34)) + + the timestamp will be: + + "1912-06-23T101234Z" + + Args: + t (datetime): a datetime object representing the + time to be represented (*MUST* be expressed + in UTC). + + If not specified will default to the current time + in UTC. + """ + if not t: + t = datetime.utcnow() + ISO8601 = "%Y-%m-%dT%H%M%SZ" + return t.strftime(ISO8601) + diff --git a/ooni/reporter.py b/ooni/reporter.py index abbd9c9..29e6049 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -20,16 +20,22 @@ from yaml.representer import * from yaml.emitter import * from yaml.serializer import * from yaml.resolver import * - -from scapy.packet import Packet - from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError
+from ooni.utils import log + +try: + from scapy.packet import Packet +except ImportError: + log.err("Scapy is not installed.") + + +from ooni import otime +from ooni.utils import geodata from ooni.utils.net import BodyReceiver, StringProducer, userAgents -from ooni.utils import otime, log, geodata
from ooni import config
diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 170d7b5..3e24804 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -12,7 +12,7 @@ from twisted.python import log as txlog from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile
-from ooni.utils import otime +from ooni import otime from ooni import config
## Get rid of the annoying "No route found for diff --git a/ooni/utils/otime.py b/ooni/utils/otime.py deleted file mode 100644 index 719230e..0000000 --- a/ooni/utils/otime.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -""" -Here is the location for all time and date related utility functions. -""" -import time -from datetime import datetime - -def utcDateNow(): - """ - Returns the datetime object of the current UTC time. - """ - return datetime.utcnow() - -def utcTimeNow(): - """ - Returns seconds since epoch in UTC time, it's of type float. - """ - return time.mktime(time.gmtime()) - -def dateToTime(date): - """ - Takes as input a datetime object and outputs the seconds since epoch. - """ - return time.mktime(date.timetuple()) - -def prettyDateNow(): - """ - Returns a good looking string for the local time. - """ - return datetime.now().ctime() - -def utcPrettyDateNow(): - """ - Returns a good looking string for utc time. - """ - return datetime.utcnow().ctime() - -def timeToPrettyDate(time_val): - return time.ctime(time_val) - -def timestamp(): - cur_time = datetime.utcnow() - d_format = "%d_%B_%Y_%H-%M-%S" - pretty = cur_time.strftime(d_format) - return pretty - - diff --git a/oonib/report/api.py b/oonib/report/api.py index 9c42a54..14a4bc1 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -16,7 +16,8 @@ from twisted.internet import reactor, defer
from cyclone import web
-from ooni.utils import randomStr, otime +from ooni import otime +from ooni.utils import randomStr from oonib import models, config from oonib.report import file_collector
diff --git a/tests/test_otime.py b/tests/test_otime.py new file mode 100644 index 0000000..80979f2 --- /dev/null +++ b/tests/test_otime.py @@ -0,0 +1,15 @@ +import unittest +from datetime import datetime +from ooni import otime + +test_date = datetime(2002, 6, 26, 22, 45, 49) + +class TestOtime(unittest.TestCase): + def test_timestamp(self): + self.assertEqual(otime.timestamp(test_date), "2002-06-26T224549Z") + + def test_fromTimestamp(self): + time_stamp = otime.timestamp(test_date) + self.assertEqual(test_date, otime.fromTimestamp(time_stamp)) + +