[tor-commits] [ooni-probe/master] Change the timestamp format to use ISO8601.
art at torproject.org
art at torproject.org
Thu Dec 6 13:45:56 UTC 2012
commit fb477a0982247974bc5f5979239ed015a17ce3ec
Author: Arturo Filastò <art at fuffa.org>
Date: Thu Dec 6 14:42:38 2012 +0100
Change the timestamp format to use ISO8601.
* Write unittests for timestamp generation functions
* Add documentation on how reports will be stored in the backend
---
HACKING | 6 ++--
docs/source/reports.rst | 43 ++++++++++++++++++++++
ooni/config.py | 2 +-
ooni/kit/__init__.py | 4 +-
ooni/otime.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++
ooni/reporter.py | 14 +++++--
ooni/utils/log.py | 2 +-
ooni/utils/otime.py | 51 --------------------------
oonib/report/api.py | 3 +-
tests/test_otime.py | 15 ++++++++
10 files changed, 169 insertions(+), 63 deletions(-)
diff --git a/HACKING b/HACKING
index 66e6d9e..f39f1e5 100644
--- a/HACKING
+++ b/HACKING
@@ -75,6 +75,9 @@ Code Structure
Handles running ooni.nettests as well as
ooni.plugoo.tests.OONITests.
+- ooni/otime.py
+ Generation of timestamps, time conversions and all the rest
+
- ooni/kit/
In here go utilities that can be used by tests.
@@ -102,9 +105,6 @@ Code Structure
Utilities for working with Tor.
XXX this code should be removed and merged into txtorcon.
-- ooni/utils/otime.py
- Generation of timestamps, time conversions and all the rest
-
- ooni/utils/txscapy.py
Tools for making scapy work well with twisted.
diff --git a/docs/source/reports.rst b/docs/source/reports.rst
new file mode 100644
index 0000000..cf5e008
--- /dev/null
+++ b/docs/source/reports.rst
@@ -0,0 +1,43 @@
+Reports
+=======
+
+The reports collected by ooniprobe are stored on
+https://ooni.torproject.org/reports/`reportFormatVersion`/`CC`/
+
+Where `CC` is the two letter country code as specified by
+http://en.wikipedia.org/wiki/ISO_3166-2.
+
+For example the reports for Italy (`CC` is `it`) of the `reportVersion` 0.1 may
+be found in:
+
+::
+ https://ooni.torproject.org/reports/0.1/IT/
+
+
+This directory shall contain the various reports for the test using the
+following convention:
+
+`ISO8601`-AS`probeASNumber`.yamloo
+
+If a collision is detected then an int (starting with 1) will get appended to
+the test.
+
+For example a report that was created on the first of January 2012 at Noon (UTC
+time) sharp from MIT (AS3) will be stored here:
+
+::
+
+ https://ooni.torproject.org/reports/0.1/US/2012-01-01T120000Z-AS3.yamloo
+
+
+Report format version changelog
+===============================
+
+In here shall go details about he major changes to the reporting format.
+
+version 0.1
+-----------
+
+Initial format version.
+
+
diff --git a/ooni/config.py b/ooni/config.py
index bc255be..b82936f 100644
--- a/ooni/config.py
+++ b/ooni/config.py
@@ -8,7 +8,7 @@ import yaml
from twisted.internet import reactor, threads, defer
-from ooni.utils import otime
+from ooni import otime
from ooni.utils import Storage
reports = Storage()
diff --git a/ooni/kit/__init__.py b/ooni/kit/__init__.py
index 9a24fae..85d2e3c 100644
--- a/ooni/kit/__init__.py
+++ b/ooni/kit/__init__.py
@@ -1,2 +1,2 @@
-__all__ = ['domclass']
-from . import domclass
+#__all__ = ['domclass']
+#from . import domclass
diff --git a/ooni/otime.py b/ooni/otime.py
new file mode 100644
index 0000000..0af4ec8
--- /dev/null
+++ b/ooni/otime.py
@@ -0,0 +1,92 @@
+"""
+Here is the location for all time and date related utility functions.
+"""
+import time
+from datetime import datetime
+
+def utcDateNow():
+ """
+ Returns the datetime object of the current UTC time.
+ """
+ return datetime.utcnow()
+
+def utcTimeNow():
+ """
+ Returns seconds since epoch in UTC time, it's of type float.
+ """
+ return time.mktime(time.gmtime())
+
+def dateToTime(date):
+ """
+ Takes as input a datetime object and outputs the seconds since epoch.
+ """
+ return time.mktime(date.timetuple())
+
+def prettyDateNow():
+ """
+ Returns a good looking string for the local time.
+ """
+ return datetime.now().ctime()
+
+def utcPrettyDateNow():
+ """
+ Returns a good looking string for utc time.
+ """
+ return datetime.utcnow().ctime()
+
+def timeToPrettyDate(time_val):
+ return time.ctime(time_val)
+
+class InvalidTimestampFormat(Exception):
+ pass
+
+def fromTimestamp(s):
+ """
+ Converts a string that is output from the timestamp function back to a
+ datetime object
+
+ Args:
+ s (str): a ISO8601 formatted string.
+ ex. 1912-06-23T101234Z"
+
+ Note: we currently only support parsing strings that are generated from the
+ timestamp function and have no intention in supporting the full standard.
+ """
+ try:
+ date_part, time_part = s.split('T')
+ hours, minutes, seconds = time_part[:2], time_part[2:4], time_part[4:6]
+ year, month, day = date_part.split('-')
+ except:
+ raise InvalidTimestampFormat(s)
+
+ return datetime(int(year), int(month), int(day), int(hours), int(minutes),
+ int(seconds))
+
+def timestamp(t=None):
+ """
+ The timestamp for ooni reports follows ISO 8601 in
+ UTC time format.
+ We do not inlcude ':' and include seconds.
+
+ Example:
+
+ if the current date is "10:12:34 AM, June 23 1912" (datetime(1912, 6,
+ 23, 10, 12, 34))
+
+ the timestamp will be:
+
+ "1912-06-23T101234Z"
+
+ Args:
+ t (datetime): a datetime object representing the
+ time to be represented (*MUST* be expressed
+ in UTC).
+
+ If not specified will default to the current time
+ in UTC.
+ """
+ if not t:
+ t = datetime.utcnow()
+ ISO8601 = "%Y-%m-%dT%H%M%SZ"
+ return t.strftime(ISO8601)
+
diff --git a/ooni/reporter.py b/ooni/reporter.py
index abbd9c9..29e6049 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -20,16 +20,22 @@ from yaml.representer import *
from yaml.emitter import *
from yaml.serializer import *
from yaml.resolver import *
-
-from scapy.packet import Packet
-
from twisted.python.util import untilConcludes
from twisted.trial import reporter
from twisted.internet import defer, reactor
from twisted.internet.error import ConnectionRefusedError
+from ooni.utils import log
+
+try:
+ from scapy.packet import Packet
+except ImportError:
+ log.err("Scapy is not installed.")
+
+
+from ooni import otime
+from ooni.utils import geodata
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
-from ooni.utils import otime, log, geodata
from ooni import config
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 170d7b5..3e24804 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -12,7 +12,7 @@ from twisted.python import log as txlog
from twisted.python.failure import Failure
from twisted.python.logfile import DailyLogFile
-from ooni.utils import otime
+from ooni import otime
from ooni import config
## Get rid of the annoying "No route found for
diff --git a/ooni/utils/otime.py b/ooni/utils/otime.py
deleted file mode 100644
index 719230e..0000000
--- a/ooni/utils/otime.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# :authors: Arturo Filastò
-# :licence: see LICENSE
-
-"""
-Here is the location for all time and date related utility functions.
-"""
-import time
-from datetime import datetime
-
-def utcDateNow():
- """
- Returns the datetime object of the current UTC time.
- """
- return datetime.utcnow()
-
-def utcTimeNow():
- """
- Returns seconds since epoch in UTC time, it's of type float.
- """
- return time.mktime(time.gmtime())
-
-def dateToTime(date):
- """
- Takes as input a datetime object and outputs the seconds since epoch.
- """
- return time.mktime(date.timetuple())
-
-def prettyDateNow():
- """
- Returns a good looking string for the local time.
- """
- return datetime.now().ctime()
-
-def utcPrettyDateNow():
- """
- Returns a good looking string for utc time.
- """
- return datetime.utcnow().ctime()
-
-def timeToPrettyDate(time_val):
- return time.ctime(time_val)
-
-def timestamp():
- cur_time = datetime.utcnow()
- d_format = "%d_%B_%Y_%H-%M-%S"
- pretty = cur_time.strftime(d_format)
- return pretty
-
-
diff --git a/oonib/report/api.py b/oonib/report/api.py
index 9c42a54..14a4bc1 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -16,7 +16,8 @@ from twisted.internet import reactor, defer
from cyclone import web
-from ooni.utils import randomStr, otime
+from ooni import otime
+from ooni.utils import randomStr
from oonib import models, config
from oonib.report import file_collector
diff --git a/tests/test_otime.py b/tests/test_otime.py
new file mode 100644
index 0000000..80979f2
--- /dev/null
+++ b/tests/test_otime.py
@@ -0,0 +1,15 @@
+import unittest
+from datetime import datetime
+from ooni import otime
+
+test_date = datetime(2002, 6, 26, 22, 45, 49)
+
+class TestOtime(unittest.TestCase):
+ def test_timestamp(self):
+ self.assertEqual(otime.timestamp(test_date), "2002-06-26T224549Z")
+
+ def test_fromTimestamp(self):
+ time_stamp = otime.timestamp(test_date)
+ self.assertEqual(test_date, otime.fromTimestamp(time_stamp))
+
+
More information about the tor-commits
mailing list