tor-commits
Threads by month
- ----- 2026 -----
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2025 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- 1 participants
- 216171 discussions
[stem/master] Marking stem.util modules and functions as being private
by atagar@torproject.org 10 Mar '13
by atagar@torproject.org 10 Mar '13
10 Mar '13
commit a7fae56f9dab6cf766699c229edb2fd0531b7bd9
Merge: d45f89d aab3bf5
Author: Damian Johnson <atagar(a)torproject.org>
Date: Sat Mar 9 23:10:26 2013 -0800
Marking stem.util modules and functions as being private
The stem.util space has several things that I'd rather not vend. Things that
are either likely to change in the future or simply don't make sense outside of
stem's internal use. Marking functions and some entire modules as being
private.
docs/api.rst | 4 -
stem/connection.py | 14 ++--
stem/control.py | 4 +-
stem/descriptor/__init__.py | 8 +-
stem/descriptor/extrainfo_descriptor.py | 4 +-
stem/descriptor/networkstatus.py | 10 ++--
stem/descriptor/router_status_entry.py | 6 +-
stem/descriptor/server_descriptor.py | 10 ++--
stem/exit_policy.py | 16 +++---
stem/response/authchallenge.py | 4 +-
stem/response/events.py | 4 +-
stem/socket.py | 6 +-
stem/util/conf.py | 2 +-
stem/util/connection.py | 86 +++++++++++++---------------
stem/util/enum.py | 2 +-
stem/util/log.py | 4 +
stem/util/ordereddict.py | 3 +
stem/util/proc.py | 3 +
stem/util/str_tools.py | 27 ++++-----
stem/util/system.py | 2 +-
stem/util/term.py | 5 +-
stem/util/tor_tools.py | 3 +
test/integ/control/controller.py | 2 +-
test/network.py | 2 +-
test/settings.cfg | 4 +-
test/unit/descriptor/server_descriptor.py | 2 +-
test/unit/util/connection.py | 60 ++++++++++----------
test/unit/util/str_tools.py | 22 ++++----
28 files changed, 160 insertions(+), 159 deletions(-)
1
0
[stem/master] Marking stem.util.str_tools functions as being private
by atagar@torproject.org 10 Mar '13
by atagar@torproject.org 10 Mar '13
10 Mar '13
commit aab3bf57a87bb09d4d572195e568e73da489f38a
Author: Damian Johnson <atagar(a)torproject.org>
Date: Sat Mar 9 23:07:23 2013 -0800
Marking stem.util.str_tools functions as being private
Swapping several of the functions that we don't want to vend as being private.
The to_bytes() and to_unicode() functions in particular are simply there for
python 3.x support, so they'll disappear if/when we drop python 2.x support.
---
stem/connection.py | 8 ++++----
stem/descriptor/__init__.py | 8 ++++----
stem/descriptor/extrainfo_descriptor.py | 2 +-
stem/descriptor/networkstatus.py | 4 ++--
stem/descriptor/router_status_entry.py | 2 +-
stem/descriptor/server_descriptor.py | 6 +++---
stem/response/authchallenge.py | 4 ++--
stem/response/events.py | 4 ++--
stem/socket.py | 6 +++---
stem/util/enum.py | 2 +-
stem/util/str_tools.py | 27 +++++++++++----------------
stem/util/system.py | 2 +-
stem/util/term.py | 2 +-
test/settings.cfg | 4 ++--
test/unit/descriptor/server_descriptor.py | 2 +-
test/unit/util/str_tools.py | 22 +++++++++++-----------
16 files changed, 50 insertions(+), 55 deletions(-)
diff --git a/stem/connection.py b/stem/connection.py
index dc70bce..4f63378 100644
--- a/stem/connection.py
+++ b/stem/connection.py
@@ -596,8 +596,8 @@ def authenticate_cookie(controller, cookie_path, suppress_ctl_errors = True):
# This seems dumb but oh well. Converting the result to unicode so it won't
# misbehave.
- auth_token_hex = binascii.b2a_hex(stem.util.str_tools.to_bytes(cookie_data))
- msg = "AUTHENTICATE %s" % stem.util.str_tools.to_unicode(auth_token_hex)
+ auth_token_hex = binascii.b2a_hex(stem.util.str_tools._to_bytes(cookie_data))
+ msg = "AUTHENTICATE %s" % stem.util.str_tools._to_unicode(auth_token_hex)
auth_response = _msg(controller, msg)
# if we got anything but an OK response then error
@@ -692,7 +692,7 @@ def authenticate_safecookie(controller, cookie_path, suppress_ctl_errors = True)
client_nonce = os.urandom(32)
try:
- client_nonce_hex = binascii.b2a_hex(stem.util.str_tools.to_bytes(client_nonce))
+ client_nonce_hex = binascii.b2a_hex(stem.util.str_tools._to_bytes(client_nonce))
authchallenge_response = _msg(controller, "AUTHCHALLENGE SAFECOOKIE %s" % client_nonce_hex)
if not authchallenge_response.is_ok():
@@ -744,7 +744,7 @@ def authenticate_safecookie(controller, cookie_path, suppress_ctl_errors = True)
CLIENT_HASH_CONSTANT,
cookie_data + client_nonce + authchallenge_response.server_nonce)
- auth_response = _msg(controller, "AUTHENTICATE %s" % (binascii.b2a_hex(stem.util.str_tools.to_bytes(client_hash))))
+ auth_response = _msg(controller, "AUTHENTICATE %s" % (binascii.b2a_hex(stem.util.str_tools._to_bytes(client_hash))))
except stem.ControllerError, exc:
try:
controller.connect()
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 59d66a4..4f0a596 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -307,7 +307,7 @@ class Descriptor(object):
if stem.prereq.is_python_3():
return self._raw_contents
else:
- return str(stem.util.str_tools.to_bytes(self._raw_contents))
+ return str(stem.util.str_tools._to_bytes(self._raw_contents))
class _UnicodeReader(object):
@@ -333,10 +333,10 @@ class _UnicodeReader(object):
return self.wrapped_file.next()
def read(self, n = -1):
- return stem.util.str_tools.to_unicode(self.wrapped_file.read(n))
+ return stem.util.str_tools._to_unicode(self.wrapped_file.read(n))
def readline(self):
- return stem.util.str_tools.to_unicode(self.wrapped_file.readline())
+ return stem.util.str_tools._to_unicode(self.wrapped_file.readline())
def readlines(self, sizehint = None):
# being careful to do in-place conversion so we don't accidently double our
@@ -348,7 +348,7 @@ class _UnicodeReader(object):
results = self.wrapped_file.readlines()
for i in xrange(len(results)):
- results[i] = stem.util.str_tools.to_unicode(results[i])
+ results[i] = stem.util.str_tools._to_unicode(results[i])
return results
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 3d7bb3c..5f1c423 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -825,7 +825,7 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor):
# our digest is calculated from everything except our signature
raw_content, ending = str(self), "\nrouter-signature\n"
raw_content = raw_content[:raw_content.find(ending) + len(ending)]
- self._digest = hashlib.sha1(stem.util.str_tools.to_bytes(raw_content)).hexdigest().upper()
+ self._digest = hashlib.sha1(stem.util.str_tools._to_bytes(raw_content)).hexdigest().upper()
return self._digest
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index e032f76..de0b945 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -265,7 +265,7 @@ class NetworkStatusDocumentV2(NetworkStatusDocument):
"""
def __init__(self, raw_content, validate = True):
- raw_content = stem.util.str_tools.to_unicode(raw_content)
+ raw_content = stem.util.str_tools._to_unicode(raw_content)
super(NetworkStatusDocumentV2, self).__init__(raw_content)
self.version = None
@@ -464,7 +464,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):
:raises: **ValueError** if the document is invalid
"""
- raw_content = stem.util.str_tools.to_unicode(raw_content)
+ raw_content = stem.util.str_tools._to_unicode(raw_content)
super(NetworkStatusDocumentV3, self).__init__(raw_content)
document_file = StringIO.StringIO(raw_content)
diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py
index ef810ec..68c1b38 100644
--- a/stem/descriptor/router_status_entry.py
+++ b/stem/descriptor/router_status_entry.py
@@ -703,7 +703,7 @@ def _decode_fingerprint(identity, validate):
fingerprint = ""
try:
- identity_decoded = base64.b64decode(stem.util.str_tools.to_bytes(identity))
+ identity_decoded = base64.b64decode(stem.util.str_tools._to_bytes(identity))
except (TypeError, binascii.Error):
if not validate:
return None
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 298b480..89795c9 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -665,7 +665,7 @@ class RelayDescriptor(ServerDescriptor):
if start >= 0 and sig_start > 0 and end > start:
for_digest = raw_descriptor[start:end]
- digest_hash = hashlib.sha1(stem.util.str_tools.to_bytes(for_digest))
+ digest_hash = hashlib.sha1(stem.util.str_tools._to_bytes(for_digest))
self._digest = digest_hash.hexdigest().upper()
else:
raise ValueError("unable to calculate digest for descriptor")
@@ -686,7 +686,7 @@ class RelayDescriptor(ServerDescriptor):
if self.fingerprint:
# calculate the signing key hash
- key_der_as_hash = hashlib.sha1(stem.util.str_tools.to_bytes(key_as_bytes)).hexdigest()
+ key_der_as_hash = hashlib.sha1(stem.util.str_tools._to_bytes(key_as_bytes)).hexdigest()
if key_der_as_hash != self.fingerprint.lower():
log.warn("Signing key hash: %s != fingerprint: %s" % (key_der_as_hash, self.fingerprint.lower()))
@@ -821,7 +821,7 @@ class RelayDescriptor(ServerDescriptor):
# get the key representation in bytes
- key_bytes = base64.b64decode(stem.util.str_tools.to_bytes(key_as_string))
+ key_bytes = base64.b64decode(stem.util.str_tools._to_bytes(key_as_string))
return key_bytes
diff --git a/stem/response/authchallenge.py b/stem/response/authchallenge.py
index c14369b..93be385 100644
--- a/stem/response/authchallenge.py
+++ b/stem/response/authchallenge.py
@@ -41,7 +41,7 @@ class AuthChallengeResponse(stem.response.ControlMessage):
if not stem.util.tor_tools.is_hex_digits(value, 64):
raise stem.ProtocolError("SERVERHASH has an invalid value: %s" % value)
- self.server_hash = binascii.a2b_hex(stem.util.str_tools.to_bytes(value))
+ self.server_hash = binascii.a2b_hex(stem.util.str_tools._to_bytes(value))
else:
raise stem.ProtocolError("Missing SERVERHASH mapping: %s" % line)
@@ -51,6 +51,6 @@ class AuthChallengeResponse(stem.response.ControlMessage):
if not stem.util.tor_tools.is_hex_digits(value, 64):
raise stem.ProtocolError("SERVERNONCE has an invalid value: %s" % value)
- self.server_nonce = binascii.a2b_hex(stem.util.str_tools.to_bytes(value))
+ self.server_nonce = binascii.a2b_hex(stem.util.str_tools._to_bytes(value))
else:
raise stem.ProtocolError("Missing SERVERNONCE mapping: %s" % line)
diff --git a/stem/response/events.py b/stem/response/events.py
index af8c8e5..a2c43ff 100644
--- a/stem/response/events.py
+++ b/stem/response/events.py
@@ -345,7 +345,7 @@ class CircuitEvent(Event):
if self.created is not None:
try:
- self.created = str_tools.parse_iso_timestamp(self.created)
+ self.created = str_tools._parse_iso_timestamp(self.created)
except ValueError, exc:
raise stem.ProtocolError("Unable to parse create date (%s): %s" % (exc, self))
@@ -401,7 +401,7 @@ class CircMinorEvent(Event):
if self.created is not None:
try:
- self.created = str_tools.parse_iso_timestamp(self.created)
+ self.created = str_tools._parse_iso_timestamp(self.created)
except ValueError, exc:
raise stem.ProtocolError("Unable to parse create date (%s): %s" % (exc, self))
diff --git a/stem/socket.py b/stem/socket.py
index 3b9498c..d459fb7 100644
--- a/stem/socket.py
+++ b/stem/socket.py
@@ -425,7 +425,7 @@ def send_message(control_file, message, raw = False):
message = send_formatting(message)
try:
- control_file.write(stem.util.str_tools.to_bytes(message))
+ control_file.write(stem.util.str_tools._to_bytes(message))
control_file.flush()
log_message = message.replace("\r\n", "\n").rstrip()
@@ -473,7 +473,7 @@ def recv_message(control_file):
line = control_file.readline()
if stem.prereq.is_python_3():
- line = stem.util.str_tools.to_unicode(line)
+ line = stem.util.str_tools._to_unicode(line)
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
@@ -542,7 +542,7 @@ def recv_message(control_file):
line = control_file.readline()
if stem.prereq.is_python_3():
- line = stem.util.str_tools.to_unicode(line)
+ line = stem.util.str_tools._to_unicode(line)
except socket.error, exc:
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")" % (exc, log.escape(raw_content)))
diff --git a/stem/util/enum.py b/stem/util/enum.py
index bd983ee..504af48 100644
--- a/stem/util/enum.py
+++ b/stem/util/enum.py
@@ -75,7 +75,7 @@ class Enum(object):
for entry in args:
if isinstance(entry, str):
- key, val = entry, stem.util.str_tools.to_camel_case(entry)
+ key, val = entry, stem.util.str_tools._to_camel_case(entry)
elif isinstance(entry, tuple) and len(entry) == 2:
key, val = entry
else:
diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py
index 14198b4..d575cfa 100644
--- a/stem/util/str_tools.py
+++ b/stem/util/str_tools.py
@@ -8,16 +8,11 @@ Toolkit for various string activity.
::
- to_bytes - normalizes string ASCII bytes
- to_unicode - normalizes string to unicode
- to_camel_case - converts a string to camel case
get_size_label - human readable label for a number of bytes
get_time_label - human readable label for a number of seconds
get_time_labels - human readable labels for each time unit
get_short_time_label - condensed time label output
parse_short_time_label - seconds represented by a short time label
-
- parse_iso_timestamp - parses an ISO timestamp as a datetime value
"""
import codecs
@@ -53,32 +48,32 @@ TIME_UNITS = (
)
if stem.prereq.is_python_3():
- def _to_bytes(msg):
+ def _to_bytes_impl(msg):
if isinstance(msg, str):
return codecs.latin_1_encode(msg, "replace")[0]
else:
return msg
- def _to_unicode(msg):
+ def _to_unicode_impl(msg):
if msg is not None and not isinstance(msg, str):
return msg.decode("utf-8", "replace")
else:
return msg
else:
- def _to_bytes(msg):
+ def _to_bytes_impl(msg):
if msg is not None and isinstance(msg, unicode):
return codecs.latin_1_encode(msg, "replace")[0]
else:
return msg
- def _to_unicode(msg):
+ def _to_unicode_impl(msg):
if msg is not None and not isinstance(msg, unicode):
return msg.decode("utf-8", "replace")
else:
return msg
-def to_bytes(msg):
+def _to_bytes(msg):
"""
Provides the ASCII bytes for the given string. This is purely to provide
python 3 compatability, normalizing the unicode/ASCII change in the version
@@ -91,10 +86,10 @@ def to_bytes(msg):
:returns: ASCII bytes for string
"""
- return _to_bytes(msg)
+ return _to_bytes_impl(msg)
-def to_unicode(msg):
+def _to_unicode(msg):
"""
Provides the unicode string for the given ASCII bytes. This is purely to
provide python 3 compatability, normalizing the unicode/ASCII change in the
@@ -105,16 +100,16 @@ def to_unicode(msg):
:returns: unicode conversion
"""
- return _to_unicode(msg)
+ return _to_unicode_impl(msg)
-def to_camel_case(label, divider = "_", joiner = " "):
+def _to_camel_case(label, divider = "_", joiner = " "):
"""
Converts the given string to camel case, ie:
::
- >>> to_camel_case("I_LIKE_PEPPERJACK!")
+ >>> _to_camel_case("I_LIKE_PEPPERJACK!")
'I Like Pepperjack!'
:param str label: input string to be converted
@@ -315,7 +310,7 @@ def parse_short_time_label(label):
raise ValueError("Non-numeric value in time entry: %s" % label)
-def parse_iso_timestamp(entry):
+def _parse_iso_timestamp(entry):
"""
Parses the ISO 8601 standard that provides for timestamps like...
diff --git a/stem/util/system.py b/stem/util/system.py
index 688475f..0a94b57 100644
--- a/stem/util/system.py
+++ b/stem/util/system.py
@@ -814,7 +814,7 @@ def _set_prctl_name(process_name):
libc = ctypes.CDLL(ctypes.util.find_library("c"))
name_buffer = ctypes.create_string_buffer(len(process_name) + 1)
- name_buffer.value = stem.util.str_tools.to_bytes(process_name)
+ name_buffer.value = stem.util.str_tools._to_bytes(process_name)
libc.prctl(PR_SET_NAME, ctypes.byref(name_buffer), 0, 0, 0)
diff --git a/stem/util/term.py b/stem/util/term.py
index 5f54b5d..05efcfe 100644
--- a/stem/util/term.py
+++ b/stem/util/term.py
@@ -87,7 +87,7 @@ def format(msg, *attr):
encodings = []
for text_attr in attr:
- text_attr, encoding = stem.util.str_tools.to_camel_case(text_attr), None
+ text_attr, encoding = stem.util.str_tools._to_camel_case(text_attr), None
encoding = FG_ENCODING.get(text_attr, encoding)
encoding = BG_ENCODING.get(text_attr, encoding)
encoding = ATTR_ENCODING.get(text_attr, encoding)
diff --git a/test/settings.cfg b/test/settings.cfg
index c184eec..8e96214 100644
--- a/test/settings.cfg
+++ b/test/settings.cfg
@@ -158,8 +158,8 @@ pyflakes.ignore stem/prereq.py => 'RSA' imported but unused
pyflakes.ignore stem/prereq.py => 'asn1' imported but unused
pyflakes.ignore stem/prereq.py => 'long_to_bytes' imported but unused
pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 60
-pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes' from line 56
-pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_unicode' from line 62
+pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes_impl' from line 51
+pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_unicode_impl' from line 57
pyflakes.ignore test/mocking.py => undefined name 'builtins'
pyflakes.ignore test/unit/response/events.py => 'from stem import *' used; unable to detect undefined names
diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py
index 07fb50c..3da01b5 100644
--- a/test/unit/descriptor/server_descriptor.py
+++ b/test/unit/descriptor/server_descriptor.py
@@ -210,7 +210,7 @@ class TestServerDescriptor(unittest.TestCase):
desc_text += "\ntrailing text that should be ignored, ho hum"
# running _parse_file should provide an iterator with a single descriptor
- desc_iter = stem.descriptor.server_descriptor._parse_file(StringIO.StringIO(stem.util.str_tools.to_unicode(desc_text)))
+ desc_iter = stem.descriptor.server_descriptor._parse_file(StringIO.StringIO(stem.util.str_tools._to_unicode(desc_text)))
desc_entries = list(desc_iter)
self.assertEquals(1, len(desc_entries))
desc = desc_entries[0]
diff --git a/test/unit/util/str_tools.py b/test/unit/util/str_tools.py
index 4b35278..550edeb 100644
--- a/test/unit/util/str_tools.py
+++ b/test/unit/util/str_tools.py
@@ -11,19 +11,19 @@ from stem.util import str_tools
class TestStrTools(unittest.TestCase):
def test_to_camel_case(self):
"""
- Checks the to_camel_case() function.
+ Checks the _to_camel_case() function.
"""
# test the pydoc example
- self.assertEquals("I Like Pepperjack!", str_tools.to_camel_case("I_LIKE_PEPPERJACK!"))
+ self.assertEquals("I Like Pepperjack!", str_tools._to_camel_case("I_LIKE_PEPPERJACK!"))
# check a few edge cases
- self.assertEquals("", str_tools.to_camel_case(""))
- self.assertEquals("Hello", str_tools.to_camel_case("hello"))
- self.assertEquals("Hello", str_tools.to_camel_case("HELLO"))
- self.assertEquals("Hello World", str_tools.to_camel_case("hello__world"))
- self.assertEquals("Hello\tworld", str_tools.to_camel_case("hello\tWORLD"))
- self.assertEquals("Hello\t\tWorld", str_tools.to_camel_case("hello__world", "_", "\t"))
+ self.assertEquals("", str_tools._to_camel_case(""))
+ self.assertEquals("Hello", str_tools._to_camel_case("hello"))
+ self.assertEquals("Hello", str_tools._to_camel_case("HELLO"))
+ self.assertEquals("Hello World", str_tools._to_camel_case("hello__world"))
+ self.assertEquals("Hello\tworld", str_tools._to_camel_case("hello\tWORLD"))
+ self.assertEquals("Hello\t\tWorld", str_tools._to_camel_case("hello__world", "_", "\t"))
def test_get_size_label(self):
"""
@@ -125,7 +125,7 @@ class TestStrTools(unittest.TestCase):
def test_parse_iso_timestamp(self):
"""
- Checks the parse_iso_timestamp() function.
+ Checks the _parse_iso_timestamp() function.
"""
test_inputs = {
@@ -138,7 +138,7 @@ class TestStrTools(unittest.TestCase):
}
for arg, expected in test_inputs.items():
- self.assertEqual(expected, str_tools.parse_iso_timestamp(arg))
+ self.assertEqual(expected, str_tools._parse_iso_timestamp(arg))
invalid_input = [
None,
@@ -150,4 +150,4 @@ class TestStrTools(unittest.TestCase):
]
for arg in invalid_input:
- self.assertRaises(ValueError, str_tools.parse_iso_timestamp, arg)
+ self.assertRaises(ValueError, str_tools._parse_iso_timestamp, arg)
1
0
10 Mar '13
commit 860e89bed7a4d6769d9d4376a770e9af19de684b
Author: Damian Johnson <atagar(a)torproject.org>
Date: Sat Mar 9 23:13:25 2013 -0800
Dropping 'last updated' timestamp from site
While I'd kinda like to have this on our site, it's stupidly broken. This is
probably due to sphinx caching (it only makes static files for things that have
changed). Presently it says that our site was last updated on December 31st
which is... very wrong. No date is better than the wrong date so dropping it.
https://trac.torproject.org/7780
---
docs/index.rst | 2 --
1 files changed, 0 insertions(+), 2 deletions(-)
diff --git a/docs/index.rst b/docs/index.rst
index 0da7d57..d95a35f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -52,5 +52,3 @@ Subsections
contents
-*Last updated:* |today|
-
1
0
[ooni-probe/master] Get Director, TaskManager's, NetTest and oonicli to talk
by isis@torproject.org 10 Mar '13
by isis@torproject.org 10 Mar '13
10 Mar '13
commit a31d54acc5d55e7bda567c37799afc8d6e2644af
Author: Arturo Filastò <art(a)fuffa.org>
Date: Wed Jan 16 18:09:39 2013 +0100
Get Director, TaskManager's, NetTest and oonicli to talk
Tests are scheduled through the MeasurementManager, Reports via the
ReporterManager and the Director keeps track of everything that is going on.
Configurable retries and timeouts are possible.
As an example, example_simple.py is run and the reports are written.
---
bin/ooniprobe | 14 ++------
nettests/examples/example_simple.py | 8 +++++
ooni/director.py | 7 +---
ooni/managers.py | 5 ++-
ooni/nettest.py | 58 +++++++++++++++++++++++++++--------
ooni/reporter.py | 20 +++++++-----
ooni/tasks.py | 18 +++++-----
tests/test_director.py | 1 -
tests/test_managers.py | 2 -
9 files changed, 83 insertions(+), 50 deletions(-)
diff --git a/bin/ooniprobe b/bin/ooniprobe
index 36f0152..695b137 100755
--- a/bin/ooniprobe
+++ b/bin/ooniprobe
@@ -1,12 +1,4 @@
#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# oonicli
-# -----------
-# Runs ooniprobe tests from command line
-#
-# :authors: Arturo Filastò, Isis Lovecruft
-# :licence: see included LICENSE file
-
import os, sys
import copy_reg
from twisted.internet import reactor
@@ -19,5 +11,7 @@ sys.path.insert(0, os.path.abspath(os.getcwd()))
from ooni.utils.hacks import patched_reduce_ex
copy_reg._reduce_ex = patched_reduce_ex
-from ooni.oonicli import run
-run()
+# from ooni.oonicli import run
+# run()
+from ooni.oonicli import runWithDirector
+runWithDirector()
diff --git a/nettests/examples/example_simple.py b/nettests/examples/example_simple.py
new file mode 100644
index 0000000..24de5a6
--- /dev/null
+++ b/nettests/examples/example_simple.py
@@ -0,0 +1,8 @@
+from twisted.internet import defer
+from ooni import nettest
+
+class MyIP(nettest.NetTestCase):
+ def test_simple(self):
+ self.report['foobar'] = 'antani'
+ return defer.succeed(42)
+
diff --git a/ooni/director.py b/ooni/director.py
index ac53f5c..6fe3859 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -118,24 +118,21 @@ class Director(object):
def measurementSucceeded(self, measurement):
self.totalMeasurementRuntime += measurement.runtime
-
self.successfulMeasurements += 1
-
- return measurement.report.write(measurement)
+ return measurement.testInstance.report
def measurementFailed(self, failure, measurement):
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
self.failures.append((failure, measurement))
+ return failure
def reportEntryFailed(self, failure):
# XXX add failure handling logic
return
def netTestDone(self, result, net_test):
- print result
- print "Completed %s" % net_test
self.activeNetTests.remove(net_test)
def startNetTest(self, net_test_loader, options):
diff --git a/ooni/managers.py b/ooni/managers.py
index 818ae5c..533e6b5 100644
--- a/ooni/managers.py
+++ b/ooni/managers.py
@@ -23,10 +23,13 @@ class TaskManager(object):
self.failures = []
def _failed(self, failure, task):
+ # XXX INFINITE RECURSION LOOP INSIDE OF THIS THING
"""
The has failed to complete, we append it to the end of the task chain
to be re-run once all the currently scheduled tasks have run.
"""
+ print "This task has failed"
+ print failure
self._active_tasks.remove(task)
self.failures.append((failure, task))
@@ -147,8 +150,6 @@ class ReportEntryManager(TaskManager):
retries = 3
concurrency = 20
- director = None
-
def succeeded(self, result, task):
pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index be1c4b3..f82354b 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -30,7 +30,7 @@ class NetTestState(object):
self.completedScheduling = False
self.allTasksDone = allTasksDone
- def created(self):
+ def taskCreated(self):
self.tasks += 1
def checkAllTasksDone(self):
@@ -38,7 +38,7 @@ class NetTestState(object):
self.doneTasks == self.tasks:
self.allTasksDone.callback(self.doneTasks)
- def taskDone(self, result):
+ def taskDone(self):
"""
This is called every time a task has finished running.
"""
@@ -238,22 +238,51 @@ class NetTest(object):
self.state.taskDone()
return result
+ def makeMeasurement(self, test_class, test_method, test_input=None):
+ """
+ Creates a new instance of :class:ooni.tasks.Measurement and add's it's
+ callbacks and errbacks.
+
+ Args:
+ test_class:
+ a subclass of :class:ooni.nettest.NetTestCase
+
+ test_method:
+ a string that represents the method to be called on test_class
+
+ test_input:
+ optional argument that represents the input to be passed to the
+ NetTestCase
+
+ """
+ measurement = Measurement(test_class, test_method, test_input)
+ measurement.netTest = self
+
+ measurement.done.addCallback(self.director.measurementSucceeded)
+ measurement.done.addErrback(self.director.measurementFailed, measurement)
+
+ measurement.done.addCallback(self.report.write)
+ measurement.done.addErrback(self.director.reportEntryFailed)
+
+ measurement.done.addBoth(self.doneReport)
+ return measurement
+
def generateMeasurements(self):
"""
This is a generator that yields measurements and registers the
callbacks for when a measurement is successful or has failed.
"""
+ self.report.open()
for test_class, test_method in self.testCases:
- for test_input in test_class.inputs:
- measurement = Measurement(test_class, test_method, test_input)
-
- measurement.done.addCallback(self.director.measurementSucceeded)
- measurement.done.addErrback(self.director.measurementFailed)
-
- measurement.done.addCallback(self.report.write)
- measurement.done.addErrback(self.director.reportEntryFailed)
+ if not test_class.inputs:
+ # XXX this is a bit dirty, refactor me
+ yield self.makeMeasurement(test_class, test_method)
+ self.state.taskCreated()
+ break
- measurement.done.addBoth(self.doneReport)
+ for test_input in test_class.inputs:
+ measurement = self.makeMeasurement(test_class, test_method,
+ test_input)
self.state.taskCreated()
yield measurement
@@ -277,7 +306,10 @@ class NetTest(object):
test_instance._checkRequiredOptions()
test_instance._checkValidOptions()
- klass.inputs = test_instance.getInputProcessor()
+ inputs = test_instance.getInputProcessor()
+ if not inputs:
+ inputs = [None]
+ klass.inputs = inputs
class NetTestCase(object):
"""
@@ -424,7 +456,7 @@ class NetTestCase(object):
return inputProcessorIterator()
- return iter(())
+ return None
def _checkValidOptions(self):
for option in self.localOptions:
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 8a39531..7d33bd1 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -33,7 +33,6 @@ from ooni import config
from ooni.tasks import ReportEntry
-
class ReporterException(Exception):
pass
@@ -204,13 +203,16 @@ class YAMLReporter(OReporter):
log.debug("Writing report with YAML reporter")
self._write('---\n')
self._write(safe_dump(entry))
+ self._write('...\n')
def createReport(self):
"""
Writes the report header and fire callbacks on self.created
"""
self._writeln("###########################################")
- self._writeln("# OONI Probe Report for %s test" % self.test_name)
+
+ self._writeln("# OONI Probe Report for %s (%s)" % (self.testDetails['test_name'],
+ self.testDetails['test_version']))
self._writeln("# %s" % otime.prettyDateNow())
self._writeln("###########################################")
@@ -390,7 +392,7 @@ class Report(object):
a deferred list that will fire once all the report entries have
been written.
"""
- dl = []
+ l = []
for reporter in self.reporters:
def writeReportEntry(result):
report_write_task = ReportEntry(reporter, measurement)
@@ -398,9 +400,10 @@ class Report(object):
return report_write_task.done
d = reporter.created.addBoth(writeReportEntry)
- dl.append(d)
+ l.append(d)
- return defer.DeferredList(dl)
+ dl = defer.DeferredList(l)
+ return dl
def close(self, _):
"""
@@ -411,9 +414,10 @@ class Report(object):
all the reports have been closed.
"""
- dl = []
+ l = []
for reporter in self.reporters:
d = defer.maybeDeferred(reporter.finish)
- dl.append(d)
- return defer.DeferredList(dl)
+ l.append(d)
+ dl = defer.DeferredList(l)
+ return dl
diff --git a/ooni/tasks.py b/ooni/tasks.py
index bd7e5b8..4f7da4c 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -103,25 +103,25 @@ class Measurement(TaskWithTimeout):
net_test:
a reference to the net_test object such measurement belongs to.
"""
- self.test_instance = test_class()
- self.test_instance.input = test_input
- self.test_instance.report = {}
- self.test_instance._start_time = time.time()
- self.test_instance._setUp()
- self.test_instance.setUp()
+ self.testInstance = test_class()
+ self.testInstance.input = test_input
+ self.testInstance.report = {}
+ self.testInstance._start_time = time.time()
+ self.testInstance._setUp()
+ self.testInstance.setUp()
- self.net_test_method = getattr(self.test_instance, test_method)
+ self.netTestMethod = getattr(self.testInstance, test_method)
TaskWithTimeout.__init__(self)
def succeeded(self, result):
- return self.netTest.succeeded(self)
+ pass
def failed(self, failure):
pass
def run(self):
- d = self.net_test_method()
+ d = self.netTestMethod()
return d
class ReportEntry(TaskWithTimeout):
diff --git a/tests/test_director.py b/tests/test_director.py
index 409047b..365c11f 100644
--- a/tests/test_director.py
+++ b/tests/test_director.py
@@ -45,7 +45,6 @@ class TestDirector(unittest.TestCase):
@d.addCallback
def done(result):
- print "SOMETHING"
self.assertEqual(self.director.successfulMeasurements, 20)
return d
diff --git a/tests/test_managers.py b/tests/test_managers.py
index 13f1847..601e305 100644
--- a/tests/test_managers.py
+++ b/tests/test_managers.py
@@ -12,8 +12,6 @@ from tests.mocks import MockTaskManager, mockFailure, MockDirector
from tests.mocks import MockNetTest, MockMeasurement, MockSuccessMeasurement
from tests.mocks import MockFailMeasurement, MockFailOnceMeasurement
-from decotrace import traced
-
class TestTaskManager(unittest.TestCase):
timeout = 1
def setUp(self):
1
0
commit 45d63b54388d4e0ff0ba750798f5263d4d1a6701
Author: Arturo Filastò <art(a)fuffa.org>
Date: Tue Jan 15 18:56:58 2013 +0100
Minor refactoring and debugging
Further code cleaning and debugging of a non deterministic bug
Refactoring and bugfixing of Director and NetTest
---
ooni/director.py | 27 +++----
ooni/managers.py | 34 ++++-----
ooni/nettest.py | 191 ++++++++++++++++++++++++++++++++----------------
ooni/oonicli.py | 50 +++++++++++--
ooni/reporter.py | 152 ++++++++++++++------------------------
ooni/tasks.py | 29 +++-----
tests/mocks.py | 18 ++---
tests/test_managers.py | 29 +++++--
tests/test_reporter.py | 14 ++--
9 files changed, 300 insertions(+), 244 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index 41100cf..ac53f5c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -51,6 +51,7 @@ class Director(object):
self.reporters = reporters
self.netTests = []
+ self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
@@ -132,20 +133,18 @@ class Director(object):
# XXX add failure handling logic
return
- def startMeasurements(self, measurements):
- self.measurementManager.schedule(measurements)
-
- def netTestDone(self, net_test):
+ def netTestDone(self, result, net_test):
+ print result
+ print "Completed %s" % net_test
self.activeNetTests.remove(net_test)
- def startNetTest(self, net_test_file, options):
+ def startNetTest(self, net_test_loader, options):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
- net_test_file:
- is either a file path or a file like object that will be used to
- generate the test_cases.
+ net_test_loader:
+ an instance of :class:ooni.nettest.NetTestLoader
options:
is a dict containing the options to be passed to the chosen net
@@ -153,13 +152,13 @@ class Director(object):
"""
report = Report(self.reporters, self.reportEntryManager)
- net_test = NetTest(net_test_file, options, report)
+ net_test = NetTest(net_test_loader, options, report)
+ net_test.setUpNetTestCases()
net_test.director = self
- self.activeNetTests.append(net_test)
- self.activeNetTests.append(net_test)
+ self.measurementManager.schedule(net_test.generateMeasurements())
- d = net_test.start()
- d.addBoth(self.netTestDone)
- return d
+ self.activeNetTests.append(net_test)
+ net_test.done.addBoth(self.netTestDone, net_test)
+ return net_test.done
diff --git a/ooni/managers.py b/ooni/managers.py
index fa59058..818ae5c 100644
--- a/ooni/managers.py
+++ b/ooni/managers.py
@@ -35,12 +35,12 @@ class TaskManager(object):
makeIterable(task))
else:
# This fires the errback when the task is done but has failed.
- task.done.callback(failure)
-
- self.failed(failure, task)
+ task.done.errback(failure)
self._fillSlots()
+ self.failed(failure, task)
+
def _fillSlots(self):
"""
Called on test completion and schedules measurements to be run for the
@@ -53,6 +53,17 @@ class TaskManager(object):
except StopIteration:
break
+ def _run(self, task):
+ """
+ This gets called to add a task to the list of currently active and
+ running tasks.
+ """
+ self._active_tasks.append(task)
+
+ d = task.start()
+ d.addCallback(self._succeeded, task)
+ d.addErrback(self._failed, task)
+
def _succeeded(self, result, task):
"""
We have successfully completed a measurement.
@@ -65,17 +76,6 @@ class TaskManager(object):
task.done.callback(task)
self.succeeded(result, task)
- def _run(self, task):
- """
- This gets called to add a task to the list of currently active and
- running tasks.
- """
- self._active_tasks.append(task)
-
- d = task.start()
- d.addCallback(self._succeeded, task)
- d.addErrback(self._failed, task)
-
@property
def failedMeasurements(self):
return len(self.failures)
@@ -136,13 +136,11 @@ class MeasurementManager(TaskManager):
retries = 2
concurrency = 10
- director = None
-
def succeeded(self, result, measurement):
- self.director.measurementSucceeded(measurement)
+ pass
def failed(self, failure, measurement):
- self.director.measurementFailed(failure, measurement)
+ pass
class ReportEntryManager(TaskManager):
# XXX tweak these values
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 9146924..be1c4b3 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -6,6 +6,8 @@ from twisted.python import usage, reflect
from ooni.tasks import Measurement
from ooni.utils import log, checkForRoot, NotRootError
+from ooni import config
+from ooni import otime
from inspect import getmembers
from StringIO import StringIO
@@ -58,70 +60,73 @@ class NetTestState(object):
self.completedScheduling = True
self.checkAllTasksDone()
-class NetTest(object):
- director = None
+class NetTestLoader(object):
method_prefix = 'test'
- def __init__(self, net_test_file, options, report):
- """
- net_test_file:
- is a file object containing the test to be run.
-
- options:
- is a dict containing the options to be passed to the net test.
- """
- self.options = options
- self.report = report
- self.test_cases = self.loadNetTest(net_test_file)
-
- # This will fire when all the measurements have been completed and
- # all the reports are done. Done means that they have either completed
- # successfully or all the possible retries have been reached.
- self.done = defer.Deferred()
-
- self.state = NetTestState(self.done)
-
- def start(self):
- """
- Set up tests and start running.
- Start tests and generate measurements.
- """
- self.setUpNetTestCases()
- self.director.startMeasurements(self.generateMeasurements())
- return self.done
-
- def doneReport(self, result):
- """
- This will get called every time a measurement is done and therefore a
- measurement is done.
-
- The state for the NetTest is informed of the fact that another task has
- reached the done state.
- """
- self.state.taskDone()
- return result
-
- def generateMeasurements(self):
- """
- This is a generator that yields measurements and registers the
- callbacks for when a measurement is successful or has failed.
- """
- for test_class, test_method in self.test_cases:
- for test_input in test_class.inputs:
- measurement = Measurement(test_class, test_method, test_input)
-
- measurement.done.addCallback(self.director.measurementSucceeded)
- measurement.done.addErrback(self.director.measurementFailed)
-
- measurement.done.addCallback(self.report.write)
- measurement.done.addErrback(self.director.reportEntryFailed)
-
- measurement.done.addBoth(self.doneReport)
-
- self.state.taskCreated()
- yield measurement
-
- self.state.allTasksScheduled()
+ def __init__(self, net_test_file):
+ self.testCases = self.loadNetTest(net_test_file)
+ # XXX Remove
+ self.testName = 'fooo'
+ self.testVersion = '0.1'
+
+ @property
+ def testDetails(self):
+ from ooni import __version__ as software_version
+
+ client_geodata = {}
+ if config.probe_ip and (config.privacy.includeip or \
+ config.privacy.includeasn or \
+ config.privacy.includecountry or \
+ config.privacy.includecity):
+ log.msg("We will include some geo data in the report")
+ client_geodata = geodata.IPToLocation(config.probe_ip)
+
+ if config.privacy.includeip:
+ client_geodata['ip'] = config.probe_ip
+ else:
+ client_geodata['ip'] = "127.0.0.1"
+
+ # Here we unset all the client geodata if the option to not include then
+ # has been specified
+ if client_geodata and not config.privacy.includeasn:
+ client_geodata['asn'] = 'AS0'
+ elif 'asn' in client_geodata:
+ # XXX this regexp should probably go inside of geodata
+ client_geodata['asn'] = \
+ re.search('AS\d+', client_geodata['asn']).group(0)
+ log.msg("Your AS number is: %s" % client_geodata['asn'])
+ else:
+ client_geodata['asn'] = None
+
+ if (client_geodata and not config.privacy.includecity) \
+ or ('city' not in client_geodata):
+ client_geodata['city'] = None
+
+ if (client_geodata and not config.privacy.includecountry) \
+ or ('countrycode' not in client_geodata):
+ client_geodata['countrycode'] = None
+
+ test_details = {'start_time': otime.utcTimeNow(),
+ 'probe_asn': client_geodata['asn'],
+ 'probe_cc': client_geodata['countrycode'],
+ 'probe_ip': client_geodata['ip'],
+ 'test_name': self.testName,
+ 'test_version': self.testVersion,
+ 'software_name': 'ooniprobe',
+ 'software_version': software_version
+ }
+ return test_details
+
+
+ @property
+ def usageOptions(self):
+ usage_options = None
+ for test_class, test_method in self.testCases:
+ if not usage_options:
+ usage_options = test_class.usageOptions
+ else:
+ assert usage_options == test_class.usageOptions
+ return usage_options
def loadNetTest(self, net_test_file):
"""
@@ -144,6 +149,10 @@ class NetTest(object):
is either a file path or a file like object that will be used to
generate the test_cases.
"""
+ # XXX
+ # self.testName =
+ # os.path.basename('/foo/bar/python.py').replace('.py','')
+ # self.testVersion = '0.1'
test_cases = None
try:
if os.path.isfile(net_test_file):
@@ -196,13 +205,67 @@ class NetTest(object):
pass
return test_cases
+class NetTest(object):
+ director = None
+
+ def __init__(self, net_test_loader, options, report):
+ """
+ net_test_file:
+ is a file object containing the test to be run.
+
+ options:
+ is a dict containing the options to be passed to the net test.
+ """
+ self.options = options
+ self.report = report
+ self.testCases = net_test_loader.testCases
+
+ # This will fire when all the measurements have been completed and
+ # all the reports are done. Done means that they have either completed
+ # successfully or all the possible retries have been reached.
+ self.done = defer.Deferred()
+
+ self.state = NetTestState(self.done)
+
+ def doneReport(self, result):
+ """
+ This will get called every time a measurement is done and therefore a
+ measurement is done.
+
+ The state for the NetTest is informed of the fact that another task has
+ reached the done state.
+ """
+ self.state.taskDone()
+ return result
+
+ def generateMeasurements(self):
+ """
+ This is a generator that yields measurements and registers the
+ callbacks for when a measurement is successful or has failed.
+ """
+ for test_class, test_method in self.testCases:
+ for test_input in test_class.inputs:
+ measurement = Measurement(test_class, test_method, test_input)
+
+ measurement.done.addCallback(self.director.measurementSucceeded)
+ measurement.done.addErrback(self.director.measurementFailed)
+
+ measurement.done.addCallback(self.report.write)
+ measurement.done.addErrback(self.director.reportEntryFailed)
+
+ measurement.done.addBoth(self.doneReport)
+
+ self.state.taskCreated()
+ yield measurement
+
+ self.state.allTasksScheduled()
def setUpNetTestCases(self):
"""
Call processTest and processOptions methods of each NetTestCase
"""
test_classes = set([])
- for test_class, test_method in self.test_cases:
+ for test_class, test_method in self.testCases:
test_classes.add(test_class)
for klass in test_classes:
@@ -373,7 +436,7 @@ class NetTestCase(object):
for required_option in self.requiredOptions:
log.debug("Checking if %s is present" % required_option)
if required_option not in self.localOptions:
- raise MissingRequiredOption
+ raise MissingRequiredOption(required_option)
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self.inputs)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index d345a25..1959de8 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -12,9 +12,13 @@ from twisted.python import usage, failure
from twisted.python.util import spewer
from ooni import nettest, runner, reporter, config
+from ooni.director import Director
+from ooni.reporter import YAMLReporter, OONIBReporter
from ooni.inputunit import InputUnitFactory
+from ooni.nettest import NetTestLoader, MissingRequiredOption
+
from ooni.utils import net
from ooni.utils import checkForRoot, NotRootError
from ooni.utils import log
@@ -143,10 +147,8 @@ def errorRunningTests(failure):
log.err("There was an error in running a test")
failure.printTraceback()
-def run():
- """
- Parses command line arguments of test.
- """
+
+def parseOptions():
cmd_line_options = Options()
if len(sys.argv) == 1:
cmd_line_options.getUsage()
@@ -155,9 +157,43 @@ def run():
except usage.UsageError, ue:
raise SystemExit, "%s: %s" % (sys.argv[0], ue)
- log.start(cmd_line_options['logfile'])
+ return dict(cmd_line_options)
+
+def runWithDirector():
+ """
+ Instance the director, parse command line options and start an ooniprobe
+ test!
+ """
+ global_options = parseOptions()
+ config.cmd_line_options = global_options
+
+ log.start(global_options['logfile'])
+
+ net_test_args = global_options.pop('subargs')
+ net_test_file = global_options['test']
+
+ net_test_loader = NetTestLoader(net_test_file)
+ options = net_test_loader.usageOptions()
+ options.parseOptions(net_test_args)
- config.cmd_line_options = cmd_line_options
+ net_test_options = dict(options)
+
+ # reporters = [YAMLReporter, OONIBReporter]
+
+ yaml_reporter = YAMLReporter(net_test_loader.testDetails)
+ reporters = [yaml_reporter]
+
+ director = Director(reporters)
+ try:
+ director.startNetTest(net_test_loader, net_test_options)
+ except MissingRequiredOption, option_name:
+ log.err('Missing required option: "%s"' % option_name)
+ print options.getUsage()
+
+def run():
+ """
+ Parses command line arguments of test.
+ """
if config.privacy.includepcap:
log.msg("Starting")
@@ -197,3 +233,5 @@ def run():
d.addErrback(errorRunningTests)
reactor.run()
+
+
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 80595f9..8a39531 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -33,6 +33,10 @@ from ooni import config
from ooni.tasks import ReportEntry
+
+class ReporterException(Exception):
+ pass
+
def createPacketReport(packet_list):
"""
Takes as input a packet a list.
@@ -110,60 +114,12 @@ def safe_dump(data, stream=None, **kw):
"""
return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)
-def getTestDetails(options):
- from ooni import __version__ as software_version
-
- client_geodata = {}
- if config.probe_ip and (config.privacy.includeip or \
- config.privacy.includeasn or \
- config.privacy.includecountry or \
- config.privacy.includecity):
- log.msg("We will include some geo data in the report")
- client_geodata = geodata.IPToLocation(config.probe_ip)
-
- if config.privacy.includeip:
- client_geodata['ip'] = config.probe_ip
- else:
- client_geodata['ip'] = "127.0.0.1"
-
- # Here we unset all the client geodata if the option to not include then
- # has been specified
- if client_geodata and not config.privacy.includeasn:
- client_geodata['asn'] = 'AS0'
- elif 'asn' in client_geodata:
- # XXX this regexp should probably go inside of geodata
- client_geodata['asn'] = \
- re.search('AS\d+', client_geodata['asn']).group(0)
- log.msg("Your AS number is: %s" % client_geodata['asn'])
- else:
- client_geodata['asn'] = None
-
- if (client_geodata and not config.privacy.includecity) \
- or ('city' not in client_geodata):
- client_geodata['city'] = None
-
- if (client_geodata and not config.privacy.includecountry) \
- or ('countrycode' not in client_geodata):
- client_geodata['countrycode'] = None
-
- test_details = {'start_time': otime.utcTimeNow(),
- 'probe_asn': client_geodata['asn'],
- 'probe_cc': client_geodata['countrycode'],
- 'probe_ip': client_geodata['ip'],
- 'test_name': options['name'],
- 'test_version': options['version'],
- 'software_name': 'ooniprobe',
- 'software_version': software_version
- }
- return test_details
-
class OReporter(object):
- created = defer.Deferred()
+ def __init__(self, test_details):
+ self.created = defer.Deferred()
+ self.testDetails = test_details
- def __init__(self, cmd_line_options):
- self.cmd_line_options = dict(cmd_line_options)
-
- def createReport(self, options):
+ def createReport(self):
"""
Override this with your own logic to implement tests.
"""
@@ -179,7 +135,8 @@ class OReporter(object):
pass
def testDone(self, test, test_name):
- # XXX
+ # XXX put this inside of Report.close
+ # or perhaps put something like this inside of netTestDone
log.msg("Finished running %s" % test_name)
test_report = dict(test.report)
@@ -198,30 +155,37 @@ class OReporter(object):
'report': test_report}
return defer.maybeDeferred(self.writeReportEntry, report)
+class InvalidDestination(ReporterException):
+ pass
+
class YAMLReporter(OReporter):
"""
These are useful functions for reporting to YAML format.
+
+ report_destination:
+ the destination directory of the report
+
"""
- def __init__(self, cmd_line_options):
- if cmd_line_options['reportfile'] is None:
- try:
- test_filename = os.path.basename(cmd_line_options['test'])
- except IndexError:
- raise TestFilenameNotSet
-
- test_name = '.'.join(test_filename.split(".")[:-1])
- frm_str = "report_%s_"+otime.timestamp()+".%s"
- reportfile = frm_str % (test_name, "yamloo")
- else:
- reportfile = cmd_line_options['reportfile']
+ def __init__(self, test_details, report_destination='.'):
+ self.reportDestination = report_destination
+
+ if not os.path.isdir(report_destination):
+ raise InvalidDestination
+
+ report_filename = "report-" + \
+ test_details['test_name'] + "-" + \
+ otime.timestamp() + ".yamloo"
- if os.path.exists(reportfile):
- log.msg("Report already exists with filename %s" % reportfile)
- pushFilenameStack(reportfile)
+ report_path = os.path.join(self.reportDestination, report_filename)
- log.debug("Creating %s" % reportfile)
- self._stream = open(reportfile, 'w+')
- OReporter.__init__(self, cmd_line_options)
+ if os.path.exists(report_path):
+ log.msg("Report already exists with filename %s" % report_path)
+ pushFilenameStack(report_path)
+
+ log.debug("Creating %s" % report_path)
+ self._stream = open(report_path, 'w+')
+
+ OReporter.__init__(self, test_details)
def _writeln(self, line):
self._write("%s\n" % line)
@@ -236,26 +200,25 @@ class YAMLReporter(OReporter):
untilConcludes(self._stream.flush)
def writeReportEntry(self, entry):
+ #XXX: all _write, _writeln inside this call should be atomic
log.debug("Writing report with YAML reporter")
self._write('---\n')
self._write(safe_dump(entry))
- self._write('...\n')
- def createReport(self, options):
+ def createReport(self):
+ """
+ Writes the report header and fire callbacks on self.created
+ """
self._writeln("###########################################")
- self._writeln("# OONI Probe Report for %s test" % options['name'])
+ self._writeln("# OONI Probe Report for %s test" % self.test_name)
self._writeln("# %s" % otime.prettyDateNow())
self._writeln("###########################################")
- test_details = getTestDetails(options)
- test_details['options'] = self.cmd_line_options
-
- self.writeReportEntry(test_details)
+ self.writeReportEntry(self.testDetails)
def finish(self):
self._stream.close()
-
class OONIBReportError(Exception):
pass
@@ -269,8 +232,9 @@ class OONIBTestDetailsLookupError(OONIBReportError):
pass
class OONIBReporter(OReporter):
- def __init__(self, cmd_line_options):
- self.backend_url = cmd_line_options['collector']
+ collector_address = ''
+ def __init__(self, test_details, collector_address):
+ self.collector_address = collector_address
self.report_id = None
from ooni.utils.txagentwithsocks import Agent
@@ -281,7 +245,7 @@ class OONIBReporter(OReporter):
except Exception, e:
log.exception(e)
- OReporter.__init__(self, cmd_line_options)
+ OReporter.__init__(self, test_details)
@defer.inlineCallbacks
def writeReportEntry(self, entry):
@@ -290,7 +254,7 @@ class OONIBReporter(OReporter):
content += safe_dump(entry)
content += '...\n'
- url = self.backend_url + '/report'
+ url = self.collector_address + '/report'
request = {'report_id': self.report_id,
'content': content}
@@ -315,12 +279,7 @@ class OONIBReporter(OReporter):
"""
Creates a report on the oonib collector.
"""
- url = self.backend_url + '/report'
-
- try:
- test_details = getTestDetails(options)
- except Exception, e:
- log.exception(e)
+ url = self.collector_address + '/report'
test_details['options'] = self.cmd_line_options
@@ -336,8 +295,8 @@ class OONIBReporter(OReporter):
request = {'software_name': test_details['software_name'],
'software_version': test_details['software_version'],
'probe_asn': test_details['probe_asn'],
- 'test_name': test_name,
- 'test_version': test_version,
+ 'test_name': test_details['test_name'],
+ 'test_version': test_details['test_version'],
'content': content
}
@@ -391,22 +350,19 @@ class Report(object):
Args:
reporters:
- a list of :class:ooni.reporter.OReporter
+ a list of :class:ooni.reporter.OReporter instances
reportEntryManager:
an instance of :class:ooni.tasks.ReportEntryManager
"""
- self.reporters = []
- for r in reporters:
- reporter = r()
- self.reporters.append(reporter)
-
- self.createReports()
+ self.reporters = reporters
self.done = defer.Deferred()
self.done.addCallback(self.close)
self.reportEntryManager = reportEntryManager
+ # XXX call this when starting test
+ # self.open()
def open(self):
"""
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 28aaca4..bd7e5b8 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -5,12 +5,13 @@ from twisted.internet import defer, reactor
class BaseTask(object):
_timer = None
+ _running = None
+
def __init__(self):
"""
If you want to schedule a task multiple times, remember to create fresh
instances of it.
"""
- self.running = False
self.failures = 0
self.startTime = time.time()
@@ -33,10 +34,10 @@ class BaseTask(object):
return result
def start(self):
- self.running = defer.maybeDeferred(self.run)
- self.running.addErrback(self._failed)
- self.running.addCallback(self._succeeded)
- return self.running
+ self._running = defer.maybeDeferred(self.run)
+ self._running.addErrback(self._failed)
+ self._running.addCallback(self._succeeded)
+ return self._running
def succeeded(self, result):
"""
@@ -67,8 +68,7 @@ class TaskWithTimeout(BaseTask):
def _timedOut(self):
"""Internal method for handling timeout failure"""
- self.timedOut()
- self.running.errback(TaskTimedOut)
+ self._running.errback(TaskTimedOut)
def _cancelTimer(self):
#import pdb; pdb.set_trace()
@@ -87,13 +87,6 @@ class TaskWithTimeout(BaseTask):
self._timer = self.clock.callLater(self.timeout, self._timedOut)
return BaseTask.start(self)
- def timedOut(self):
- """
- Override this with the operations to happen when the task has timed
- out.
- """
- pass
-
class Measurement(TaskWithTimeout):
def __init__(self, test_class, test_method, test_input):
"""
@@ -116,7 +109,8 @@ class Measurement(TaskWithTimeout):
self.test_instance._start_time = time.time()
self.test_instance._setUp()
self.test_instance.setUp()
- self.test = getattr(self.test_instance, test_method)
+
+ self.net_test_method = getattr(self.test_instance, test_method)
TaskWithTimeout.__init__(self)
@@ -126,11 +120,8 @@ class Measurement(TaskWithTimeout):
def failed(self, failure):
pass
- def timedOut(self):
- self.netTest.timedOut()
-
def run(self):
- d = defer.maybeDeferred(self.test)
+ d = self.net_test_method()
return d
class ReportEntry(TaskWithTimeout):
diff --git a/tests/mocks.py b/tests/mocks.py
index fa57927..b692b39 100644
--- a/tests/mocks.py
+++ b/tests/mocks.py
@@ -17,6 +17,7 @@ class MockMeasurementFailOnce(BaseTask):
class MockMeasurementManager(TaskManager):
def __init__(self):
self.successes = []
+ TaskManager.__init__(self)
def failed(self, failure, task):
pass
@@ -34,8 +35,11 @@ class MockReporter(object):
def createReport(self):
pass
+class MockFailure(Exception):
+ pass
+
## from test_managers
-mockFailure = failure.Failure(Exception('mock'))
+mockFailure = failure.Failure(MockFailure('mock'))
class MockSuccessTask(BaseTask):
def run(self):
@@ -71,15 +75,6 @@ class MockFailTaskWithTimeout(TaskWithTimeout):
def run(self):
return defer.fail(mockFailure)
-class MockTaskManager(TaskManager):
- def __init__(self):
- self.successes = []
-
- def failed(self, failure, task):
- pass
-
- def succeeded(self, result, task):
- self.successes.append((result, task))
class MockNetTest(object):
def __init__(self):
@@ -135,13 +130,14 @@ class MockOReporter(object):
def createReport(self):
pass
-
class MockTaskManager(TaskManager):
def __init__(self):
self.successes = []
+ TaskManager.__init__(self)
def failed(self, failure, task):
pass
def succeeded(self, result, task):
self.successes.append((result, task))
+
diff --git a/tests/test_managers.py b/tests/test_managers.py
index 1e469c7..13f1847 100644
--- a/tests/test_managers.py
+++ b/tests/test_managers.py
@@ -2,21 +2,23 @@ from twisted.trial import unittest
from twisted.python import failure
from twisted.internet import defer, task
-from ooni.tasks import BaseTask, TaskWithTimeout
+from ooni.tasks import BaseTask, TaskWithTimeout, TaskTimedOut
from ooni.managers import TaskManager, MeasurementManager
-from tests.mocks import MockSuccessTask, MockFailTask, MockFailOnceTask
+from tests.mocks import MockSuccessTask, MockFailTask, MockFailOnceTask, MockFailure
from tests.mocks import MockSuccessTaskWithTimeout, MockFailTaskThatTimesOut
from tests.mocks import MockTimeoutOnceTask, MockFailTaskWithTimeout
from tests.mocks import MockTaskManager, mockFailure, MockDirector
from tests.mocks import MockNetTest, MockMeasurement, MockSuccessMeasurement
from tests.mocks import MockFailMeasurement, MockFailOnceMeasurement
+from decotrace import traced
+
class TestTaskManager(unittest.TestCase):
timeout = 1
def setUp(self):
self.measurementManager = MockTaskManager()
- self.measurementManager.concurrency = 10
+ self.measurementManager.concurrency = 20
self.measurementManager.retries = 2
self.measurementManager.start()
@@ -57,6 +59,12 @@ class TestTaskManager(unittest.TestCase):
return d
+ def test_schedule_failing_with_mock_failure_task(self):
+ mock_task = MockFailTask()
+ self.measurementManager.schedule(mock_task)
+ self.assertFailure(mock_task.done, MockFailure)
+ return mock_task.done
+
def test_schedule_successful_one_task(self):
return self.schedule_successful_tasks(MockSuccessTask)
@@ -143,17 +151,22 @@ class TestTaskManager(unittest.TestCase):
return mock_task.done
- def test_task_retry_and_succeed_56_tasks(self):
+ def dd_test_task_retry_and_succeed_56_tasks(self):
+ """
+ XXX this test fails in a non-deterministic manner.
+ """
all_done = []
- for x in range(56):
+ number = 56
+ for x in range(number):
mock_task = MockFailOnceTask()
all_done.append(mock_task.done)
self.measurementManager.schedule(mock_task)
d = defer.DeferredList(all_done)
+
@d.addCallback
def done(res):
- self.assertEqual(len(self.measurementManager.failures), 56)
+ self.assertEqual(len(self.measurementManager.failures), number)
for task_result, task_instance in self.measurementManager.successes:
self.assertEqual(task_result, 42)
@@ -192,11 +205,11 @@ class TestMeasurementManager(unittest.TestCase):
mock_task = MockFailMeasurement(self.mockNetTest)
self.measurementManager.schedule(mock_task)
- @mock_task.done.addCallback
+ @mock_task.done.addErrback
def done(failure):
self.assertEqual(len(self.measurementManager.failures), 3)
- self.assertEqual(failure, (mockFailure, mock_task))
+ self.assertEqual(failure, mockFailure)
self.assertEqual(len(self.mockNetTest.successes), 0)
return mock_task.done
diff --git a/tests/test_reporter.py b/tests/test_reporter.py
index e1c7fca..e99debb 100644
--- a/tests/test_reporter.py
+++ b/tests/test_reporter.py
@@ -3,26 +3,28 @@ from twisted.trial import unittest
from ooni.reporter import Report, YAMLReporter, OONIBReporter
from ooni.managers import ReportEntryManager, TaskManager
-from ooni.nettest import NetTest
+from ooni.nettest import NetTest, NetTestState
-from ooni.tasks import TaskMediator, TaskWithTimeout
+from ooni.tasks import TaskWithTimeout
from tests.mocks import MockOReporter, MockTaskManager
from tests.mocks import MockMeasurement, MockNetTest
mockReportOptions = {'name':'foo_test', 'version': '0.1'}
+class MockState(NetTestState):
+ pass
+
class TestReport(unittest.TestCase):
def setUp(self):
- self.report = Report([MockOReporter])
- self.report.reportEntryManager = MockTaskManager()
+ self.taskManager = MockTaskManager()
+ self.report = Report([MockOReporter], self.taskManager)
+ self.state = MockState()
def test_report_alltasksdone_callback_fires(self):
for m in range(10):
measurement = MockMeasurement(MockNetTest())
self.report.write(measurement)
- self.report.report_mediator.allTasksScheduled()
-
@self.report.done.addCallback
def done(reporters):
self.assertEqual(len(reporters), 1)
1
0
[ooni-probe/master] Get test version from net_test_file and parse all options
by isis@torproject.org 10 Mar '13
by isis@torproject.org 10 Mar '13
10 Mar '13
commit 2da452336aeb1eac53809305661ba727d8daef7c
Author: aagbsn <aagbsn(a)extc.org>
Date: Wed Jan 16 18:22:49 2013 +0000
Get test version from net_test_file and parse all options
Adds helper method _parseNetTestOptions to NetTestLoader in order
to assemble the various options and flags into a single object
Also add functionality to read the test name and version from the
NetTestCase
Check that the required option is not None
localOptions is a dict, and the required option may be None.
This adds a test for this case.
---
ooni/nettest.py | 42 ++++++++++++++++++++++++++++++++----------
1 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index f82354b..7c4278a 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -65,9 +65,6 @@ class NetTestLoader(object):
def __init__(self, net_test_file):
self.testCases = self.loadNetTest(net_test_file)
- # XXX Remove
- self.testName = 'fooo'
- self.testVersion = '0.1'
@property
def testDetails(self):
@@ -118,12 +115,36 @@ class NetTestLoader(object):
return test_details
+ def _parseNetTestOptions(self, klass):
+ """
+ Helper method to assemble the options into a single UsageOptions object
+ """
+ usage_options = klass.usageOptions
+
+ if not hasattr(usage_options, 'optParameters'):
+ usage_options.optParameters = []
+
+ if klass.inputFile:
+ usage_options.optParameters.append(klass.inputFile)
+
+ if klass.baseParameters:
+ for parameter in klass.baseParameters:
+ usage_options.optParameters.append(parameter)
+
+ if klass.baseFlags:
+ if not hasattr(usage_options, 'optFlags'):
+ usage_options.optFlags = []
+ for flag in klass.baseFlags:
+ usage_options.optFlags.append(flag)
+
+ return usage_options
+
@property
def usageOptions(self):
usage_options = None
for test_class, test_method in self.testCases:
if not usage_options:
- usage_options = test_class.usageOptions
+ usage_options = self._parseNetTestOptions(test_class)
else:
assert usage_options == test_class.usageOptions
return usage_options
@@ -149,10 +170,6 @@ class NetTestLoader(object):
is either a file path or a file like object that will be used to
generate the test_cases.
"""
- # XXX
- # self.testName =
- # os.path.basename('/foo/bar/python.py').replace('.py','')
- # self.testVersion = '0.1'
test_cases = None
try:
if os.path.isfile(net_test_file):
@@ -168,6 +185,10 @@ class NetTestLoader(object):
if not test_cases:
raise NoTestCasesFound
+ test_class, _ = test_cases[0]
+ self.testVersion = test_class.version
+ self.testName = os.path.basename(net_test_file).strip('.py')
+
return test_cases
def _loadNetTestFromFileObject(self, net_test_string):
@@ -467,8 +488,9 @@ class NetTestCase(object):
def _checkRequiredOptions(self):
for required_option in self.requiredOptions:
log.debug("Checking if %s is present" % required_option)
- if required_option not in self.localOptions:
- raise MissingRequiredOption(required_option)
+ if required_option not in self.localOptions or \
+ self.localOptions[required_option] == None:
+ raise MissingRequiredOption(required_option)
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self.inputs)
1
0
[ooni-probe/master] Nettests are now running properly with the TaskManager
by isis@torproject.org 10 Mar '13
by isis@torproject.org 10 Mar '13
10 Mar '13
commit cf274700c3e43f86d30a7fd5eb5cd093dbfc4e0a
Author: Arturo Filastò <art(a)fuffa.org>
Date: Wed Jan 16 20:58:29 2013 +0100
Nettests are now running properly with the TaskManager
Issue retries, make tests timeout, etc.
---
ooni/director.py | 4 ++-
ooni/errors.py | 2 +-
ooni/nettest.py | 97 +++++++++++++++++++++++++----------------------------
ooni/reporter.py | 2 -
4 files changed, 50 insertions(+), 55 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index 6fe3859..80cf22c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,6 +1,6 @@
from ooni.managers import ReportEntryManager, MeasurementManager
from ooni.reporter import Report
-
+from ooni.utils import log
from ooni.nettest import NetTest
class Director(object):
@@ -117,11 +117,13 @@ class Director(object):
self.totalMeasurements += 1
def measurementSucceeded(self, measurement):
+ log.msg("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
return measurement.testInstance.report
def measurementFailed(self, failure, measurement):
+ log.msg("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
diff --git a/ooni/errors.py b/ooni/errors.py
index c7ecfae..cd4a136 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -14,7 +14,7 @@ from txsocksx.errors import HostUnreachable, ConnectionRefused
from txsocksx.errors import TTLExpired, CommandNotSupported
from socket import gaierror
-
+from ooni.utils import log
def handleAllFailures(failure):
"""
Here we make sure to trap all the failures that are supported by the
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 7c4278a..728d093 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -15,51 +15,6 @@ from StringIO import StringIO
class NoTestCasesFound(Exception):
pass
-class NetTestState(object):
- def __init__(self, allTasksDone):
- """
- This keeps track of the state of a running NetTests case.
-
- Args:
- allTasksDone is a deferred that will get fired once all the NetTest
- cases have reached a final done state.
- """
- self.doneTasks = 0
- self.tasks = 0
-
- self.completedScheduling = False
- self.allTasksDone = allTasksDone
-
- def taskCreated(self):
- self.tasks += 1
-
- def checkAllTasksDone(self):
- if self.completedScheduling and \
- self.doneTasks == self.tasks:
- self.allTasksDone.callback(self.doneTasks)
-
- def taskDone(self):
- """
- This is called every time a task has finished running.
- """
- self.doneTasks += 1
- self.checkAllTasksDone()
-
- def allTasksScheduled(self):
- """
- This should be called once all the tasks that need to run have been
- scheduled.
-
- XXX this is ghetto.
- The reason for which we are calling allTasksDone inside of the
- allTasksScheduled method is called after all tasks are done, then we
- will run into a race condition. The race is that we don't end up
- checking that all the tasks are complete because no task is to be
- scheduled.
- """
- self.completedScheduling = True
- self.checkAllTasksDone()
-
class NetTestLoader(object):
method_prefix = 'test'
@@ -226,6 +181,51 @@ class NetTestLoader(object):
pass
return test_cases
+class NetTestState(object):
+ def __init__(self, allTasksDone):
+ """
+ This keeps track of the state of a running NetTests case.
+
+ Args:
+ allTasksDone is a deferred that will get fired once all the NetTest
+ cases have reached a final done state.
+ """
+ self.doneTasks = 0
+ self.tasks = 0
+
+ self.completedScheduling = False
+ self.allTasksDone = allTasksDone
+
+ def taskCreated(self):
+ self.tasks += 1
+
+ def checkAllTasksDone(self):
+ if self.completedScheduling and \
+ self.doneTasks == self.tasks:
+ self.allTasksDone.callback(self.doneTasks)
+
+ def taskDone(self):
+ """
+ This is called every time a task has finished running.
+ """
+ self.doneTasks += 1
+ self.checkAllTasksDone()
+
+ def allTasksScheduled(self):
+ """
+ This should be called once all the tasks that need to run have been
+ scheduled.
+
+ XXX this is ghetto.
+ The reason for which we are calling allTasksDone inside of the
+ allTasksScheduled method is called after all tasks are done, then we
+ will run into a race condition. The race is that we don't end up
+ checking that all the tasks are complete because no task is to be
+ scheduled.
+ """
+ self.completedScheduling = True
+ self.checkAllTasksDone()
+
class NetTest(object):
director = None
@@ -295,12 +295,7 @@ class NetTest(object):
"""
self.report.open()
for test_class, test_method in self.testCases:
- if not test_class.inputs:
- # XXX this is a bit dirty, refactor me
- yield self.makeMeasurement(test_class, test_method)
- self.state.taskCreated()
- break
-
+ log.debug("Running %s %s" % (test_class, test_method))
for test_input in test_class.inputs:
measurement = self.makeMeasurement(test_class, test_method,
test_input)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 7d33bd1..30c4817 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -363,8 +363,6 @@ class Report(object):
self.done.addCallback(self.close)
self.reportEntryManager = reportEntryManager
- # XXX call this when starting test
- # self.open()
def open(self):
"""
1
0
[ooni-probe/master] Refactoring of the architecture of the taskManager
by isis@torproject.org 10 Mar '13
by isis@torproject.org 10 Mar '13
10 Mar '13
commit 813230cc416338740bf72acb04b97042ebd5261a
Author: Arturo Filastò <art(a)fuffa.org>
Date: Tue Jan 15 18:01:45 2013 +0100
Refactoring of the architecture of the taskManager
Minimize coupling, make clearer the chain of responsibilities
---
ooni/director.py | 23 ++++++++--
ooni/managers.py | 36 +++++---------
ooni/nettest.py | 123 +++++++++++++++++++++++++++++++++---------------
ooni/reporter.py | 70 +++++++++++++++++++--------
ooni/tasks.py | 65 ++-----------------------
tests/test_nettest.py | 3 +
6 files changed, 174 insertions(+), 146 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index 582bb84..41100cf 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -120,13 +120,25 @@ class Director(object):
self.successfulMeasurements += 1
+ return measurement.report.write(measurement)
+
def measurementFailed(self, failure, measurement):
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
self.failures.append((failure, measurement))
- def startTest(self, net_test_file, options):
+ def reportEntryFailed(self, failure):
+ # XXX add failure handling logic
+ return
+
+ def startMeasurements(self, measurements):
+ self.measurementManager.schedule(measurements)
+
+ def netTestDone(self, net_test):
+ self.activeNetTests.remove(net_test)
+
+ def startNetTest(self, net_test_file, options):
"""
Create the Report for the NetTest and start the report NetTest.
@@ -139,12 +151,15 @@ class Director(object):
is a dict containing the options to be passed to the chosen net
test.
"""
- report = Report(self.reporters)
- report.reportEntryManager = self.reportEntryManager
+ report = Report(self.reporters, self.reportEntryManager)
net_test = NetTest(net_test_file, options, report)
- net_test.measurementManager = self.measurementManager
+ net_test.director = self
+
+ self.activeNetTests.append(net_test)
+ self.activeNetTests.append(net_test)
d = net_test.start()
+ d.addBoth(self.netTestDone)
return d
diff --git a/ooni/managers.py b/ooni/managers.py
index 3983705..fa59058 100644
--- a/ooni/managers.py
+++ b/ooni/managers.py
@@ -15,13 +15,12 @@ def makeIterable(item):
class TaskManager(object):
retries = 2
- failures = []
concurrency = 10
- completedTasks = 0
-
- _tasks = iter(())
- _active_tasks = []
+ def __init__(self):
+ self._tasks = iter(())
+ self._active_tasks = []
+ self.failures = []
def _failed(self, failure, task):
"""
@@ -35,7 +34,8 @@ class TaskManager(object):
self._tasks = itertools.chain(self._tasks,
makeIterable(task))
else:
- task.done.callback((failure, task))
+ # This fires the errback when the task is done but has failed.
+ task.done.callback(failure)
self.failed(failure, task)
@@ -58,11 +58,11 @@ class TaskManager(object):
We have successfully completed a measurement.
"""
self._active_tasks.remove(task)
- self.completedTasks += 1
self._fillSlots()
- task.done.callback(result)
+ # Fires the done deferred when the task has completed
+ task.done.callback(task)
self.succeeded(result, task)
def _run(self, task):
@@ -105,12 +105,6 @@ class TaskManager(object):
self._fillSlots()
- def started(self, task):
- """
- This hook will get called every time a task has been started.
- """
- pass
-
def failed(self, failure, task):
"""
This hoook is called every time a task has failed.
@@ -138,28 +132,24 @@ class MeasurementManager(TaskManager):
NetTest on the contrary is aware of the typology of measurements that it is
dispatching as they are logically grouped by test file.
"""
+ # XXX tweak these values
retries = 2
-
- failures = []
concurrency = 10
director = None
- def started(self, measurement):
- self.director.measurementStarted(measurement)
-
def succeeded(self, result, measurement):
self.director.measurementSucceeded(measurement)
def failed(self, failure, measurement):
self.director.measurementFailed(failure, measurement)
-
class ReportEntryManager(TaskManager):
- director = None
+ # XXX tweak these values
+ retries = 3
+ concurrency = 20
- def started(self, task):
- pass
+ director = None
def succeeded(self, result, task):
pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index d6cadc3..9146924 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -4,7 +4,7 @@ from twisted.internet import defer, reactor
from twisted.trial.runner import filenameToModule
from twisted.python import usage, reflect
-from ooni.tasks import Measurement, TaskMediator
+from ooni.tasks import Measurement
from ooni.utils import log, checkForRoot, NotRootError
from inspect import getmembers
@@ -13,8 +13,53 @@ from StringIO import StringIO
class NoTestCasesFound(Exception):
pass
+class NetTestState(object):
+ def __init__(self, allTasksDone):
+ """
+ This keeps track of the state of a running NetTests case.
+
+ Args:
+ allTasksDone is a deferred that will get fired once all the NetTest
+ cases have reached a final done state.
+ """
+ self.doneTasks = 0
+ self.tasks = 0
+
+ self.completedScheduling = False
+ self.allTasksDone = allTasksDone
+
+ def created(self):
+ self.tasks += 1
+
+ def checkAllTasksDone(self):
+ if self.completedScheduling and \
+ self.doneTasks == self.tasks:
+ self.allTasksDone.callback(self.doneTasks)
+
+ def taskDone(self, result):
+ """
+ This is called every time a task has finished running.
+ """
+ self.doneTasks += 1
+ self.checkAllTasksDone()
+
+ def allTasksScheduled(self):
+ """
+ This should be called once all the tasks that need to run have been
+ scheduled.
+
+ XXX this is ghetto.
+ The reason for which we are calling allTasksDone inside of the
+ allTasksScheduled method is called after all tasks are done, then we
+ will run into a race condition. The race is that we don't end up
+ checking that all the tasks are complete because no task is to be
+ scheduled.
+ """
+ self.completedScheduling = True
+ self.checkAllTasksDone()
+
class NetTest(object):
- measurementManager = None
+ director = None
method_prefix = 'test'
def __init__(self, net_test_file, options, report):
@@ -29,17 +74,12 @@ class NetTest(object):
self.report = report
self.test_cases = self.loadNetTest(net_test_file)
- self.allMeasurementsDone = defer.Deferred()
- self.allReportsDone = defer.Deferred()
-
- # This should fire when all the measurements have been completed and
+ # This will fire when all the measurements have been completed and
# all the reports are done. Done means that they have either completed
# successfully or all the possible retries have been reached.
- self.done = defer.DeferredList([self.allMeasurementsDone,
- self.allReportsDone])
+ self.done = defer.Deferred()
- # XXX Fire the done when also all the reporting tasks have been completed.
- # self.done = self.allMeasurementsDone
+ self.state = NetTestState(self.done)
def start(self):
"""
@@ -47,10 +87,42 @@ class NetTest(object):
Start tests and generate measurements.
"""
self.setUpNetTestCases()
- self.measurementManager.schedule(self.generateMeasurements())
-
+ self.director.startMeasurements(self.generateMeasurements())
return self.done
+ def doneReport(self, result):
+ """
+ This will get called every time a measurement is done and therefore a
+ measurement is done.
+
+ The state for the NetTest is informed of the fact that another task has
+ reached the done state.
+ """
+ self.state.taskDone()
+ return result
+
+ def generateMeasurements(self):
+ """
+ This is a generator that yields measurements and registers the
+ callbacks for when a measurement is successful or has failed.
+ """
+ for test_class, test_method in self.test_cases:
+ for test_input in test_class.inputs:
+ measurement = Measurement(test_class, test_method, test_input)
+
+ measurement.done.addCallback(self.director.measurementSucceeded)
+ measurement.done.addErrback(self.director.measurementFailed)
+
+ measurement.done.addCallback(self.report.write)
+ measurement.done.addErrback(self.director.reportEntryFailed)
+
+ measurement.done.addBoth(self.doneReport)
+
+ self.state.taskCreated()
+ yield measurement
+
+ self.state.allTasksScheduled()
+
def loadNetTest(self, net_test_file):
"""
Creates all the necessary test_cases (a list of tuples containing the
@@ -124,33 +196,6 @@ class NetTest(object):
pass
return test_cases
- def succeeded(self, measurement):
- """
- This gets called when a measurement has succeeded.
- """
- self.report.write(measurement)
-
- def generateMeasurements(self):
- """
- This is a generator that yields measurements and sets their timeout
- value and their netTest attribute.
- """
- task_mediator = TaskMediator(self.allMeasurementsDone)
- for test_class, test_method in self.test_cases:
- for test_input in test_class.inputs:
- measurement = Measurement(test_class, test_method,
- test_input, self, task_mediator)
- measurement.netTest = self
- yield measurement
- task_mediator.allTasksScheduled()
-
- @task_mediator.allTasksDone.addCallback
- def done(result):
- """
- Once all the MeasurementsTasks have been completed all the report
- tasks will have been scheduled.
- """
- self.report.report_mediator.allTasksScheduled()
def setUpNetTestCases(self):
"""
diff --git a/ooni/reporter.py b/ooni/reporter.py
index d24edcc..80595f9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -31,7 +31,7 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents
from ooni import config
-from ooni.tasks import ReportEntry, TaskMediator
+from ooni.tasks import ReportEntry
def createPacketReport(packet_list):
"""
@@ -158,6 +158,8 @@ def getTestDetails(options):
return test_details
class OReporter(object):
+ created = defer.Deferred()
+
def __init__(self, cmd_line_options):
self.cmd_line_options = dict(cmd_line_options)
@@ -380,15 +382,19 @@ class ReportClosed(Exception):
pass
class Report(object):
- reportEntryManager = None
-
- def __init__(self, reporters):
+ def __init__(self, reporters, reportEntryManager):
"""
- This will instantiate all the reporters and add them to the list of
- available reporters.
+ This is an abstraction layer on top of all the configured reporters.
+
+ It allows to lazily write to the reporters that are to be used.
+
+ Args:
+
+ reporters:
+ a list of :class:ooni.reporter.OReporter
- net_test:
- is a reference to the net_test to which the report object belongs to.
+ reportEntryManager:
+ an instance of :class:ooni.tasks.ReportEntryManager
"""
self.reporters = []
for r in reporters:
@@ -398,13 +404,14 @@ class Report(object):
self.createReports()
self.done = defer.Deferred()
- self.done.addCallback(self.finish)
+ self.done.addCallback(self.close)
- self.report_mediator = TaskMediator(self.done)
+ self.reportEntryManager = reportEntryManager
- def createReports(self):
+ def open(self):
"""
- This will create all the reports that need to be created.
+ This will create all the reports that need to be created and fires the
+ created callback of the reporter whose report got created.
"""
for reporter in self.reporters:
d = defer.maybeDeferred(reporter.createReport)
@@ -413,20 +420,41 @@ class Report(object):
def write(self, measurement):
"""
This is a lazy call that will write to all the reporters by waiting on
- the created callback to fire.
+ them to be created.
- The report_write_task is created before we attach the callback so that
- the report mediator is aware of the total number of created reportEntry
- tasks.
+ Will return a deferred that will fire once the report for the specified
+ measurement have been written to all the reporters.
+
+ Args:
+
+ measurement:
+ an instance of :class:ooni.tasks.Measurement
+
+ Returns:
+ a deferred list that will fire once all the report entries have
+ been written.
"""
+ dl = []
for reporter in self.reporters:
- report_write_task = ReportEntry(reporter, measurement,
- self.report_mediator)
- @reporter.created.addCallback
- def cb(result):
+ def writeReportEntry(result):
+ report_write_task = ReportEntry(reporter, measurement)
self.reportEntryManager.schedule(report_write_task)
+ return report_write_task.done
- def finish(self, result):
+ d = reporter.created.addBoth(writeReportEntry)
+ dl.append(d)
+
+ return defer.DeferredList(dl)
+
+ def close(self, _):
+ """
+ Close the report by calling it's finish method.
+
+ Returns:
+ a :class:twisted.internet.defer.DeferredList that will fire when
+ all the reports have been closed.
+
+ """
dl = []
for reporter in self.reporters:
d = defer.maybeDeferred(reporter.finish)
diff --git a/ooni/tasks.py b/ooni/tasks.py
index f2a9bae..28aaca4 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -5,7 +5,7 @@ from twisted.internet import defer, reactor
class BaseTask(object):
_timer = None
- def __init__(self, mediator=None):
+ def __init__(self):
"""
If you want to schedule a task multiple times, remember to create fresh
instances of it.
@@ -19,10 +19,8 @@ class BaseTask(object):
# This is a deferred that gets called when a test has reached it's
# final status, this means: all retries have been attempted or the test
# has successfully executed.
+ # Such deferred will be called on completion by the TaskManager.
self.done = defer.Deferred()
- if mediator:
- mediator.created()
- self.done.addCallback(mediator.taskDone)
def _failed(self, failure):
self.failures += 1
@@ -97,8 +95,7 @@ class TaskWithTimeout(BaseTask):
pass
class Measurement(TaskWithTimeout):
- def __init__(self, test_class, test_method, test_input, net_test,
- mediator):
+ def __init__(self, test_class, test_method, test_input):
"""
test_class:
is the class, subclass of NetTestCase, of the test to be run
@@ -121,9 +118,7 @@ class Measurement(TaskWithTimeout):
self.test_instance.setUp()
self.test = getattr(self.test_instance, test_method)
- self.netTest = net_test
-
- TaskWithTimeout.__init__(self, mediator)
+ TaskWithTimeout.__init__(self)
def succeeded(self, result):
return self.netTest.succeeded(self)
@@ -139,60 +134,12 @@ class Measurement(TaskWithTimeout):
return d
class ReportEntry(TaskWithTimeout):
- def __init__(self, reporter, measurement, task_mediator):
+ def __init__(self, reporter, measurement):
self.reporter = reporter
self.measurement = measurement
- TaskWithTimeout.__init__(self, task_mediator)
+ TaskWithTimeout.__init__(self)
def run(self):
return self.reporter.writeReportEntry(self.measurement)
-
-class TaskMediator(object):
- def __init__(self, allTasksDone):
- """
- This implements a Mediator/Observer pattern to keep track of when Tasks
- that are logically linked together have all reached a final done stage.
-
- Args:
- allTasksDone is a deferred that will get fired once all the tasks
- have been completed.
- """
- self.doneTasks = 0
- self.tasks = 0
-
- self.completedScheduling = False
- self.allTasksDone = allTasksDone
-
- def created(self):
- self.tasks += 1
-
- def checkAllTasksDone(self):
- if self.completedScheduling and \
- self.doneTasks == self.tasks:
- self.allTasksDone.callback(self.doneTasks)
-
- def taskDone(self, result):
- """
- This is called every time a task has finished running.
- """
- self.doneTasks += 1
- self.checkAllTasksDone()
-
- def allTasksScheduled(self):
- """
- This should be called once all the tasks that need to run have been
- scheduled.
-
- XXX this is ghetto.
- The reason for which we are calling allTasksDone inside of the
- allTasksScheduled method is called after all tasks are done, then we
- will run into a race condition. The race is that we don't end up
- checking that all the tasks are complete because no task is to be
- scheduled.
- """
- self.completedScheduling = True
- self.checkAllTasksDone()
-
-
diff --git a/tests/test_nettest.py b/tests/test_nettest.py
index 3c7bdf6..a029fae 100644
--- a/tests/test_nettest.py
+++ b/tests/test_nettest.py
@@ -10,6 +10,8 @@ from ooni.nettest import FailureToLoadNetTest
from ooni.tasks import BaseTask
from ooni.utils import NotRootError
+from ooni.director import Director
+
from ooni.managers import TaskManager
from tests.mocks import MockMeasurement, MockMeasurementFailOnce
@@ -223,6 +225,7 @@ class TestNetTest(unittest.TestCase):
net_test = NetTest(StringIO(net_test_string_with_file),
dummyOptionsWithFile, MockReporter())
net_test.measurementManager = MockMeasurementManager()
+ net_test.director = Director()
d = net_test.start()
@d.addCallback
1
0
commit 8a889335fea991c69afbe001e4584615b484eafb
Author: Arturo Filastò <art(a)fuffa.org>
Date: Wed Jan 16 21:00:12 2013 +0100
Start the reactor in the Director
---
ooni/oonicli.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 1959de8..ac1b080 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -190,6 +190,8 @@ def runWithDirector():
log.err('Missing required option: "%s"' % option_name)
print options.getUsage()
+ reactor.run()
+
def run():
"""
Parses command line arguments of test.
1
0
[ooni-probe/master] Add a shutdown hook for when everything is done
by isis@torproject.org 10 Mar '13
by isis@torproject.org 10 Mar '13
10 Mar '13
commit 96b6c644955114f17d15da924ad39a5b7c3583d7
Author: Arturo Filastò <art(a)fuffa.org>
Date: Wed Jan 16 21:02:55 2013 +0100
Add a shutdown hook for when everything is done
---
ooni/oonicli.py | 13 ++++++++++---
1 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index ac1b080..71de7e1 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -159,6 +159,13 @@ def parseOptions():
return dict(cmd_line_options)
+def shutdown(result):
+ """
+ This will get called once all the operations that need to be done in the
+ current oonicli session have been completed.
+ """
+ reator.stop()
+
def runWithDirector():
"""
Instance the director, parse command line options and start an ooniprobe
@@ -185,13 +192,13 @@ def runWithDirector():
director = Director(reporters)
try:
- director.startNetTest(net_test_loader, net_test_options)
+ d = director.startNetTest(net_test_loader, net_test_options)
+ d.addBoth(shutdown)
+ reactor.run()
except MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print options.getUsage()
- reactor.run()
-
def run():
"""
Parses command line arguments of test.
1
0