[tor-commits] [ooni-probe/master] * Added an InputUnitProcessor class for generating generators for

[isis at torproject.org]

commit b27f5ad66d0f6cf96c2277d3f4165d7d13e0b84f
Author: Isis Lovecruft <isis at torproject.org>
Date:   Wed Oct 31 16:46:40 2012 +0000

    * Added an InputUnitProcessor class for generating generators for
      processing inputs. In ooni.inputunit
    * Also added a couple of exception classes.
---
 ooni/inputunit.py |  132 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 129 insertions(+), 3 deletions(-)

diff --git a/ooni/inputunit.py b/ooni/inputunit.py
index e5b6187..ab46515 100644
--- a/ooni/inputunit.py
+++ b/ooni/inputunit.py
@@ -1,5 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# inputunit.py
+# ------------
+# Classes and function for working with OONI TestCase inputs.
+#
+# @authors: Arturo Filasto, Isis Lovecruft
+# @version: 0.1.0-alpha
+# @license: see included LICENSE file
+# @copyright: 2012 Arturo Filasto, Isis Lovecruft, The Tor Project Inc.
+#
+
 from twisted.trial import unittest
 
+from zope.interface.exceptions import BrokenImplementation
+
+
+def simpleInputUnitProcessor(input_unit):
+    """A simple InputUnit generator without parsing."""
+    try:
+        assert hasattr(input_unit, '__iter__'), "inputs must be iterable!"
+    except AssertionError, ae:
+        raise BrokenImplementation, ae
+    else:
+        for i in input_unit:
+            yield i
+
+
 class PatchedPyUnitResultAdapter(unittest.PyUnitResultAdapter):
     def __init__(self, original):
         """
@@ -13,8 +39,8 @@ unittest.PyUnitResultAdapter = PatchedPyUnitResultAdapter
 
 class InputUnitFactory(object):
     """
-    This is a factory that takes the size of input units to be generated a set
-    of units that is a python iterable item and outputs InputUnit objects
+    This is a factory that takes the size of input units to be generated and a
+    set of units that is a python iterable item, and outputs InputUnit objects
     containing inputUnitSize elements.
 
     This object is a python iterable, this means that it does not need to keep
@@ -48,7 +74,6 @@ class InputUnitFactory(object):
 
         return InputUnit(input_unit_elements)
 
-
 class InputUnit(object):
     """
     This is a python iterable object that contains the input elements to be
@@ -76,5 +101,106 @@ class InputUnit(object):
     def append(self, input):
         self._inputs.append(input)
 
+class IUProcessorExit(GeneratorExit):
+    """InputUnitProcessor has exited."""
+
+class IUProcessorAborted(Exception):
+    """The InputUnitProcessor was aborted with Inputs remaining."""
+
+class InputUnitProcessor(InputUnit):
+    """
+    Create a generator for returning inputs one-by-one from a
+    :class:`InputUnit` (or any other iterable defined within an instance of
+    :class:`ooni.nettest.TestCase`), and a generator function.
+
+    The :ivar:generator can be a custom generator, or chain of generators, for
+    customized parsing of an InputUnit, or it can be an imported
+    function. There are useful imported functions in the builtin
+    :mod:`itertools`. If no :ivar:`generator` is given, the default one strips
+    whitespace characters, then returns the Input if the input does not begin
+    with a crunch (#) or bang (!). :)
+
+    If :ivar:catchStopIter is True, then catch any StopIterations and return a
+    2-tuple containing (boolean, list(unprocessed)):
+
+        (True, None) when :ivar:iterable is empty, or
+        (False, [unprocessed]) if :ivar:iterable was not empty.
+
+    If :ivar:catchStopIter is False (default), then we catch the StopIteration
+    exception, mark :attr:`empty` as 'True', and reraise the StopIteration.
 
+    xxx fill me in with parameter details
+    """
+    empty = False
+
+    def __init__(self, iterable, input_filter=None, catch_err=False):
+        """
+        Create an InputUnitProcessor.
+
+        xxx fill me in
+        """
+        from itertools import takewhile
+        from types     import GeneratorType
+
+        assert hasattr(iterable, "__iter__"), "That's not an iterable!"
+
+        self._iterable = iterable
+        self._infilter = input_filter
+        self._noerr    = catch_err
+        self._empty    = self.empty
 
+    def __len__(self):
+        return len(self.iterable)
+
+    def __unprocessed__(self):
+        if not self.isdone():
+            unprocessed = \
+                [u for u in self._unit if self._unit is not StopIteration]
+            return (False, unprocessed)
+        else:
+            return (True, None)
+
+    def isdone(self):
+        return self._empty
+
+    def throw(self, exception, message="", traceback=None):
+        try:
+            raise exception, message, traceback
+        except:
+            yield self._iterable.next()
+
+    @staticmethod
+    def __strip__(x):
+        return x.strip()
+
+    @staticmethod
+    def _default_input_filter(x):
+        if not x.startswith('#') and not x.startswith('!'):
+            return True
+        else:
+            return False
+
+    def __make_filter__(self):
+        if self.input_filter and hasattr(self.input_filter, "__call__"):
+            return self.input_filter
+        else:
+            return self._default_input_filter
+
+    def finish(self):
+        if self._noerr:
+            return self.__unprocessed__()
+        else:
+            if not self.isdone():
+                raise IUProcessorAborted
+            else:
+                raise IUProcessorExit
+
+    def process(self):
+        carbon = self.__make_filter__()
+        try:
+            yield takewhile(carbon,
+                            [i for i in self.__strip__(self._iterable)]
+                            ).next()
+        except StopIteration, si:
+            self._empty = True
+            yield self.finish()