[tor-commits] [ooni-probe/master] Refactor the logic for dumping the YAML reports of packets.

art at torproject.org art at torproject.org
Sat Nov 24 13:29:27 UTC 2012


commit 620a591c290d4b47803dc2f4cf07e343d56d3618
Author: Arturo Filastò <art at fuffa.org>
Date:   Sat Nov 24 14:24:48 2012 +0100

    Refactor the logic for dumping the YAML reports of packets.
    * Override the yaml.representer.represent_data as a hackish quick fix to the
      problem.
    * Outline a possible long term solution to it.
---
 nettests/core/traceroute.py |    8 ++++-
 ooni/reporter.py            |   78 +++++++++++++++++++++++++++++++++++++++++--
 ooni/utils/hacks.py         |   45 -------------------------
 3 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py
index a6fc9f1..6a443d9 100644
--- a/nettests/core/traceroute.py
+++ b/nettests/core/traceroute.py
@@ -50,6 +50,12 @@ class TracerouteTest(scapyt.BaseScapyTest):
         return max_ttl, timeout
 
 
+    def postProcessor(self, report):
+        tcp_hops = report['test_tcp_traceroute']
+        udp_hops = report['test_udp_traceroute']
+        icmp_hops = report['test_icmp_traceroute']
+
+
     def test_tcp_traceroute(self):
         """
         Does a traceroute to the destination by sending TCP SYN packets
@@ -126,7 +132,7 @@ class TracerouteTest(scapyt.BaseScapyTest):
         dl = []
         max_ttl, timeout = self.max_ttl_and_timeout()
         packets = IP(dst=self.localOptions['backend'],
-                    ttl=(1,max_ttl),id=RandShort())/ICMP()
+                    ttl=(1,max_ttl), id=RandShort())/ICMP()
 
         d = self.sr(packets, timeout=timeout)
         d.addCallback(finished)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 7fc49e2..1e2924d 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -15,6 +15,13 @@ import yaml
 import json
 import traceback
 
+from yaml.representer import *
+from yaml.emitter import *
+from yaml.serializer import *
+from yaml.resolver import *
+
+from scapy.packet import Packet
+
 from twisted.python.util import untilConcludes
 from twisted.trial import reporter
 from twisted.internet import defer, reactor
@@ -23,10 +30,75 @@ from twisted.internet.error import ConnectionRefusedError
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
 from ooni.utils import otime, log, geodata
 
-from ooni.utils.hacks import OSafeRepresenter, OSafeDumper
 from ooni import config
 
-from scapy.all import packet
+def createPacketReport(packet_list):
+    """
+    Takes as input a packet a list.
+
+    Returns a dict containing a dict with the packet
+    summary and the raw packet.
+    """
+    report = []
+    for packet in packet_list:
+        report.append({'raw_packet': str(packet),
+            'summary': str(packet.summary())})
+    return report
+
+class OSafeRepresenter(SafeRepresenter):
+    """
+    This is a custom YAML representer that allows us to represent reports
+    safely.
+    It extends the SafeRepresenter to be able to also represent complex
+    numbers and scapy packet.
+    """
+    def represent_data(self, data):
+        """
+        This is very hackish. There is for sure a better way either by using
+        the add_multi_representer or add_representer, the issue though lies in
+        the fact that Scapy packets are metaclasses that leads to
+        yaml.representer.get_classobj_bases to not be able to properly get the
+        base of class of a Scapy packet.
+        XXX fully debug this problem
+        """
+        if isinstance(data, Packet):
+            data = createPacketReport(data)
+        return SafeRepresenter.represent_data(self, data)
+
+    def represent_complex(self, data):
+        if data.imag == 0.0:
+            data = u'%r' % data.real
+        elif data.real == 0.0:
+            data = u'%rj' % data.imag
+        elif data.imag > 0:
+            data = u'%r+%rj' % (data.real, data.imag)
+        else:
+            data = u'%r%rj' % (data.real, data.imag)
+        return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
+
+OSafeRepresenter.add_representer(complex,
+                                 OSafeRepresenter.represent_complex)
+
+class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
+    """
+    This is a modification of the YAML Safe Dumper to use our own Safe
+    Representer that supports complex numbers.
+    """
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        OSafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
 
 class NoTestIDSpecified(Exception):
     pass
@@ -103,7 +175,7 @@ class OReporter(object):
         log.debug("Writing report")
         test_report = dict(test.report)
 
-        if isinstance(test.input, packet.Packet):
+        if isinstance(test.input, Packet):
             test_input = createPacketReport(test.input)
         else:
             test_input = test.input
diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py
index 8912bbb..d4e832c 100644
--- a/ooni/utils/hacks.py
+++ b/ooni/utils/hacks.py
@@ -8,10 +8,6 @@
 # :authors: Arturo Filastò
 # :licence: see LICENSE
 
-from yaml.representer import *
-from yaml.emitter import *
-from yaml.serializer import *
-from yaml.resolver import *
 
 import copy_reg
 
@@ -67,44 +63,3 @@ def patched_reduce_ex(self, proto):
     else:
         return copy_reg._reconstructor, args
 
-class OSafeRepresenter(SafeRepresenter):
-    """
-    This is a custom YAML representer that allows us to represent reports
-    safely.
-    It extends the SafeRepresenter to be able to also represent complex numbers
-    """
-    def represent_complex(self, data):
-        if data.imag == 0.0:
-            data = u'%r' % data.real
-        elif data.real == 0.0:
-            data = u'%rj' % data.imag
-        elif data.imag > 0:
-            data = u'%r+%rj' % (data.real, data.imag)
-        else:
-            data = u'%r%rj' % (data.real, data.imag)
-        return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
-
-OSafeRepresenter.add_representer(complex,
-                                 OSafeRepresenter.represent_complex)
-
-class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
-    """
-    This is a modification of the YAML Safe Dumper to use our own Safe
-    Representer that supports complex numbers.
-    """
-    def __init__(self, stream,
-            default_style=None, default_flow_style=None,
-            canonical=None, indent=None, width=None,
-            allow_unicode=None, line_break=None,
-            encoding=None, explicit_start=None, explicit_end=None,
-            version=None, tags=None):
-        Emitter.__init__(self, stream, canonical=canonical,
-                indent=indent, width=width,
-                allow_unicode=allow_unicode, line_break=line_break)
-        Serializer.__init__(self, encoding=encoding,
-                explicit_start=explicit_start, explicit_end=explicit_end,
-                version=version, tags=tags)
-        OSafeRepresenter.__init__(self, default_style=default_style,
-                default_flow_style=default_flow_style)
-        Resolver.__init__(self)
-





More information about the tor-commits mailing list