[tor-commits] [onionperf/master] Refine error codes into TOR or TGEN errors.

karsten at torproject.org karsten at torproject.org
Mon Aug 10 09:30:41 UTC 2020


commit 4533b39591cc0a6df124e017366ea71343c842bf
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Jul 17 22:24:43 2020 +0200

    Refine error codes into TOR or TGEN errors.
    
    With this change we include more detailed error codes in visualization
    output. In order to do so we map TGen transfers/streams to TorCtl
    STREAM event details based on source ports and unix_ts_end timestamps.
    This code reuses some concepts used in metrics-lib.
    
    Implements tpo/metrics/onionperf#34218.
---
 onionperf/analysis.py      |   6 +++
 onionperf/visualization.py | 107 +++++++++++++++++++++++++++------------------
 2 files changed, 71 insertions(+), 42 deletions(-)

diff --git a/onionperf/analysis.py b/onionperf/analysis.py
index e269f6a..8fcc0a6 100644
--- a/onionperf/analysis.py
+++ b/onionperf/analysis.py
@@ -97,6 +97,12 @@ class OPAnalysis(Analysis):
         except:
             return None
 
+    def get_tor_streams(self, node):
+        try:
+            return self.json_db['data'][node]['tor']['streams']
+        except:
+            return None
+
     @classmethod
     def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()):
         filepath = os.path.abspath(os.path.expanduser("{0}".format(filename)))
diff --git a/onionperf/visualization.py b/onionperf/visualization.py
index 80c0781..5e7f92b 100644
--- a/onionperf/visualization.py
+++ b/onionperf/visualization.py
@@ -54,50 +54,57 @@ class TGenVisualization(Visualization):
         streams = []
         for (analyses, label) in self.datasets:
             for analysis in analyses:
-                if analysis.json_db['version'] >= '3':
-                    for client in analysis.get_nodes():
-                         tgen_streams = analysis.get_tgen_streams(client)
-                         for stream_id, stream_data in tgen_streams.items():
-                             stream = {"id": stream_id, "label": label,
-                                         "filesize_bytes": int(stream_data["stream_info"]["recvsize"]),
-                                         "error_code": None}
-                             stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public"
-                             if "time_info" in stream_data:
-                                 s = stream_data["time_info"]
-                                 if "usecs-to-first-byte-recv" in s:
-                                     stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000
-                                 if "usecs-to-last-byte-recv" in s:
-                                     stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000
-                             if "elapsed_seconds" in stream_data:
-                                 s = stream_data["elapsed_seconds"]
-                                 # Explanation of the math below for computing Mbps: From stream_info/recvsize
-                                 # and payload_progress_recv fields we can compute the number of seconds that
-                                 # have elapsed between receiving bytes 524,288 and 1,048,576, which is a
-                                 # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits.
-                                 # We want the reciprocal of that value with unit megabits per second.
-                                 if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]:
-                                      stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"])
-                             if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE":
-                                 stream["error_code"] = stream_data["transport_info"]["error"]
-                             if "unix_ts_start" in stream_data:
-                                 stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"])
-                             streams.append(stream)
-                else:
-                    for client in analysis.get_nodes():
-                        tgen_transfers = analysis.get_tgen_transfers(client)
-                        for transfer_id, transfer_data in tgen_transfers.items():
+                for client in analysis.get_nodes():
+                    tor_streams_by_source_port = {}
+                    tor_streams = analysis.get_tor_streams(client)
+                    for tor_stream in tor_streams.values():
+                        if "source" in tor_stream and ":" in tor_stream["source"]:
+                            source_port = tor_stream["source"].split(":")[1]
+                            tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream)
+                    tgen_streams = analysis.get_tgen_streams(client)
+                    tgen_transfers = analysis.get_tgen_transfers(client)
+                    while tgen_streams or tgen_transfers:
+                        error_code = None
+                        source_port = None
+                        unix_ts_end = None
+                        # Explanation of the math below for computing Mbps: For 1 MiB and 5 MiB
+                        # downloads we can extract the number of seconds that have elapsed between
+                        # receiving bytes 524,288 and 1,048,576, which is a total amount of 524,288
+                        # bytes or 4,194,304 bits or 4.194304 megabits. We want the reciprocal of
+                        # that value with unit megabits per second.
+                        if tgen_streams:
+                            stream_id, stream_data = tgen_streams.popitem()
+                            stream = {"id": stream_id, "label": label,
+                                      "filesize_bytes": int(stream_data["stream_info"]["recvsize"]),
+                                      "error_code": None}
+                            stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public"
+                            if "time_info" in stream_data:
+                                s = stream_data["time_info"]
+                                if "usecs-to-first-byte-recv" in s:
+                                    stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000
+                                if "usecs-to-last-byte-recv" in s:
+                                    stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000
+                            if "elapsed_seconds" in stream_data:
+                                s = stream_data["elapsed_seconds"]
+                                if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]:
+                                     stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"])
+                            if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE":
+                                error_code = stream_data["transport_info"]["error"]
+                            if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2:
+                                source_port = stream_data["transport_info"]["local"].split(":")[2]
+                            if "unix_ts_end" in stream_data:
+                                unix_ts_end = stream_data["unix_ts_end"]
+                            if "unix_ts_start" in stream_data:
+                                stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"])
+                        elif tgen_transfers:
+                            transfer_id, transfer_data = tgen_transfers.popitem()
                             stream = {"id": transfer_id, "label": label,
-                                        "filesize_bytes": transfer_data["filesize_bytes"],
-                                        "error_code": None}
+                                      "filesize_bytes": transfer_data["filesize_bytes"],
+                                      "error_code": None}
                             stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public"
                             if "elapsed_seconds" in transfer_data:
                                s = transfer_data["elapsed_seconds"]
                                if "payload_progress" in s:
-                                   # Explanation of the math below for computing Mbps: From filesize_bytes
-                                   # and payload_progress fields we can compute the number of seconds that
-                                   # have elapsed between receiving bytes 524,288 and 1,048,576, which is a
-                                   # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits.
-                                   # We want the reciprocal of that value with unit megabits per second.
                                    if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]:
                                        stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"])
                                    if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]:
@@ -107,11 +114,27 @@ class TGenVisualization(Visualization):
                                if "last_byte" in s:
                                    stream["time_to_last_byte"] = s["last_byte"]
                             if "error_code" in transfer_data and transfer_data["error_code"] != "NONE":
-                                stream["error_code"] = transfer_data["error_code"]
+                                error_code = transfer_data["error_code"]
+                            if "endpoint_local" in transfer_data and len(transfer_data["endpoint_local"].split(":")) > 2:
+                                source_port = transfer_data["endpoint_local"].split(":")[2]
+                            if "unix_ts_end" in transfer_data:
+                                unix_ts_end = transfer_data["unix_ts_end"]
                             if "unix_ts_start" in transfer_data:
                                 stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"])
-                            streams.append(stream)
-
+                        if error_code:
+                            if error_code == "PROXY":
+                                error_code_parts = ["TOR"]
+                            else:
+                                error_code_parts = ["TGEN", error_code]
+                            if source_port and unix_ts_end:
+                                for tor_stream in tor_streams_by_source_port[source_port]:
+                                    if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0:
+                                        if "failure_reason_local" in tor_stream:
+                                            error_code_parts.append(tor_stream["failure_reason_local"])
+                                            if "failure_reason_remote" in tor_stream:
+                                                error_code_parts.append(tor_stream["failure_reason_remote"])
+                            stream["error_code"] = "/".join(error_code_parts)
+                        streams.append(stream)
         self.data = pd.DataFrame.from_records(streams, index="id")
 
     def __plot_firstbyte_ecdf(self):





More information about the tor-commits mailing list