July 2020 - tor-commits - lists.torproject.org

[onionperf/develop] Updates analysis version to 3.0
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 5c4c77bb0e0823f6fa9ae8046b86c911915265c9 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Jul 9 15:56:40 2020 +0100 Updates analysis version to 3.0 --- onionperf/analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 2466aad..245d9ae 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -26,7 +26,7 @@ class OPAnalysis(Analysis): def __init__(self, nickname=None, ip_address=None): super().__init__(nickname, ip_address) - self.json_db = {'type':'onionperf', 'version':'2.0', 'data':{}} + self.json_db = {'type' : 'onionperf', 'version' : '3.0', 'data' : {}} self.torctl_filepaths = [] def add_torctl_file(self, filepath): @@ -113,7 +113,7 @@ class OPAnalysis(Analysis): if 'type' not in db or 'version' not in db: logging.warning("'type' or 'version' not present in database") return None - elif db['type'] != 'onionperf' or str(db['version']) >= '3.': + elif db['type'] != 'onionperf' or str(db['version']) >= '4.': logging.warning("type or version not supported (type={0}, version={1})".format(db['type'], db['version'])) return None else:

1 0

[onionperf/develop] Remove stream_summary from analysis results
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 560194bb1f4f25975cde6123fde75e7ff3754ddb Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jul 10 21:03:19 2020 +0100 Remove stream_summary from analysis results --- onionperf/analysis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 64b4b5b..53d879c 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -65,8 +65,9 @@ class OPAnalysis(Analysis): self.measurement_ip = "unknown" self.json_db['data'].setdefault(self.nickname, {'measurement_ip' : self.measurement_ip}).setdefault(json_db_key, parser.get_data()) - self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") - self.json_db['data'][self.nickname]["tgen"].pop("init_ts") + self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") + self.json_db['data'][self.nickname]["tgen"].pop("init_ts") + self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") self.did_analysis = True

1 0

[onionperf/develop] Updates visualization code to use new dictionary structure
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 1b6c211ea77c7ba7f96c0cc53e7d392b9f6eb1b7 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Jul 9 15:33:39 2020 +0100 Updates visualization code to use new dictionary structure --- onionperf/analysis.py | 7 +++++++ onionperf/visualization.py | 44 +++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 245d9ae..64b4b5b 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -92,6 +92,13 @@ class OPAnalysis(Analysis): logging.info("done!") + + def get_tgen_streams(self, node): + try: + return self.json_db['data'][node]['tgen']['streams'] + except: + return None + @classmethod def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()): filepath = os.path.abspath(os.path.expanduser("{0}".format(filename))) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 0a2a9d9..48c837b 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -51,38 +51,36 @@ class TGenVisualization(Visualization): self.page.close() def __extract_data_frame(self): - transfers = [] + streams = [] for (analyses, label) in self.datasets: for analysis in analyses: for client in analysis.get_nodes(): - tgen_transfers = analysis.get_tgen_transfers(client) - for transfer_id, transfer_data in tgen_transfers.items(): - transfer = {"transfer_id": transfer_id, "label": label, - "filesize_bytes": transfer_data["filesize_bytes"], - "error_code": None} - transfer["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" - if "elapsed_seconds" in transfer_data: - s = transfer_data["elapsed_seconds"] + tgen_streams = analysis.get_tgen_streams(client) + for stream_id, stream_data in tgen_streams.items(): + stream = {"stream_id": stream_id, "label": label, + "filesize_bytes": stream_data["stream_info"]["recvsize"]} + stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" + if "time_info" in stream_data: + s = stream_data["time_info"] if "payload_progress" in s: # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: - transfer["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) - if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]: - transfer["mbps"] = 4.194304 / (s["payload_progress"]["0.2"] - s["payload_progress"]["0.1"]) - if "first_byte" in s: - transfer["time_to_first_byte"] = s["first_byte"] - if "last_byte" in s: - transfer["time_to_last_byte"] = s["last_byte"] - if "error_code" in transfer_data and transfer_data["error_code"] != "NONE": - transfer["error_code"] = transfer_data["error_code"] - if "unix_ts_start" in transfer_data: - transfer["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) - transfers.append(transfer) - self.data = pd.DataFrame.from_records(transfers, index="transfer_id") + if stream_data["stream_info"]["recv_size"] == 5242880 and "0.2" in s["elapsed_seconds"]["payload_progress_recv"]: + stream["mbps"] = 4.194304 / (s["elapsed_seconds"]["payload_progress_recv"]["0.2"] - s["elapsed_seconds"]["payload_progress_recv"]["0.1"]) + + if "usecs-to-first-byte-recv" in s: + stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 + if "usecs-to-last-byte-recv" in s: + stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 + if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": + stream["error_code"] = stream_data["transport_info"]["error"] + if "unix_ts_start" in stream_data: + stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) + streams.append(stream) + self.data = pd.DataFrame.from_records(streams, index="stream_id") def __plot_firstbyte_ecdf(self): for server in self.data["server"].unique():

1 0

[onionperf/develop] Adds support for previous analyses
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 074842a412c2d7ede45a40eb4fb610cd32619c11 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Jul 11 11:38:54 2020 +0100 Adds support for previous analyses --- onionperf/analysis.py | 6 ++++ onionperf/visualization.py | 72 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 53d879c..49d6dba 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -100,6 +100,12 @@ class OPAnalysis(Analysis): except: return None + def get_tgen_transfers(self, node): + try: + return self.json_db['data'][node]['tgen']['transfers'] + except: + return None + @classmethod def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()): filepath = os.path.abspath(os.path.expanduser("{0}".format(filename))) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 48c837b..68ad751 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -54,32 +54,62 @@ class TGenVisualization(Visualization): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: - for client in analysis.get_nodes(): - tgen_streams = analysis.get_tgen_streams(client) - for stream_id, stream_data in tgen_streams.items(): - stream = {"stream_id": stream_id, "label": label, - "filesize_bytes": stream_data["stream_info"]["recvsize"]} - stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" - if "time_info" in stream_data: - s = stream_data["time_info"] - if "payload_progress" in s: + if analysis.json_db['version'] >= '3': + for client in analysis.get_nodes(): + tgen_streams = analysis.get_tgen_streams(client) + for stream_id, stream_data in tgen_streams.items(): + stream = {"stream_id": stream_id, "label": label, + "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} + stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" + if "time_info" in stream_data: + s = stream_data["time_info"] + if "usecs-to-first-byte-recv" in s: + stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 + if "usecs-to-last-byte-recv" in s: + stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 + if "elapsed_seconds" in stream_data: + s = stream_data["elapsed_seconds"] + # Explanation of the math below for computing Mbps: From filesize_bytes + # and payload_progress fields we can compute the number of seconds that + # have elapsed between receiving bytes 524,288 and 1,048,576, which is a + # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. + # We want the reciprocal of that value with unit megabits per second. + if stream_data["stream_info"]["recvsize"] == 5242880 and "0.2" in s["payload_progress_recv"]: + stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) + if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": + stream["error_code"] = stream_data["transport_info"]["error"] + if "unix_ts_start" in stream_data: + stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) + streams.append(stream) + else: + for client in analysis.get_nodes(): + tgen_transfers = analysis.get_tgen_transfers(client) + for transfer_id, transfer_data in tgen_transfers.items(): + stream = {"stream_id": transfer_id, "label": label, + "filesize_bytes": transfer_data["filesize_bytes"]} + stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" + if "elapsed_seconds" in transfer_data: + s = transfer_data["elapsed_seconds"] + if "payload_progress" in s: # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if stream_data["stream_info"]["recv_size"] == 5242880 and "0.2" in s["elapsed_seconds"]["payload_progress_recv"]: - stream["mbps"] = 4.194304 / (s["elapsed_seconds"]["payload_progress_recv"]["0.2"] - s["elapsed_seconds"]["payload_progress_recv"]["0.1"]) - - if "usecs-to-first-byte-recv" in s: - stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 - if "usecs-to-last-byte-recv" in s: - stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 - if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": - stream["error_code"] = stream_data["transport_info"]["error"] - if "unix_ts_start" in stream_data: - stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) - streams.append(stream) + if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: + stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) + if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]: + stream["mbps"] = 4.194304 / (s["payload_progress"]["0.2"] - s["payload_progress"]["0.1"]) + if "first_byte" in s: + stream["time_to_first_byte"] = s["first_byte"] + if "last_byte" in s: + stream["time_to_last_byte"] = s["last_byte"] + if "error_code" in transfer_data and transfer_data["error_code"] != "NONE": + stream["error_code"] = transfer_data["error_code"] + if "unix_ts_start" in transfer_data: + stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) + streams.append(stream) + self.data = pd.DataFrame.from_records(streams, index="stream_id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/develop] Update path to TGen 1.0.0 binary in README.md.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 5d5bff153dfd4a4b98d2bd3632cc4e813498ae0a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:45:01 2020 +0200 Update path to TGen 1.0.0 binary in README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 032dd41..e3df3bf 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ cmake .. make ``` -The TGen binary will be contained in `~/tgen/build/tgen`, which is also the path that needs to be passed to OnionPerf's `--tgen` parameter when doing measurements. +The TGen binary will be contained in `~/tgen/build/src/tgen`, which is also the path that needs to be passed to OnionPerf's `--tgen` parameter when doing measurements. ### OnionPerf

1 0

[onionperf/develop] Make a few whitespace fixes.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 2eceb8c7044a770059f4d1ad06241b60216d4b84 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:50:41 2020 +0200 Make a few whitespace fixes. --- onionperf/analysis.py | 4 ++-- onionperf/visualization.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 49d6dba..8506756 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -26,7 +26,7 @@ class OPAnalysis(Analysis): def __init__(self, nickname=None, ip_address=None): super().__init__(nickname, ip_address) - self.json_db = {'type' : 'onionperf', 'version' : '3.0', 'data' : {}} + self.json_db = {'type': 'onionperf', 'version': '3.0', 'data': {}} self.torctl_filepaths = [] def add_torctl_file(self, filepath): @@ -64,7 +64,7 @@ class OPAnalysis(Analysis): if self.measurement_ip is None: self.measurement_ip = "unknown" - self.json_db['data'].setdefault(self.nickname, {'measurement_ip' : self.measurement_ip}).setdefault(json_db_key, parser.get_data()) + self.json_db['data'].setdefault(self.nickname, {'measurement_ip': self.measurement_ip}).setdefault(json_db_key, parser.get_data()) self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") self.json_db['data'][self.nickname]["tgen"].pop("init_ts") self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 7cc36d1..52d08fc 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -68,7 +68,7 @@ class TGenVisualization(Visualization): stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 if "usecs-to-last-byte-recv" in s: stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 - if "elapsed_seconds" in stream_data: + if "elapsed_seconds" in stream_data: s = stream_data["elapsed_seconds"] # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that @@ -93,11 +93,11 @@ class TGenVisualization(Visualization): if "elapsed_seconds" in transfer_data: s = transfer_data["elapsed_seconds"] if "payload_progress" in s: - # Explanation of the math below for computing Mbps: From filesize_bytes - # and payload_progress fields we can compute the number of seconds that - # have elapsed between receiving bytes 524,288 and 1,048,576, which is a - # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. - # We want the reciprocal of that value with unit megabits per second. + # Explanation of the math below for computing Mbps: From filesize_bytes + # and payload_progress fields we can compute the number of seconds that + # have elapsed between receiving bytes 524,288 and 1,048,576, which is a + # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. + # We want the reciprocal of that value with unit megabits per second. if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]:

1 0

[onionperf/develop] Renames stream_id to id
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit deff90a2b21b4be8e51036dd352d1f4c398836ea Author: Ana Custura <ana(a)netstat.org.uk> Date: Sun Jul 12 15:04:35 2020 +0100 Renames stream_id to id --- onionperf/visualization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 68ad751..f85c10f 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -58,7 +58,7 @@ class TGenVisualization(Visualization): for client in analysis.get_nodes(): tgen_streams = analysis.get_tgen_streams(client) for stream_id, stream_data in tgen_streams.items(): - stream = {"stream_id": stream_id, "label": label, + stream = {"id": stream_id, "label": label, "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" if "time_info" in stream_data: @@ -85,7 +85,7 @@ class TGenVisualization(Visualization): for client in analysis.get_nodes(): tgen_transfers = analysis.get_tgen_transfers(client) for transfer_id, transfer_data in tgen_transfers.items(): - stream = {"stream_id": transfer_id, "label": label, + stream = {"id": transfer_id, "label": label, "filesize_bytes": transfer_data["filesize_bytes"]} stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" if "elapsed_seconds" in transfer_data: @@ -110,7 +110,7 @@ class TGenVisualization(Visualization): stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) streams.append(stream) - self.data = pd.DataFrame.from_records(streams, index="stream_id") + self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self): for server in self.data["server"].unique():

1 0

[onionperf/develop] Updates README.md following the update to TGen 1.0.0
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 7282eea4f8349fb8e95a5e0c8c81f574abe8a30d Author: Ana Custura <ana(a)netstat.org.uk> Date: Sun Jul 12 15:45:29 2020 +0100 Updates README.md following the update to TGen 1.0.0 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 77384c6..032dd41 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ sudo apt install cmake libglib2.0-dev libigraph0-dev make cd ~/ git clone https://github.com/shadow/tgen.git cd tgen/ -git checkout -b v0.0.1 v0.0.1 mkdir build cd build/ cmake .. @@ -281,7 +280,7 @@ The PDF output file contains visualizations of the following metrics: The CSV output file contains the same data that is visualized in the PDF file. It contains the following columns: -- `transfer_id` is the identifier used in the TGen client logs which may be useful to look up more details about a specific measurement. +- `id` is the identifier used in the TGen client logs which may be useful to look up more details about a specific measurement. - `error_code` is an optional error code if a measurement did not succeed. - `filesize_bytes` is the requested file size in bytes. - `label` is the data set label as given in the `--data/-d` parameter to the `visualize` mode.

1 0

[onionperf/develop] Restore #40004 fix that got lost during rebase.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 120e425ebf78d36b3e1efd06de1e6cbe134a4a58 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:47:58 2020 +0200 Restore #40004 fix that got lost during rebase. --- onionperf/visualization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f85c10f..7cc36d1 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -59,7 +59,8 @@ class TGenVisualization(Visualization): tgen_streams = analysis.get_tgen_streams(client) for stream_id, stream_data in tgen_streams.items(): stream = {"id": stream_id, "label": label, - "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} + "filesize_bytes": int(stream_data["stream_info"]["recvsize"]), + "error_code": None} stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" if "time_info" in stream_data: s = stream_data["time_info"] @@ -86,7 +87,8 @@ class TGenVisualization(Visualization): tgen_transfers = analysis.get_tgen_transfers(client) for transfer_id, transfer_data in tgen_transfers.items(): stream = {"id": transfer_id, "label": label, - "filesize_bytes": transfer_data["filesize_bytes"]} + "filesize_bytes": transfer_data["filesize_bytes"], + "error_code": None} stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" if "elapsed_seconds" in transfer_data: s = transfer_data["elapsed_seconds"]

1 0

[onionperf/develop] Remove now unused imports.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 0189c3a73339c7b2554aadb1c60b1dc92d534572 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:57:11 2020 +0200 Remove now unused imports. --- onionperf/analysis.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 8506756..19c0192 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -4,11 +4,8 @@ See LICENSE for licensing information ''' -import sys, os, re, json, datetime, logging +import os, re, json, datetime, logging -from multiprocessing import Pool, cpu_count -from signal import signal, SIGINT, SIG_IGN -from socket import gethostname from abc import ABCMeta, abstractmethod # stem imports

1 0