August 2020 - tor-commits - lists.torproject.org

[onionperf/master] Adds support for previous analyses
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 074842a412c2d7ede45a40eb4fb610cd32619c11 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Jul 11 11:38:54 2020 +0100 Adds support for previous analyses --- onionperf/analysis.py | 6 ++++ onionperf/visualization.py | 72 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 53d879c..49d6dba 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -100,6 +100,12 @@ class OPAnalysis(Analysis): except: return None + def get_tgen_transfers(self, node): + try: + return self.json_db['data'][node]['tgen']['transfers'] + except: + return None + @classmethod def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()): filepath = os.path.abspath(os.path.expanduser("{0}".format(filename))) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 48c837b..68ad751 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -54,32 +54,62 @@ class TGenVisualization(Visualization): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: - for client in analysis.get_nodes(): - tgen_streams = analysis.get_tgen_streams(client) - for stream_id, stream_data in tgen_streams.items(): - stream = {"stream_id": stream_id, "label": label, - "filesize_bytes": stream_data["stream_info"]["recvsize"]} - stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" - if "time_info" in stream_data: - s = stream_data["time_info"] - if "payload_progress" in s: + if analysis.json_db['version'] >= '3': + for client in analysis.get_nodes(): + tgen_streams = analysis.get_tgen_streams(client) + for stream_id, stream_data in tgen_streams.items(): + stream = {"stream_id": stream_id, "label": label, + "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} + stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" + if "time_info" in stream_data: + s = stream_data["time_info"] + if "usecs-to-first-byte-recv" in s: + stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 + if "usecs-to-last-byte-recv" in s: + stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 + if "elapsed_seconds" in stream_data: + s = stream_data["elapsed_seconds"] + # Explanation of the math below for computing Mbps: From filesize_bytes + # and payload_progress fields we can compute the number of seconds that + # have elapsed between receiving bytes 524,288 and 1,048,576, which is a + # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. + # We want the reciprocal of that value with unit megabits per second. + if stream_data["stream_info"]["recvsize"] == 5242880 and "0.2" in s["payload_progress_recv"]: + stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) + if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": + stream["error_code"] = stream_data["transport_info"]["error"] + if "unix_ts_start" in stream_data: + stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) + streams.append(stream) + else: + for client in analysis.get_nodes(): + tgen_transfers = analysis.get_tgen_transfers(client) + for transfer_id, transfer_data in tgen_transfers.items(): + stream = {"stream_id": transfer_id, "label": label, + "filesize_bytes": transfer_data["filesize_bytes"]} + stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" + if "elapsed_seconds" in transfer_data: + s = transfer_data["elapsed_seconds"] + if "payload_progress" in s: # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if stream_data["stream_info"]["recv_size"] == 5242880 and "0.2" in s["elapsed_seconds"]["payload_progress_recv"]: - stream["mbps"] = 4.194304 / (s["elapsed_seconds"]["payload_progress_recv"]["0.2"] - s["elapsed_seconds"]["payload_progress_recv"]["0.1"]) - - if "usecs-to-first-byte-recv" in s: - stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 - if "usecs-to-last-byte-recv" in s: - stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 - if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": - stream["error_code"] = stream_data["transport_info"]["error"] - if "unix_ts_start" in stream_data: - stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) - streams.append(stream) + if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: + stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) + if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]: + stream["mbps"] = 4.194304 / (s["payload_progress"]["0.2"] - s["payload_progress"]["0.1"]) + if "first_byte" in s: + stream["time_to_first_byte"] = s["first_byte"] + if "last_byte" in s: + stream["time_to_last_byte"] = s["last_byte"] + if "error_code" in transfer_data and transfer_data["error_code"] != "NONE": + stream["error_code"] = transfer_data["error_code"] + if "unix_ts_start" in transfer_data: + stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) + streams.append(stream) + self.data = pd.DataFrame.from_records(streams, index="stream_id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/master] Updates analysis version to 3.0
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 5c4c77bb0e0823f6fa9ae8046b86c911915265c9 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Jul 9 15:56:40 2020 +0100 Updates analysis version to 3.0 --- onionperf/analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 2466aad..245d9ae 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -26,7 +26,7 @@ class OPAnalysis(Analysis): def __init__(self, nickname=None, ip_address=None): super().__init__(nickname, ip_address) - self.json_db = {'type':'onionperf', 'version':'2.0', 'data':{}} + self.json_db = {'type' : 'onionperf', 'version' : '3.0', 'data' : {}} self.torctl_filepaths = [] def add_torctl_file(self, filepath): @@ -113,7 +113,7 @@ class OPAnalysis(Analysis): if 'type' not in db or 'version' not in db: logging.warning("'type' or 'version' not present in database") return None - elif db['type'] != 'onionperf' or str(db['version']) >= '3.': + elif db['type'] != 'onionperf' or str(db['version']) >= '4.': logging.warning("type or version not supported (type={0}, version={1})".format(db['type'], db['version'])) return None else:

1 0

[onionperf/master] Remove stream_summary from analysis results
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 560194bb1f4f25975cde6123fde75e7ff3754ddb Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jul 10 21:03:19 2020 +0100 Remove stream_summary from analysis results --- onionperf/analysis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 64b4b5b..53d879c 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -65,8 +65,9 @@ class OPAnalysis(Analysis): self.measurement_ip = "unknown" self.json_db['data'].setdefault(self.nickname, {'measurement_ip' : self.measurement_ip}).setdefault(json_db_key, parser.get_data()) - self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") - self.json_db['data'][self.nickname]["tgen"].pop("init_ts") + self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") + self.json_db['data'][self.nickname]["tgen"].pop("init_ts") + self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") self.did_analysis = True

1 0

[onionperf/master] Updates visualization code to use new dictionary structure
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 1b6c211ea77c7ba7f96c0cc53e7d392b9f6eb1b7 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Jul 9 15:33:39 2020 +0100 Updates visualization code to use new dictionary structure --- onionperf/analysis.py | 7 +++++++ onionperf/visualization.py | 44 +++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 245d9ae..64b4b5b 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -92,6 +92,13 @@ class OPAnalysis(Analysis): logging.info("done!") + + def get_tgen_streams(self, node): + try: + return self.json_db['data'][node]['tgen']['streams'] + except: + return None + @classmethod def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()): filepath = os.path.abspath(os.path.expanduser("{0}".format(filename))) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 0a2a9d9..48c837b 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -51,38 +51,36 @@ class TGenVisualization(Visualization): self.page.close() def __extract_data_frame(self): - transfers = [] + streams = [] for (analyses, label) in self.datasets: for analysis in analyses: for client in analysis.get_nodes(): - tgen_transfers = analysis.get_tgen_transfers(client) - for transfer_id, transfer_data in tgen_transfers.items(): - transfer = {"transfer_id": transfer_id, "label": label, - "filesize_bytes": transfer_data["filesize_bytes"], - "error_code": None} - transfer["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" - if "elapsed_seconds" in transfer_data: - s = transfer_data["elapsed_seconds"] + tgen_streams = analysis.get_tgen_streams(client) + for stream_id, stream_data in tgen_streams.items(): + stream = {"stream_id": stream_id, "label": label, + "filesize_bytes": stream_data["stream_info"]["recvsize"]} + stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" + if "time_info" in stream_data: + s = stream_data["time_info"] if "payload_progress" in s: # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: - transfer["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) - if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]: - transfer["mbps"] = 4.194304 / (s["payload_progress"]["0.2"] - s["payload_progress"]["0.1"]) - if "first_byte" in s: - transfer["time_to_first_byte"] = s["first_byte"] - if "last_byte" in s: - transfer["time_to_last_byte"] = s["last_byte"] - if "error_code" in transfer_data and transfer_data["error_code"] != "NONE": - transfer["error_code"] = transfer_data["error_code"] - if "unix_ts_start" in transfer_data: - transfer["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) - transfers.append(transfer) - self.data = pd.DataFrame.from_records(transfers, index="transfer_id") + if stream_data["stream_info"]["recv_size"] == 5242880 and "0.2" in s["elapsed_seconds"]["payload_progress_recv"]: + stream["mbps"] = 4.194304 / (s["elapsed_seconds"]["payload_progress_recv"]["0.2"] - s["elapsed_seconds"]["payload_progress_recv"]["0.1"]) + + if "usecs-to-first-byte-recv" in s: + stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 + if "usecs-to-last-byte-recv" in s: + stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 + if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": + stream["error_code"] = stream_data["transport_info"]["error"] + if "unix_ts_start" in stream_data: + stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) + streams.append(stream) + self.data = pd.DataFrame.from_records(streams, index="stream_id") def __plot_firstbyte_ecdf(self): for server in self.data["server"].unique():

1 0

[onionperf/master] Updates README.md following the update to TGen 1.0.0
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 7282eea4f8349fb8e95a5e0c8c81f574abe8a30d Author: Ana Custura <ana(a)netstat.org.uk> Date: Sun Jul 12 15:45:29 2020 +0100 Updates README.md following the update to TGen 1.0.0 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 77384c6..032dd41 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ sudo apt install cmake libglib2.0-dev libigraph0-dev make cd ~/ git clone https://github.com/shadow/tgen.git cd tgen/ -git checkout -b v0.0.1 v0.0.1 mkdir build cd build/ cmake .. @@ -281,7 +280,7 @@ The PDF output file contains visualizations of the following metrics: The CSV output file contains the same data that is visualized in the PDF file. It contains the following columns: -- `transfer_id` is the identifier used in the TGen client logs which may be useful to look up more details about a specific measurement. +- `id` is the identifier used in the TGen client logs which may be useful to look up more details about a specific measurement. - `error_code` is an optional error code if a measurement did not succeed. - `filesize_bytes` is the requested file size in bytes. - `label` is the data set label as given in the `--data/-d` parameter to the `visualize` mode.

1 0

[onionperf/master] Renames stream_id to id
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit deff90a2b21b4be8e51036dd352d1f4c398836ea Author: Ana Custura <ana(a)netstat.org.uk> Date: Sun Jul 12 15:04:35 2020 +0100 Renames stream_id to id --- onionperf/visualization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 68ad751..f85c10f 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -58,7 +58,7 @@ class TGenVisualization(Visualization): for client in analysis.get_nodes(): tgen_streams = analysis.get_tgen_streams(client) for stream_id, stream_data in tgen_streams.items(): - stream = {"stream_id": stream_id, "label": label, + stream = {"id": stream_id, "label": label, "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" if "time_info" in stream_data: @@ -85,7 +85,7 @@ class TGenVisualization(Visualization): for client in analysis.get_nodes(): tgen_transfers = analysis.get_tgen_transfers(client) for transfer_id, transfer_data in tgen_transfers.items(): - stream = {"stream_id": transfer_id, "label": label, + stream = {"id": transfer_id, "label": label, "filesize_bytes": transfer_data["filesize_bytes"]} stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" if "elapsed_seconds" in transfer_data: @@ -110,7 +110,7 @@ class TGenVisualization(Visualization): stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) streams.append(stream) - self.data = pd.DataFrame.from_records(streams, index="stream_id") + self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self): for server in self.data["server"].unique():

1 0

[onionperf/master] Add JSON schema for analysis file format 3.0.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 7c1ae345effc8197e35ffffde86d318893d42285 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 17:43:08 2020 +0200 Add JSON schema for analysis file format 3.0. This schema can be used to validate an analysis file using version 3.0 by using the following commands: ``` pip3 install jsonschema unxz final_onionperf.analysis.json.xz jsonschema -i final_onionperf.analysis.json onionperf-3.0.json ``` Implements tpo/metrics/onionperf#40003. --- schema/onionperf-3.0.json | 546 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 546 insertions(+) diff --git a/schema/onionperf-3.0.json b/schema/onionperf-3.0.json new file mode 100644 index 0000000..f670c4c --- /dev/null +++ b/schema/onionperf-3.0.json @@ -0,0 +1,546 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://gitlab.torproject.org/tpo/metrics/onionperf/-/raw/master/schema/oni…", + "type": "object", + "title": "OnionPerf analysis JSON file format 3.0", + "required": [ + "data", + "type", + "version" + ], + "properties": { + "data": { + "type": "object", + "title": "Measurement data by source name", + "propertyNames": { + "pattern": "^[A-Za-z0-9-]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Measurement data from a single source", + "required": [ + "measurement_ip", + "tgen", + "tor" + ], + "properties": { + "measurement_ip": { + "type": "string", + "title": "Public IP address of the measuring host." + }, + "tgen": { + "type": "object", + "title": "Measurement data obtained from client-side TGen logs", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "object", + "title": "Measurement data, by TGen stream identifier", + "additionalProperties": { + "type": "object", + "title": "Information on a single measurement, obtained from a single [stream-success] or [stream-error] log message (except for elapsed_seconds)", + "required": [ + "byte_info", + "is_complete", + "is_error", + "is_success", + "stream_id", + "stream_info", + "time_info", + "transport_info", + "unix_ts_end", + "unix_ts_start" + ], + "properties": { + "byte_info": { + "type": "object", + "title": "Information on sent and received bytes", + "required": [ + "payload-bytes-recv", + "payload-bytes-send", + "payload-progress-recv", + "payload-progress-send", + "total-bytes-recv", + "total-bytes-send" + ], + "properties": { + "payload-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes received" + }, + "payload-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes sent" + }, + "payload-progress-recv": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of receiving payload in percent" + }, + "payload-progress-send": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of sending payload in percent" + }, + "total-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes received" + }, + "total-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes sent" + } + } + }, + "elapsed_seconds": { + "type": "object", + "title": "Elapsed seconds until a given number or fraction of payload bytes have been received or sent, obtained from [stream-status], [stream-success], and [stream-error] log messages, only included if the measurement was a success", + "properties": { + "payload_bytes_recv": { + "type": "object", + "title": "Number of received payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_bytes_send": { + "type": "object", + "title": "Number of sent payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_recv": { + "type": "object", + "title": "Fraction of received payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_send": { + "type": "object", + "title": "Fraction of sent payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + } + } + }, + "is_complete": { + "type": "boolean", + "title": "Whether the stream finished, no matter the error state, which is always true, or otherwise the measurement would not be included here" + }, + "is_error": { + "type": "boolean", + "title": "Whether an error occurred" + }, + "is_success": { + "type": "boolean", + "title": "Whether the measurement was a success" + }, + "stream_id": { + "type": "string", + "title": "Stream identifier" + }, + "stream_info": { + "type": "object", + "title": "Information about the TGen stream", + "required": [ + "error", + "id", + "name", + "peername", + "recvsize", + "recvstate", + "sendsize", + "sendstate", + "vertexid" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "id": { + "type": "string", + "title": "Stream numerical identifier, or 0 if the stream failed" + }, + "name": { + "type": "string", + "title": "Hostname of the TGen client" + }, + "peername": { + "type": "string", + "title": "Hostname of the TGen server" + }, + "recvsize": { + "type": "string", + "title": "Number of expected payload bytes in the response" + }, + "recvstate": { + "type": "string", + "title": "Last recorded receive state of the stream, one of RECV_{NONE,AUTHENTICATE,HEADER,MODEL,PAYLOAD,CHECKSUM,SUCCESS,ERROR}" + }, + "sendsize": { + "type": "string", + "title": "Number of expected payload bytes in the request" + }, + "sendstate": { + "type": "string", + "title": "Last recorded send state of the stream, one of SEND_{NONE,COMMAND,RESPONSE,PAYLOAD,CHECKSUM,FLUSH,SUCCESS,ERROR}" + }, + "vertexid": { + "type": "string", + "title": "Vertex identifier in the TGen model" + } + } + }, + "time_info": { + "type": "object", + "title": "Elapsed time until reaching given substeps in a measurement", + "required": [ + "created-ts", + "now-ts", + "usecs-to-checksum-recv", + "usecs-to-checksum-send", + "usecs-to-command", + "usecs-to-first-byte-recv", + "usecs-to-first-byte-send", + "usecs-to-last-byte-recv", + "usecs-to-last-byte-send", + "usecs-to-proxy-choice", + "usecs-to-proxy-init", + "usecs-to-proxy-request", + "usecs-to-proxy-response", + "usecs-to-response", + "usecs-to-socket-connect", + "usecs-to-socket-create" + ], + "properties": { + "created-ts": { + "type": "string", + "title": "Montonic system time when TGen created this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "now-ts": { + "type": "string", + "title": "Montonic system time when TGen computed elapsed microseconds for this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "usecs-to-checksum-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the checksum from the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-checksum-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the checksum to the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-command": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the command to the TGen server, or -1 if missing (step 7)" + }, + "usecs-to-first-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-first-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-last-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-last-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-proxy-choice": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS choice from the Tor client, or -1 if missing (step 4)" + }, + "usecs-to-proxy-init": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS initialization to the Tor client, or -1 if missing (step 3)" + }, + "usecs-to-proxy-request": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS request to the Tor client, or -1 if missing (step 5)" + }, + "usecs-to-proxy-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS response from the Tor client, or -1 if missing (step 6)" + }, + "usecs-to-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the command from the TGen server, or -1 if missing (step 8)" + }, + "usecs-to-socket-connect": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has connected to the Tor client's SOCKS port, or -1 if missing (step 2)" + }, + "usecs-to-socket-create": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has opened a TCP connection to the Tor client's SOCKS port, or -1 if missing (step 1)" + } + } + }, + "transport_info": { + "type": "object", + "title": "Information about the TGen transport", + "required": [ + "error", + "fd", + "local", + "proxy", + "remote", + "state" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "fd": { + "type": "string", + "title": "File descriptor" + }, + "local": { + "type": "string", + "title": "Local host name, IP address, and TCP port" + }, + "proxy": { + "type": "string", + "title": "Proxy host name, IP address, and TCP port" + }, + "remote": { + "type": "string", + "title": "Remote host name, IP address, and TCP port" + }, + "state": { + "type": "string", + "title": "Last recorded state of the transport, one of CONNECT,INIT,CHOICE,REQUEST,AUTH_{REQUEST,RESPONSE},RESPONSE_{STATUS,TYPE,IPV4,NAMELEN,NAME},SUCCESS_{OPEN,EOF},ERROR" + } + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the measurement, obtained from the log time of the [stream-success] or [stream-error] log message, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the measurement, obtained by subtracting the largest number of elapsed microseconds in time_info from unix_ts_end, given in seconds since the epoch" + } + } + } + } + } + }, + "tor": { + "type": "object", + "title": "Metadata obtained from client-side Tor controller logs", + "required": [ + "circuits", + "streams" + ], + "properties": { + "circuits": { + "type": "object", + "title": "Information about Tor circuits, by circuit identifier, obtained from CIRC and CIRC_MINOR events, for all circuits created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor circuit", + "required": [ + "circuit_id", + "elapsed_seconds", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "build_quantile": { + "type": "number", + "title": "Circuit build time quantile, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC LAUNCHED event" + }, + "build_timeout": { + "type": "integer", + "title": "Circuit build time in milliseconds, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC event with status LAUNCHED" + }, + "buildtime_seconds": { + "type": "number", + "title": "Build time in seconds, computed as time elapsed between CIRC LAUNCHED and CIRC BUILT events" + }, + "circuit_id": { + "type": "integer", + "title": "Circuit identifier, obtained from CIRC and CIRC_MINOR events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging CIRC and CIRC_MINOR events", + "items": { + "type": "array", + "title": "Elapsed seconds until reaching a given circuit status change", + "items": [ + { + "type": "string", + "title": "Circuit status change" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from CIRC FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from CIRC FAILED events" + }, + "path": { + "type": "array", + "title": "Path information", + "items": { + "type": "array", + "title": "Elapsed seconds until extending the circuit to a given relay", + "items": [ + { + "type": "string", + "pattern": "^\\$[0-9A-Z]{40}~[0-9a-zA-Z]{1,19}$", + "title": "Relay fingerprint and nickname" + }, + { + "type": "number", + "minimum": 0, + "title": "Elapsed seconds" + } + ] + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the circuit, obtained from the log time of the last CIRC CLOSED or CIRC FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the circuit, obtained from the log time of the CIRC LAUNCHED event, given in seconds since the epoch" + } + } + } + }, + "streams": { + "type": "object", + "title": "Information about Tor stream, by stream identifier, obtained from STREAM events, for all streams created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor stream", + "required": [ + "circuit_id", + "elapsed_seconds", + "stream_id", + "target", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "circuit_id": { + "title": "Circuit identifier, obtained from STREAM events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging STREAM events", + "items": { + "type": "array", + "items": [ + { + "type": "string", + "title": "Stream purpose and STREAM event status" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from STREAM FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from STREAM FAILED events" + }, + "source": { + "type": "string", + "title": "Stream source IP address and TCP port, obtained from STREAM NEW or STREAM NEWRESOLVE events" + }, + "stream_id": { + "type": "integer", + "title": "Stream identifier, unique at least for the lifetime of this stream" + }, + "target": { + "type": "string", + "title": "Stream target domain name and TCP port, obtained from STREAM events", + "examples": [ + "jzxfvaupigl7hkemf4jhfi2vrruvbb7ucyiwdolkkc2hf3xlm34f3qyd.onion:8080" + ] + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the stream, obtained from the log time of the last STREAM CLOSED or STREAM FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the stream, obtained from the log time of the first STREAM NEW or STREAM NEWRESOLVE event, given in seconds since the epoch" + } + } + } + } + } + } + } + } + }, + "type": { + "type": "string", + "title": "Document type", + "const": "onionperf" + }, + "version": { + "type": "string", + "title": "Document version", + "pattern": "^3\\.[0-9]+$" + } + } +}

1 0

[onionperf/master] Restore #40004 fix that got lost during rebase.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 120e425ebf78d36b3e1efd06de1e6cbe134a4a58 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:47:58 2020 +0200 Restore #40004 fix that got lost during rebase. --- onionperf/visualization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f85c10f..7cc36d1 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -59,7 +59,8 @@ class TGenVisualization(Visualization): tgen_streams = analysis.get_tgen_streams(client) for stream_id, stream_data in tgen_streams.items(): stream = {"id": stream_id, "label": label, - "filesize_bytes": int(stream_data["stream_info"]["recvsize"])} + "filesize_bytes": int(stream_data["stream_info"]["recvsize"]), + "error_code": None} stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" if "time_info" in stream_data: s = stream_data["time_info"] @@ -86,7 +87,8 @@ class TGenVisualization(Visualization): tgen_transfers = analysis.get_tgen_transfers(client) for transfer_id, transfer_data in tgen_transfers.items(): stream = {"id": transfer_id, "label": label, - "filesize_bytes": transfer_data["filesize_bytes"]} + "filesize_bytes": transfer_data["filesize_bytes"], + "error_code": None} stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" if "elapsed_seconds" in transfer_data: s = transfer_data["elapsed_seconds"]

1 0

[onionperf/master] Make a few whitespace fixes.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 2eceb8c7044a770059f4d1ad06241b60216d4b84 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:50:41 2020 +0200 Make a few whitespace fixes. --- onionperf/analysis.py | 4 ++-- onionperf/visualization.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 49d6dba..8506756 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -26,7 +26,7 @@ class OPAnalysis(Analysis): def __init__(self, nickname=None, ip_address=None): super().__init__(nickname, ip_address) - self.json_db = {'type' : 'onionperf', 'version' : '3.0', 'data' : {}} + self.json_db = {'type': 'onionperf', 'version': '3.0', 'data': {}} self.torctl_filepaths = [] def add_torctl_file(self, filepath): @@ -64,7 +64,7 @@ class OPAnalysis(Analysis): if self.measurement_ip is None: self.measurement_ip = "unknown" - self.json_db['data'].setdefault(self.nickname, {'measurement_ip' : self.measurement_ip}).setdefault(json_db_key, parser.get_data()) + self.json_db['data'].setdefault(self.nickname, {'measurement_ip': self.measurement_ip}).setdefault(json_db_key, parser.get_data()) self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") self.json_db['data'][self.nickname]["tgen"].pop("init_ts") self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 7cc36d1..52d08fc 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -68,7 +68,7 @@ class TGenVisualization(Visualization): stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 if "usecs-to-last-byte-recv" in s: stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 - if "elapsed_seconds" in stream_data: + if "elapsed_seconds" in stream_data: s = stream_data["elapsed_seconds"] # Explanation of the math below for computing Mbps: From filesize_bytes # and payload_progress fields we can compute the number of seconds that @@ -93,11 +93,11 @@ class TGenVisualization(Visualization): if "elapsed_seconds" in transfer_data: s = transfer_data["elapsed_seconds"] if "payload_progress" in s: - # Explanation of the math below for computing Mbps: From filesize_bytes - # and payload_progress fields we can compute the number of seconds that - # have elapsed between receiving bytes 524,288 and 1,048,576, which is a - # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. - # We want the reciprocal of that value with unit megabits per second. + # Explanation of the math below for computing Mbps: From filesize_bytes + # and payload_progress fields we can compute the number of seconds that + # have elapsed between receiving bytes 524,288 and 1,048,576, which is a + # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. + # We want the reciprocal of that value with unit megabits per second. if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]:

1 0

[onionperf/master] Update path to TGen 1.0.0 binary in README.md.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 5d5bff153dfd4a4b98d2bd3632cc4e813498ae0a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:45:01 2020 +0200 Update path to TGen 1.0.0 binary in README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 032dd41..e3df3bf 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ cmake .. make ``` -The TGen binary will be contained in `~/tgen/build/tgen`, which is also the path that needs to be passed to OnionPerf's `--tgen` parameter when doing measurements. +The TGen binary will be contained in `~/tgen/build/src/tgen`, which is also the path that needs to be passed to OnionPerf's `--tgen` parameter when doing measurements. ### OnionPerf

1 0