August 2020 - tor-commits - lists.torproject.org

[onionperf/master] Fix visualizations.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit b9dbc158666940bc1bb2c7ffccee2b73873c7368 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 23:18:29 2020 +0200 Fix visualizations. Turns out that stream_info/recvsize contains a string, not an int. --- onionperf/visualization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 52d08fc..80c0781 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -70,12 +70,12 @@ class TGenVisualization(Visualization): stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 if "elapsed_seconds" in stream_data: s = stream_data["elapsed_seconds"] - # Explanation of the math below for computing Mbps: From filesize_bytes - # and payload_progress fields we can compute the number of seconds that + # Explanation of the math below for computing Mbps: From stream_info/recvsize + # and payload_progress_recv fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if stream_data["stream_info"]["recvsize"] == 5242880 and "0.2" in s["payload_progress_recv"]: + if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]: stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": stream["error_code"] = stream_data["transport_info"]["error"]

1 0

[onionperf/master] Removes unused import and do_complete attribute from tor parser
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit c1cd60e05e23ef022076df9665a9d23c891561ca Author: Ana Custura <ana(a)netstat.org.uk> Date: Wed Jul 15 11:56:50 2020 +0100 Removes unused import and do_complete attribute from tor parser --- onionperf/analysis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 26b00b2..e269f6a 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -10,7 +10,7 @@ from abc import ABCMeta, abstractmethod # stem imports from stem import CircEvent, CircStatus, CircPurpose, StreamStatus -from stem.response.events import CircuitEvent, CircMinorEvent, StreamEvent, BandwidthEvent, BuildTimeoutSetEvent +from stem.response.events import CircuitEvent, CircMinorEvent, StreamEvent, BuildTimeoutSetEvent from stem.response import ControlMessage, convert # tgentools imports @@ -34,14 +34,14 @@ class OPAnalysis(Analysis): return self.date_filter = date_filter - tgen_parser = TGenParser(date_filter=self.date_filter) + super().analyze(do_complete=True) torctl_parser = TorCtlParser(date_filter=self.date_filter) - for (filepaths, parser, json_db_key) in [(self.tgen_filepaths, tgen_parser, 'tgen'), (self.torctl_filepaths, torctl_parser, 'tor')]: + for (filepaths, parser, json_db_key) in [(self.torctl_filepaths, torctl_parser, 'tor')]: if len(filepaths) > 0: for filepath in filepaths: logging.info("parsing log file at {0}".format(filepath)) - parser.parse(util.DataSource(filepath), do_complete=True) + parser.parse(util.DataSource(filepath)) if self.nickname is None: parsed_name = parser.get_name() @@ -128,7 +128,7 @@ class OPAnalysis(Analysis): class Parser(object, metaclass=ABCMeta): @abstractmethod - def parse(self, source, do_complete=True): + def parse(self, source): pass @abstractmethod def get_data(self): @@ -404,7 +404,7 @@ class TorCtlParser(Parser): return True - def parse(self, source, do_complete=True): + def parse(self, source): source.open(newline='\r\n') for line in source: # ignore line parsing errors

1 0

[onionperf/master] Retrieve error_code from the stream_info dict
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 89c8a8f826ee0dc460feb81f67e6bebfbbb9981f Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Jul 23 15:04:57 2020 +0100 Retrieve error_code from the stream_info dict --- onionperf/visualization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 5e7f92b..daa5c3d 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -88,8 +88,8 @@ class TGenVisualization(Visualization): s = stream_data["elapsed_seconds"] if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]: stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) - if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": - error_code = stream_data["transport_info"]["error"] + if "error" in stream_data["stream_info"] and stream_data["stream_info"]["error"] != "NONE": + error_code = stream_data["stream_info"]["error"] if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2: source_port = stream_data["transport_info"]["local"].split(":")[2] if "unix_ts_end" in stream_data:

1 0

[onionperf/master] Refine error codes into TOR or TGEN errors.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 4533b39591cc0a6df124e017366ea71343c842bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 17 22:24:43 2020 +0200 Refine error codes into TOR or TGEN errors. With this change we include more detailed error codes in visualization output. In order to do so we map TGen transfers/streams to TorCtl STREAM event details based on source ports and unix_ts_end timestamps. This code reuses some concepts used in metrics-lib. Implements tpo/metrics/onionperf#34218. --- onionperf/analysis.py | 6 +++ onionperf/visualization.py | 107 +++++++++++++++++++++++++++------------------ 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index e269f6a..8fcc0a6 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -97,6 +97,12 @@ class OPAnalysis(Analysis): except: return None + def get_tor_streams(self, node): + try: + return self.json_db['data'][node]['tor']['streams'] + except: + return None + @classmethod def load(cls, filename="onionperf.analysis.json.xz", input_prefix=os.getcwd()): filepath = os.path.abspath(os.path.expanduser("{0}".format(filename))) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 80c0781..5e7f92b 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -54,50 +54,57 @@ class TGenVisualization(Visualization): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: - if analysis.json_db['version'] >= '3': - for client in analysis.get_nodes(): - tgen_streams = analysis.get_tgen_streams(client) - for stream_id, stream_data in tgen_streams.items(): - stream = {"id": stream_id, "label": label, - "filesize_bytes": int(stream_data["stream_info"]["recvsize"]), - "error_code": None} - stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" - if "time_info" in stream_data: - s = stream_data["time_info"] - if "usecs-to-first-byte-recv" in s: - stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 - if "usecs-to-last-byte-recv" in s: - stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 - if "elapsed_seconds" in stream_data: - s = stream_data["elapsed_seconds"] - # Explanation of the math below for computing Mbps: From stream_info/recvsize - # and payload_progress_recv fields we can compute the number of seconds that - # have elapsed between receiving bytes 524,288 and 1,048,576, which is a - # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. - # We want the reciprocal of that value with unit megabits per second. - if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]: - stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) - if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": - stream["error_code"] = stream_data["transport_info"]["error"] - if "unix_ts_start" in stream_data: - stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) - streams.append(stream) - else: - for client in analysis.get_nodes(): - tgen_transfers = analysis.get_tgen_transfers(client) - for transfer_id, transfer_data in tgen_transfers.items(): + for client in analysis.get_nodes(): + tor_streams_by_source_port = {} + tor_streams = analysis.get_tor_streams(client) + for tor_stream in tor_streams.values(): + if "source" in tor_stream and ":" in tor_stream["source"]: + source_port = tor_stream["source"].split(":")[1] + tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream) + tgen_streams = analysis.get_tgen_streams(client) + tgen_transfers = analysis.get_tgen_transfers(client) + while tgen_streams or tgen_transfers: + error_code = None + source_port = None + unix_ts_end = None + # Explanation of the math below for computing Mbps: For 1 MiB and 5 MiB + # downloads we can extract the number of seconds that have elapsed between + # receiving bytes 524,288 and 1,048,576, which is a total amount of 524,288 + # bytes or 4,194,304 bits or 4.194304 megabits. We want the reciprocal of + # that value with unit megabits per second. + if tgen_streams: + stream_id, stream_data = tgen_streams.popitem() + stream = {"id": stream_id, "label": label, + "filesize_bytes": int(stream_data["stream_info"]["recvsize"]), + "error_code": None} + stream["server"] = "onion" if ".onion:" in stream_data["transport_info"]["remote"] else "public" + if "time_info" in stream_data: + s = stream_data["time_info"] + if "usecs-to-first-byte-recv" in s: + stream["time_to_first_byte"] = float(s["usecs-to-first-byte-recv"])/1000000 + if "usecs-to-last-byte-recv" in s: + stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 + if "elapsed_seconds" in stream_data: + s = stream_data["elapsed_seconds"] + if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]: + stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) + if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": + error_code = stream_data["transport_info"]["error"] + if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2: + source_port = stream_data["transport_info"]["local"].split(":")[2] + if "unix_ts_end" in stream_data: + unix_ts_end = stream_data["unix_ts_end"] + if "unix_ts_start" in stream_data: + stream["start"] = datetime.datetime.utcfromtimestamp(stream_data["unix_ts_start"]) + elif tgen_transfers: + transfer_id, transfer_data = tgen_transfers.popitem() stream = {"id": transfer_id, "label": label, - "filesize_bytes": transfer_data["filesize_bytes"], - "error_code": None} + "filesize_bytes": transfer_data["filesize_bytes"], + "error_code": None} stream["server"] = "onion" if ".onion:" in transfer_data["endpoint_remote"] else "public" if "elapsed_seconds" in transfer_data: s = transfer_data["elapsed_seconds"] if "payload_progress" in s: - # Explanation of the math below for computing Mbps: From filesize_bytes - # and payload_progress fields we can compute the number of seconds that - # have elapsed between receiving bytes 524,288 and 1,048,576, which is a - # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. - # We want the reciprocal of that value with unit megabits per second. if transfer_data["filesize_bytes"] == 1048576 and "1.0" in s["payload_progress"]: stream["mbps"] = 4.194304 / (s["payload_progress"]["1.0"] - s["payload_progress"]["0.5"]) if transfer_data["filesize_bytes"] == 5242880 and "0.2" in s["payload_progress"]: @@ -107,11 +114,27 @@ class TGenVisualization(Visualization): if "last_byte" in s: stream["time_to_last_byte"] = s["last_byte"] if "error_code" in transfer_data and transfer_data["error_code"] != "NONE": - stream["error_code"] = transfer_data["error_code"] + error_code = transfer_data["error_code"] + if "endpoint_local" in transfer_data and len(transfer_data["endpoint_local"].split(":")) > 2: + source_port = transfer_data["endpoint_local"].split(":")[2] + if "unix_ts_end" in transfer_data: + unix_ts_end = transfer_data["unix_ts_end"] if "unix_ts_start" in transfer_data: stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) - streams.append(stream) - + if error_code: + if error_code == "PROXY": + error_code_parts = ["TOR"] + else: + error_code_parts = ["TGEN", error_code] + if source_port and unix_ts_end: + for tor_stream in tor_streams_by_source_port[source_port]: + if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: + if "failure_reason_local" in tor_stream: + error_code_parts.append(tor_stream["failure_reason_local"]) + if "failure_reason_remote" in tor_stream: + error_code_parts.append(tor_stream["failure_reason_remote"]) + stream["error_code"] = "/".join(error_code_parts) + streams.append(stream) self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/master] Remove README_JSON.md and point to JSON schema.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit a817e5ae5ece1d753f3688ee579e87e93951a779 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 16 22:14:28 2020 +0200 Remove README_JSON.md and point to JSON schema. --- README.md | 2 +- README_JSON.md | 213 --------------------------------------------------------- 2 files changed, 1 insertion(+), 214 deletions(-) diff --git a/README.md b/README.md index ecd7c0c..b9bf7e6 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,7 @@ For example, the following command analyzes current log files of a running (or s onionperf analyze --tgen ~/onionperf-data/tgen-client/onionperf.tgen.log --torctl ~/onionperf-data/tor-client/onionperf.torctl.log ``` -The output analysis file is written to `onionperf.analysis.json.xz` in the current working directory. The file format is described in more detail in `README_JSON.md`. +The output analysis file is written to `onionperf.analysis.json.xz` in the current working directory. The file format is described in more detail in `schema/onionperf-3.0.json`. The same analysis files are written automatically as part of ongoing measurements once per day at UTC midnight and can be found in `onionperf-data/htdocs/`. diff --git a/README_JSON.md b/README_JSON.md deleted file mode 100644 index f0ad281..0000000 --- a/README_JSON.md +++ /dev/null @@ -1,213 +0,0 @@ -# DB Structure - -This document describes the structure of the json database file that gets exported in `analysis` mode and gets placed in the www docroot when running in `measure` mode. - -The structure is given here with variable keys marked as such. - - { - "data": { # generic keyword - "phantomtrain": { # nickname of the OnionPerf client, hostname if not set - "measurement_ip" : "192.168.1.1", # public-facing IP address of the machine used for the measurements - "tgen": { # to indicate data from TGen - "transfers": { # the records for transfers TGen attempted - "transfer1m:1": { # the id of a single transfer - "elapsed_seconds": { # timing for various steps in transfer, in seconds - "checksum": 0.0, # step 12 if using a proxy, else step 8 (initial GET/PUT) - "command": 0.319006, # step 7 if using a proxy, else step 3 (initial GET/PUT) - "first_byte": 0.0, # step 9 if using a proxy, else step 5 (initial GET/PUT) - "last_byte": 0.0, # step 11 if using a proxy, else step 7 (initial GET/PUT) - "payload_bytes": { # similar to payload_progress below - "10240": 0.0, # number of payload bytes completed : seconds to complete it - "20480": 0.0, - "51200": 0.0, - "102400": 0.0, - "204800": 0.0, - "512000": 0.0, - "1048576": 0.0, - "2097152": 0.0, - "5242880": 0.0 - }, - "payload_progress": { # step 10 if using a proxy, else step 6 (initial GET/PUT) - "0.0": 0.0, # percent of payload completed : seconds to complete it - "0.1": 0.0, - "0.2": 0.0, - "0.3": 0.0, - "0.4": 0.0, - "0.5": 0.0, - "0.6": 0.0, - "0.7": 0.0, - "0.8": 0.0, - "0.9": 0.0, - "1.0": 0.0 - }, - "proxy_choice": 0.000233, # step 4 if using a proxy, else absent - "proxy_init": 0.000151, # step 3 if using a proxy, else absent - "proxy_request": 0.010959, # step 5 if using a proxy, else absent - "proxy_response": 0.318873, # step 6 if using a proxy, else absent - "response": 0.0, # step 8 if using a proxy, else step 4 (initial GET/PUT) - "socket_connect": 0.000115, # step 2 - "socket_create": 2e-06 # step 1 - }, - "endpoint_local": "localhost:127.0.0.1:45416", # tgen client socket name:ip:port - "endpoint_proxy": "localhost:127.0.0.1:27942", # proxy socket name:ip:port, if present - "endpoint_remote": "server1.peach-hosting.com:216.17.99.183:6666", # tgen server hostname:ip:port - "error_code": "READ", # 'NONE' or a code to indicate the type of error - "filesize_bytes": 1048576, # size of the transfer payload - "hostname_local": "puntaburros.amerinoc.com", # client machine hostname - "hostname_remote": "(null)", # server machine hostname - "is_commander": true, # true if client (initiated the transfer), else false - "is_complete": true, # if the transfer finished, no matter the error state - "is_error": true, # if there was an error in the transfer - "is_success": false, # if the transfer completed and checksum passed - "method": "GET", # transfer method (GET,PUT) - "payload_bytes_status": 0, # cumulative number of payload bytes received - "total_bytes_read": 0, # total bytes read from the socket - "total_bytes_write": 50, # total written to the socket - "transfer_id": "transfer1m:1", # the id of this transfer, unique to this run of OnionPerf - "unix_ts_end": 1456699868.006196, # initial start time of the transfer - "unix_ts_start": 1456699868.006196 # final end time of the transfer - }, - }, - "transfers_summary": { # summary stats of all transfers in the 'transfers' section - "errors": { - "PROXY": { # PROXY type errors - "1456654221": [ # the second at which the error occurred - 51200 # transfer filesizes that had errors, one entry for each error during this second - ], - }, - "READ": { # READ type errors - "1456618782": [ # second at which the error occurred - 51200 # transfer filesize, one for each error at this time - ], - }, - "time_to_first_byte": { # time to receive the first byte of the payload - "51200": { # file size - "1456707932": [ # the second at which the transfer completed - 0.36213199999999995 # time to first byte, in seconds - ], - }, - }, - "time_to_last_byte": { # time to receive the last byte of the payload - "51200": { # file size - "1456707932": [ # the second at which the transfer completed - 0.6602399999999999 # time to last byte, in seconds - ], - } - } - }, - }, - "tor": { # indicates data from Tor - "bandwidth_summary": { # from Tor's BW controller event - "bytes_read": { - "1456617599": 0, # unix time in seconds : number of bytes - }, - "bytes_written": { - "1456617599": 0, # unix time in seconds : number of bytes - } - }, - "streams": { # info about each stream - "23": { # stream ID - "circuit_id": "4", # circuit on which this stream was attached - "elapsed_seconds": [ # time in seconds to reach various points, from STREAM Tor events - [ - "USER:NEW", # stream purpose : stream status (from Tor) - 0.0 - ], - [ - "USER:SENTCONNECT", - 0.0 - ], - [ - "USER:REMAP", - 0.31999993324279785 - ], - [ - "USER:SUCCEEDED", - 0.31999993324279785 - ], - [ - "USER:CLOSED", - 0.9800000190734863 - ] - ], - "source": "127.0.0.1:48786", # ip:port of client that created stream - "stream_id": 23, # id of the stream used by Tor - "target": "216.17.99.183:6666", # ip:port of the remote end of the stream - "unix_ts_end": 1456707932.67, # unix time in seconds that the stream started - "unix_ts_start": 1456707931.69 # unix time in seconds that the stream ended - }, - }, - "streams_summary": { # summary stats about all streams - "lifetimes": { # time streams were alive, in seconds - "DIR_FETCH": [ # streams of type DIR_FETCH, one entry for each - 3.950000047683716, - ], - "USER": [ # streams of type USER, one entry for each - 114.72000002861023, - ] - } - }, - "circuits": { # info about each circuit - "10": { # circuit ID - "build_quantile": 0.8, - "build_timeout": 1500, - "buildtime_seconds": 1.0900001525878906, - "circuit_id": 10, - "elapsed_seconds": [ # time in seconds to reach various points, from CIRC Tor events - [ - "GENERAL:LAUNCHED", # circuit purpose : circuit status (from Tor) - 0.0 - ], - [ - "GENERAL:EXTENDED", - 0.75 - ], - [ - "GENERAL:EXTENDED", - 0.9100000858306885 - ], - [ - "GENERAL:EXTENDED", - 1.0900001525878906 - ], - [ - "GENERAL:BUILT", - 1.0900001525878906 - ], - [ - "GENERAL:CLOSED", - 3601.0 - ] - ], - "path": [ # fingerprint~nickname for each relay in the path, seconds to extend to that relay - [ - "$BB60F5BA113A0B8B44B7B37DE3567FE561E92F78~Casper04", - 0.75 - ], - [ - "$2FD0BA57A34DC2792AF470398F72F37F9E51DC2D~serotonin", - 0.9100000858306885 - ], - [ - "$DE7DE889E0D1A5F397AE35642060B84999581203~DigiGesTor2e3", - 1.0900001525878906 - ] - ], - "unix_ts_end": 1456711533.59, # unix time in seconds that the circuit started - "unix_ts_start": 1456707932.59 # unix time in seconds that the circuit ended - }, - }, - "circuits_summary": { # summary stats about all circuits - "buildtimes": [ # time to build circuits in seconds, one entry for each circuit - 1.2100000381469727, - ], - "lifetimes": [ # lifetime of circuits in seconds, one entry for each circuit - 60.99000000953674, - ], - } - } - } - }, - "type": "onionperf", # document type - "version": "2.0" # document version - }

1 0

[onionperf/master] Do some repository housekeeping.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 7576ce3ede211722db59bde3ccbc7e79ac4ac60c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 23 20:18:03 2020 +0200 Do some repository housekeeping. Fixes #40006. --- .gitignore | 4 ++-- .gitlab-ci.yml | 23 ----------------------- .gitmodules | 0 README.md | 6 ++++++ Vagrantfile | 35 ----------------------------------- debian/changelog | 5 ----- debian/compat | 1 - debian/control | 40 ---------------------------------------- debian/install | 1 - debian/rules | 15 --------------- debian/source/format | 1 - onionperf/__init__.py | 1 + onionperf/analysis.py | 1 + onionperf/measurement.py | 1 + onionperf/model.py | 1 + onionperf/monitor.py | 1 + onionperf/onionperf | 1 + onionperf/util.py | 1 + onionperf/visualization.py | 1 + run_tests.sh | 3 --- 20 files changed, 16 insertions(+), 126 deletions(-) diff --git a/.gitignore b/.gitignore index 5e79b0e..e3e7b28 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ -/.onionperf/* onionperf-data +onionperf-private venv *.json.xz *.pdf +*.csv *.pyc -onionperf/docs/_build .coverage diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index aed7769..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,23 +0,0 @@ -variables: - GIT_STRATEGY: clone - -stages: - - test - -test: - stage: test - image: debian:buster - coverage: '/TOTAL.+ ([0-9]{1,3}%)/' - script: - - apt -y update - - apt -y install git cmake make build-essential gcc libigraph0-dev libglib2.0-dev python-dev libxml2-dev python-lxml python-networkx python-scipy python-matplotlib python-numpy libevent-dev libssl-dev python-stem tor python-nose python-cov-core - - git clone https://github.com/shadow/tgen.git - - mkdir -p tgen/build - - pushd tgen/build - - cmake .. - - make - - ln -s `pwd`/tgen /usr/bin/ - - popd - - python setup.py build - - python setup.py install - - ./run_tests.sh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index b9bf7e6..ad53a9e 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,12 @@ deactivate However, in order to perform measurements or analyses, the virtual environment needs to be activated first. This will ensure all the paths are found. +If needed, unit tests are run with the following command: + +```shell +cd ~/onionperf/ +python3 -m nose --with-coverage --cover-package=onionperf +``` ## Measurement diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 1fdd10f..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,35 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -$setup_onionperf = <<SCRIPT -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt install -y build-essential autoconf cmake libglib2.0-dev libigraph0-dev libevent-dev libssl-dev python3 git python3-stem python3-lxml python3-networkx python3-matplotlib python3-numpy python3-scipy -cd ~ -git clone https://git.torproject.org/tor.git -cd tor -./autogen.sh -./configure --disable-asciidoc -make -mv src/app/tor /usr/local/bin/ -cd ~ -git clone https://github.com/shadow/shadow.git -cd shadow/src/plugin/shadow-plugin-tgen -mkdir build -cd build -cmake .. -DSKIP_SHADOW=ON -DCMAKE_MODULE_PATH=`pwd`/../../../../cmake/ -make -mv tgen /usr/local/bin/ -cd /vagrant -python3 setup.py build -python3 setup.py install -SCRIPT - -Vagrant.configure("2") do |config| - config.vm.box = "debian/stretch64" - - config.vm.define "oniondev" do |oniondev| - oniondev.vm.provision :shell, :inline => $setup_onionperf - end - -end diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 7671854..0000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -onionperf (2019052301) stretch-backports; urgency=medium - - * Initial release. - - -- Iain R. Learmonth <irl(a)debian.org> Thu, 23 May 2019 12:42:20 +0100 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index b4de394..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -11 diff --git a/debian/control b/debian/control deleted file mode 100644 index c8ffa68..0000000 --- a/debian/control +++ /dev/null @@ -1,40 +0,0 @@ -Source: onionperf -Maintainer: Tor Metrics <metrics-team(a)lists.torproject.org> -Section: net -Priority: optional -Build-Depends: debhelper (>= 11), - dh-python, - cmake, - make, - build-essential, - gcc, - libigraph0-dev, - libglib2.0-dev, - python, - python-ipaddress, - python-lxml, - python-matplotlib, - python-netifaces, - python-networkx, - python-nose, - python-numpy, - python-scipy, - python-stem (>=1.7) -Standards-Version: 4.1.3 - -Package: onionperf -Architecture: any -Depends: ${python:Depends}, - ${shlibs:Depends}, - ${misc:Depends}, - python-ipaddress, - python-lxml, - python-matplotlib, - python-netifaces, - python-networkx, - python-nose, - python-numpy, - python-scipy, - python-stem (>=1.7) -Description: Tor Performance Measurement Tool - Tor Performance Measurement Tool diff --git a/debian/install b/debian/install deleted file mode 100644 index 6d40677..0000000 --- a/debian/install +++ /dev/null @@ -1 +0,0 @@ -tgen/build/tgen usr/bin diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 64d85ba..0000000 --- a/debian/rules +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/make -f - -%: - dh $@ --with python2 --buildsystem=pybuild - -override_dh_auto_test: - ./run_tests.sh - -override_dh_auto_build: - ( cd tgen; \ - mkdir build; \ - cd build; \ - cmake ..; \ - make ) - dh_auto_build diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/onionperf/__init__.py b/onionperf/__init__.py index 79e37c3..f323b49 100644 --- a/onionperf/__init__.py +++ b/onionperf/__init__.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/analysis.py b/onionperf/analysis.py index e269f6a..6325764 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 823f37a..af1fa0d 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/model.py b/onionperf/model.py index f06f8e0..cb45f51 100644 --- a/onionperf/model.py +++ b/onionperf/model.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/monitor.py b/onionperf/monitor.py index 5387bff..9bf9691 100644 --- a/onionperf/monitor.py +++ b/onionperf/monitor.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/onionperf b/onionperf/onionperf index 36520ad..e8024ce 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -3,6 +3,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/util.py b/onionperf/util.py index cb9f065..4eee911 100644 --- a/onionperf/util.py +++ b/onionperf/util.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 80c0781..f88c984 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -1,6 +1,7 @@ ''' OnionPerf Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project See LICENSE for licensing information ''' diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 7a30e61..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -PYTHONPATH=. python3 -m nose --with-coverage --cover-package=onionperf

1 0

[onionperf/master] Rotate error code on y axis tick labels.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 30aa6dcb52d31403ab366599b13d7fa4db3adb80 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 23 21:36:10 2020 +0200 Rotate error code on y axis tick labels. Still part of #34218. --- onionperf/visualization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index daa5c3d..3abcb3f 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -292,6 +292,7 @@ class TGenVisualization(Visualization): g.set(title=title, xlabel=xlabel, ylabel=ylabel, xlim=(xmin - 0.03 * (xmax - xmin), xmax + 0.03 * (xmax - xmin))) plt.xticks(rotation=10) + plt.yticks(rotation=80) sns.despine() self.page.savefig() plt.close()

1 0

[onionperf/master] Add CHANGELOG.md.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit a671736fa1e7328c08e2a1644be280c3c045d29a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 23 19:44:48 2020 +0200 Add CHANGELOG.md. --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..54b632b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,67 @@ +# Changes in version 0.6 - 2020-??-?? + + - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and + update analysis results file version to 3.0. Implements #33974. + - Remove summaries from analysis results files, and remove the + `onionperf analyze -s/--do-simple-parse` switch. Implements #40005. + - Add JSON schema for analysis results file format 3.0. Implements + #40003. + +# Changes in version 0.5 - 2020-07-02 + + - Add new graph showing the cumulative distribution function of + throughput in Mbps. Implements #33257. + - Improve `README.md` to make it more useful to developers and + researchers. Implements #40001. + - Always include the `error_code` column in visualization CSV output, + regardless of whether data contains measurements with an error code + or not. Fixes #40004. + - Write generated torrc files to disk for debugging purposes. + Implements #40002. + +# Changes in version 0.4 - 2020-06-16 + + - Include all measurements when analyzing log files at midnight as + part of `onionperf measure`, not just the ones from the day before. + Also add `onionperf analyze -x/--date-prefix` switch to prepend a + given date string to an analysis results file. Fixes #29369. + - Add `size`, `last_modified`, and `sha256` fields to index.xml. + Implements #29365. + - Add support for single onion services using the switch `onionperf + measure -s/--single-onion`. Implements #29368. + - Remove unused `onionperf measure --traffic-model` switch. + Implements #29370. + - Make `onionperf measure -o/--onion-only` and `onionperf measure + -i/--inet-only` switches mutually exclusive. Fixes #34316. + - Accept one or more paths to analysis results files or directories + of such files per dataset in `onionperf visualize -d/--data` to + include all contained measurements in a dataset. Implements #34191. + +# Changes in version 0.3 - 2020-05-30 + + - Automatically compress logs when rotating them. Fixes #33396. + - Update to Python 3. Implements #29367. + - Integrate reprocessing mode into analysis mode. Implements #34142. + - Record download times of smaller file sizes from partial completion + times. Implements #26673. + - Stop generating .tpf files. Implements #34141. + - Update analysis results file version to 2.0. Implements #34224. + - Export visualized data to a CSV file. Implements #33258. + - Remove version 2 onion service support. Implements #33434. + - Reduce timeout and stallout values. Implements #34024. + - Remove 50 KiB and 1 MiB downloads. Implements #34023. + - Remove existing Tor control log visualizations. Implements #34214. + - Update to Networkx version 2.4. Fixes #34298. + - Update time to first/last byte definitions to include the time + between starting a measurement and receiving the first/last byte. + Implements #34215. + - Update `requirements.txt` to actual requirements, and switch from + distutils to setuptools. Fixes #30586. + - Split visualizations into public and onion service measurements. + Fixes #34216. + +# Changes from before 2020-04 + + - Changes made before 2020-04 are not listed here. See `git log` for + details. +

1 0

[onionperf/master] Fix unit tests.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 899dbf2518a191167d779063a85c6c0285a68b7b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 23 21:10:22 2020 +0200 Fix unit tests. Latest TGenTools contain a fix for correctly computing the start time of failed streams. We have to update our unit tests. Related to #30362. --- onionperf/tests/test_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/tests/test_analysis.py b/onionperf/tests/test_analysis.py index 82e2043..6a8280b 100644 --- a/onionperf/tests/test_analysis.py +++ b/onionperf/tests/test_analysis.py @@ -48,7 +48,7 @@ def test_stream_complete_event_init(): assert_equals(complete.time_info['usecs-to-proxy-choice'], '348') assert_equals(complete.time_info['usecs-to-socket-connect'], '210') assert_equals(complete.time_info['usecs-to-socket-create'], '11') - assert_equals(complete.unix_ts_start, 1555940480.6472511) + assert_equals(complete.unix_ts_start, 1555940359.2286081) def test_stream_error_event(): @@ -146,7 +146,7 @@ def test_stream_object_end_to_end(): 'now-ts': '5948446579043' }, 'stream_id': '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', - 'unix_ts_start': 1555940480.6472511 + 'unix_ts_start': 1555940359.2286081 }) def test_parsing_parse_error():

1 0

[onionperf/master] Merge branch 'task-34218' into develop
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit dd42ef1d5193bf96a59ee336ff6aff8f53bfd979 Merge: 899dbf2 30aa6dc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 23 21:37:11 2020 +0200 Merge branch 'task-34218' into develop onionperf/analysis.py | 6 +++ onionperf/visualization.py | 108 +++++++++++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 42 deletions(-)

1 0