September 2020 - tor-commits - lists.torproject.org

[onionperf/develop] Add path validation to filter interface
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit 1a81b3dff756cf197d661f931485443d1454c7c4 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Aug 21 01:38:01 2020 +0100 Add path validation to filter interface --- onionperf/onionperf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onionperf/onionperf b/onionperf/onionperf index b1e7bd3..96c6869 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -439,6 +439,9 @@ def analyze(args): def filter(args): from onionperf.filtering import Filtering + p = os.path.abspath(os.path.expanduser(args.input)) + if not os.path.exists(p): + raise argparse.ArgumentTypeError("path '%s' does not exist" % args.input) filtering = Filtering() filtering.read_input(args.input) if args.include_fingerprints is not None:

1 0

[onionperf/develop] Fix matching of tor_circuit ids
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit 26ff79a3e57f1dfb0c961cceb167534bb39d17f3 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Aug 21 02:14:32 2020 +0100 Fix matching of tor_circuit ids --- onionperf/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 435a1bc..204b161 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -73,7 +73,7 @@ class Filtering(object): for tor_stream in tor_streams_by_source_port[source_port]: if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: circuit_id = tor_stream["circuit_id"] - if circuit_id and circuit_id in tor_circuits: + if circuit_id and str(circuit_id) in tor_circuits: tor_circuit = tor_circuits[circuit_id] path = tor_circuit["path"] keep = True

1 0

[onionperf/develop] Add new filter mode to filter analysis results.
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit dcab214322356db9d975673d19146ff46ec86b4f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 18 21:05:45 2020 +0200 Add new filter mode to filter analysis results. Implements tpo/metrics/onionperf#33260. --- CHANGELOG.md | 6 +++ onionperf/analysis.py | 11 ++++++ onionperf/filtering.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ onionperf/onionperf | 51 +++++++++++++++++++++++++ 4 files changed, 168 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a86ee..c57695e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 0.7 - 2020-??-?? + + - Add a new `onionperf filter` mode that takes an OnionPerf analysis + results file as input, applies filters, and produces a new + OnionPerf analysis results file as output. + # Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and diff --git a/onionperf/analysis.py b/onionperf/analysis.py index ea07d0e..b2f483f 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -62,6 +62,11 @@ class OPAnalysis(Analysis): self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") self.did_analysis = True + def set_tgen_transfers(self, node, tgen_transfers): + self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers + + def set_tgen_streams(self, node, tgen_streams): + self.json_db['data'][node]['tgen']['streams'] = tgen_streams def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None): if filename is None: @@ -98,6 +103,12 @@ class OPAnalysis(Analysis): except: return None + def get_tor_circuits(self, node): + try: + return self.json_db['data'][node]['tor']['circuits'] + except: + return None + def get_tor_streams(self, node): try: return self.json_db['data'][node]['tor']['streams'] diff --git a/onionperf/filtering.py b/onionperf/filtering.py new file mode 100644 index 0000000..435a1bc --- /dev/null +++ b/onionperf/filtering.py @@ -0,0 +1,100 @@ +''' + OnionPerf + Authored by Rob Jansen, 2015 + Copyright 2015-2020 The Tor Project + See LICENSE for licensing information +''' + +import re +from onionperf.analysis import OPAnalysis + +class Filtering(object): + + def __init__(self): + self.fingerprints_to_include = None + self.fingerprints_to_exclude = None + self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})") + + def read_input(self, path): + self.analysis = OPAnalysis.load(filename=path) + + def include_fingerprints(self, path): + self.fingerprints_to_include = [] + with open(path, 'rt') as f: + for line in f: + fingerprint_match = self.fingerprint_pattern.match(line) + if fingerprint_match: + fingerprint = fingerprint_match.group(1).upper() + self.fingerprints_to_include.append(fingerprint) + + def exclude_fingerprints(self, path): + self.exclude_fingerprints = [] + with open(path, 'rt') as f: + for line in f: + fingerprint_match = self.fingerprint_pattern.match(line) + if fingerprint_match: + fingerprint = fingerprint_match.group(1).upper() + self.exclude_fingerprints.append(fingerprint) + + def apply_filters(self): + if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: + return + for source in self.analysis.get_nodes(): + tor_streams_by_source_port = {} + tor_streams = self.analysis.get_tor_streams(source) + for tor_stream in tor_streams.values(): + if "source" in tor_stream and ":" in tor_stream["source"]: + source_port = tor_stream["source"].split(":")[1] + tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream) + tor_circuits = self.analysis.get_tor_circuits(source) + tgen_streams = self.analysis.get_tgen_streams(source) + tgen_transfers = self.analysis.get_tgen_transfers(source) + retained_tgen_streams = {} + retained_tgen_transfers = {} + while tgen_streams or tgen_transfers: + stream_id = None + transfer_id = None + source_port = None + unix_ts_end = None + keep = False + if tgen_streams: + stream_id, stream_data = tgen_streams.popitem() + if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2: + source_port = stream_data["transport_info"]["local"].split(":")[2] + if "unix_ts_end" in stream_data: + unix_ts_end = stream_data["unix_ts_end"] + elif tgen_transfers: + transfer_id, transfer_data = tgen_transfers.popitem() + if "endpoint_local" in transfer_data and len(transfer_data["endpoint_local"].split(":")) > 2: + source_port = transfer_data["endpoint_local"].split(":")[2] + if "unix_ts_end" in transfer_data: + unix_ts_end = transfer_data["unix_ts_end"] + if source_port and unix_ts_end: + for tor_stream in tor_streams_by_source_port[source_port]: + if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: + circuit_id = tor_stream["circuit_id"] + if circuit_id and circuit_id in tor_circuits: + tor_circuit = tor_circuits[circuit_id] + path = tor_circuit["path"] + keep = True + for long_name, _ in path: + fingerprint_match = self.fingerprint_pattern.match(long_name) + if fingerprint_match: + fingerprint = fingerprint_match.group(1).upper() + if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include: + keep = False + break + if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude: + keep = False + break + if keep: + if stream_id: + retained_tgen_streams[stream_id] = stream_data + if transfer_id: + retained_tgen_transfers[transfer_id] = transfer_data + self.analysis.set_tgen_streams(source, retained_tgen_streams) + self.analysis.set_tgen_transfers(source, retained_tgen_transfers) + + def write_output(self, path): + self.analysis.save(filename=path) + diff --git a/onionperf/onionperf b/onionperf/onionperf index e8024ce..b1e7bd3 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -74,6 +74,15 @@ Stats files in the default Torperf format can also be exported. HELP_ANALYZE = """ Analyze Tor and TGen output """ + +DESC_FILTER = """ +Takes an OnionPerf analysis results file as input, applies filters, +and produces a new OnionPerf analysis results file as output. +""" +HELP_FILTER = """ +Filter OnionPerf analysis results +""" + DESC_VISUALIZE = """ Loads an OnionPerf json file, e.g., one produced with the `analyze` subcommand, and plots various interesting performance metrics to PDF files. @@ -280,6 +289,36 @@ files generated by this script will be written""", action="store", dest="date_prefix", default=None) + # filter + filter_parser = sub_parser.add_parser('filter', description=DESC_FILTER, help=HELP_FILTER, + formatter_class=my_formatter_class) + filter_parser.set_defaults(func=filter, formatter_class=my_formatter_class) + + filter_parser.add_argument('-i', '--input', + help="""read the OnionPerf analysis results at PATH as input""", + metavar="PATH", required="True", + action="store", dest="input") + + filter_parser.add_argument('--include-fingerprints', + help="""include only measurements with known circuit path and with all + relays being contained in the fingerprints file located at + PATH""", + metavar="PATH", action="store", dest="include_fingerprints", + default=None) + + filter_parser.add_argument('--exclude-fingerprints', + help="""exclude measurements without known circuit path or with any + relays being contained in the fingerprints file located at + PATH""", + metavar="PATH", action="store", dest="exclude_fingerprints", + default=None) + + filter_parser.add_argument('-o', '--output', + help="""write the filtered output OnionPerf analysis results file to + PATH""", + metavar="PATH", required="True", + action="store", dest="output") + # visualize visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE, formatter_class=my_formatter_class) @@ -397,6 +436,18 @@ def analyze(args): else: logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed") +def filter(args): + from onionperf.filtering import Filtering + + filtering = Filtering() + filtering.read_input(args.input) + if args.include_fingerprints is not None: + filtering.include_fingerprints(args.include_fingerprints) + if args.exclude_fingerprints is not None: + filtering.exclude_fingerprints(args.exclude_fingerprints) + filtering.apply_filters() + filtering.write_output(args.output) + def visualize(args): from onionperf.visualization import TGenVisualization from onionperf.analysis import OPAnalysis

1 0

[onionperf/develop] Fix naming of exclusion list
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit d9f8b8fe36fabb30bf96802130a68b9020f6a875 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Aug 21 03:07:30 2020 +0100 Fix naming of exclusion list --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 204b161..ab96f1e 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -28,13 +28,13 @@ class Filtering(object): self.fingerprints_to_include.append(fingerprint) def exclude_fingerprints(self, path): - self.exclude_fingerprints = [] + self.fingerprints_to_exclude = [] with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() - self.exclude_fingerprints.append(fingerprint) + self.fingerprints_to_exclude.append(fingerprint) def apply_filters(self): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None:

1 0

[onionperf/develop] Prevent incorrectly keeping transfers for empty incl./excl. lists
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit efe2ef11eb91e803e44229f5ede0209ead0aae21 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 14:18:01 2020 +0100 Prevent incorrectly keeping transfers for empty incl./excl. lists --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index b431eb9..9e7b34f 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -79,10 +79,10 @@ class Filtering(object): fingerprint_match = self.fingerprint_pattern.match(long_name) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() - if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include: + if self.fingerprints_to_include is not None and fingerprint not in self.fingerprints_to_include: keep = False break - if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude: + if self.fingerprints_to_exclude is not None and fingerprint in self.fingerprints_to_exclude: keep = False break if keep:

1 0

[onionperf/develop] Only try to load analysis files in an input directory
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit d21d41e4d504b48ac5617530830da320a89a9eed Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 15:10:32 2020 +0100 Only try to load analysis files in an input directory --- onionperf/onionperf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index 7c16aea..e3f49c8 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -458,11 +458,12 @@ def filter(args): output_dir, output_file = os.path.split(output_path) filtering.apply_filters(input_path=input_path, output_dir=output_dir, output_file=output_file) else: - for dirpath, dirnames, filenames in os.walk(input_path): - for filename in filenames: - input_file = os.path.join(dirpath, filename) - output_dir = os.path.join(output_path, os.path.relpath(dirpath, input_path)) - filtering.apply_filters(input_path=input_file, output_dir=output_dir, output_file=filename) + from onionperf import reprocessing + analyses = reprocessing.collect_logs(input_path, '*onionperf.analysis.*') + for analysis in analyses: + full_output_path = os.path.join(output_path, os.path.relpath(analysis, input_path)) + output_dir, output_file = os.path.split(full_output_path) + filtering.apply_filters(input_path=analysis, output_dir=output_dir, output_file=output_file) def visualize(args): from onionperf.visualization import TGenVisualization

1 0

[onionperf/develop] Add filter section to README.md
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit 52506522bc6dc679318e2af801f7c3a94b6947d0 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 17:01:13 2020 +0100 Add filter section to README.md --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index ad53a9e..224972c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ + [Troubleshooting](#troubleshooting) * [Analysis](#analysis) + [Analyzing measurement results](#analyzing-measurement-results) + + [Filtering measurement results](#filtering-measurement-results) + [Visualizing measurement results](#visualizing-measurement-results) + [Interpreting the PDF output format](#interpreting-the-pdf-output-format) + [Interpreting the CSV output format](#interpreting-the-csv-output-format) @@ -251,6 +252,28 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` +### Filtering measurement results + +OnionPerf measurement results can be filtered based on Tor relay fingerprints. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. + +Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. + +For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': + +```shell +onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt +``` + +The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. +Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. + +OnionPerf's `filter` command usage can be inspected with: + +```shell +onionperf filter --help +``` + ### Visualizing measurement results

1 0

[onionperf/develop] Move filters and filter metadata to analysis files
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit 95b749a8fc690825c0a828b8473c58faea7ad912 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 01:51:54 2020 +0100 Move filters and filter metadata to analysis files --- onionperf/filtering.py | 25 ++++++++++++++++++------- onionperf/onionperf | 9 +-------- onionperf/visualization.py | 14 +++++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 1b614d6..c008c03 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -7,6 +7,7 @@ import re from onionperf.analysis import OPAnalysis +from collections import defaultdict class Filtering(object): @@ -14,9 +15,11 @@ class Filtering(object): self.fingerprints_to_include = None self.fingerprints_to_exclude = None self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})") + self.filters = defaultdict(list) def include_fingerprints(self, path): self.fingerprints_to_include = [] + self.fingerprints_to_include_path = path with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) @@ -26,6 +29,7 @@ class Filtering(object): def exclude_fingerprints(self, path): self.fingerprints_to_exclude = [] + self.fingerprints_to_exclude_path = path with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) @@ -33,12 +37,16 @@ class Filtering(object): fingerprint = fingerprint_match.group(1).upper() self.fingerprints_to_exclude.append(fingerprint) - def apply_filters(self, input_path, output_dir, output_file): - self.analysis = OPAnalysis.load(filename=input_path) + def filter_tor_circuits(self, analysis): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: return - for source in self.analysis.get_nodes(): - tor_circuits = self.analysis.get_tor_circuits(source) + self.filters["tor/circuits"] = [] + if self.fingerprints_to_include: + self.filters["tor/circuits"].append({"name": "include_fingerprints", "filepath": self.fingerprints_to_include_path }) + if self.fingerprints_to_exclude: + self.filters["tor/circuits"].append({"name": "exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path }) + for source in analysis.get_nodes(): + tor_circuits = analysis.get_tor_circuits(source) filtered_circuit_ids = [] for circuit_id, tor_circuit in tor_circuits.items(): keep = False @@ -56,8 +64,11 @@ class Filtering(object): keep = False break if not keep: - filtered_circuit_ids.append(circuit_id) - for circuit_id in filtered_circuit_ids: - del(tor_circuits[circuit_id]) + tor_circuits[circuit_id]["filtered"] = True + + def apply_filters(self, input_path, output_dir, output_file): + self.analysis = OPAnalysis.load(filename=input_path) + self.filter_tor_circuits(self.analysis) + self.analysis.json_db["filters"] = self.filters self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False) diff --git a/onionperf/onionperf b/onionperf/onionperf index 1efa8cb..108af4e 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -342,13 +342,6 @@ files generated by this script will be written""", required="True", action=PathStringArgsAction, dest="datasets") - visualize_parser.add_argument('--outer-join', - help="""Include measurements without an existing mapping between TGen - transfers/streams and Tor streams/circuits, which is the - equivalent of an outer join in the database sense""", - action="store_true", dest="outer_join", - default=False) - visualize_parser.add_argument('-p', '--prefix', help="a STRING filename prefix for graphs we generate", metavar="STRING", type=str, @@ -489,7 +482,7 @@ def visualize(args): if analysis is not None: analyses.append(analysis) tgen_viz.add_dataset(analyses, label) - tgen_viz.plot_all(args.prefix, outer_join=args.outer_join) + tgen_viz.plot_all(args.prefix) def type_nonnegative_integer(value): i = int(value) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 0f69879..f5bc03f 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -31,11 +31,11 @@ class Visualization(object, metaclass=ABCMeta): class TGenVisualization(Visualization): - def plot_all(self, output_prefix, outer_join=False): + def plot_all(self, output_prefix): if len(self.datasets) > 0: prefix = output_prefix + '.' if output_prefix is not None else '' ts = time.strftime("%Y-%m-%d_%H:%M:%S") - self.__extract_data_frame(outer_join) + self.__extract_data_frame() self.data.to_csv("{0}onionperf.viz.{1}.csv".format(prefix, ts)) sns.set_context("paper") self.page = PdfPages("{0}onionperf.viz.{1}.pdf".format(prefix, ts)) @@ -51,7 +51,7 @@ class TGenVisualization(Visualization): self.__plot_errors_time() self.page.close() - def __extract_data_frame(self, outer_join=False): + def __extract_data_frame(self): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: @@ -145,8 +145,12 @@ class TGenVisualization(Visualization): if "failure_reason_remote" in tor_stream: error_code_parts.append(tor_stream["failure_reason_remote"]) stream["error_code"] = "/".join(error_code_parts) - if tor_circuit or outer_join: - streams.append(stream) + + if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: + if tor_circuit and "filtered" not in tor_circuit.keys(): + streams.append(stream) + else: + streams.append(stream) self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/develop] Clarify filter terminology
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit a466221c5b947bb894a36fd3e5472397f1b0b03e Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 12:41:46 2020 +0100 Clarify filter terminology --- onionperf/filtering.py | 2 +- onionperf/visualization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 7ef6168..0ba9ebc 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -64,7 +64,7 @@ class Filtering(object): keep = False break if not keep: - tor_circuits[circuit_id]["filtered"] = True + tor_circuits[circuit_id]["filtered_out"] = True tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f5bc03f..2cd3161 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -147,7 +147,7 @@ class TGenVisualization(Visualization): stream["error_code"] = "/".join(error_code_parts) if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: - if tor_circuit and "filtered" not in tor_circuit.keys(): + if tor_circuit and "filtered_out" not in tor_circuit.keys(): streams.append(stream) else: streams.append(stream)

1 0

[onionperf/develop] Add JSON schema for analysis version 4.0.
by karsten＠torproject.org 16 Sep '20

16 Sep '20

commit a4a4dcc97d07b2668fbab593165119bd52f694ed Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Sep 16 11:19:58 2020 +0200 Add JSON schema for analysis version 4.0. --- schema/onionperf-4.0.json | 577 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 577 insertions(+) diff --git a/schema/onionperf-4.0.json b/schema/onionperf-4.0.json new file mode 100644 index 0000000..3c4dba3 --- /dev/null +++ b/schema/onionperf-4.0.json @@ -0,0 +1,577 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://gitlab.torproject.org/tpo/metrics/onionperf/-/raw/master/schema/oni…", + "type": "object", + "title": "OnionPerf analysis JSON file format 4.0", + "required": [ + "data", + "type", + "version" + ], + "properties": { + "data": { + "type": "object", + "title": "Measurement data by source name", + "propertyNames": { + "pattern": "^[A-Za-z0-9-]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Measurement data from a single source", + "required": [ + "measurement_ip", + "tgen", + "tor" + ], + "properties": { + "measurement_ip": { + "type": "string", + "title": "Public IP address of the measuring host." + }, + "tgen": { + "type": "object", + "title": "Measurement data obtained from client-side TGen logs", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "object", + "title": "Measurement data, by TGen stream identifier", + "additionalProperties": { + "type": "object", + "title": "Information on a single measurement, obtained from a single [stream-success] or [stream-error] log message (except for elapsed_seconds)", + "required": [ + "byte_info", + "is_complete", + "is_error", + "is_success", + "stream_id", + "stream_info", + "time_info", + "transport_info", + "unix_ts_end", + "unix_ts_start" + ], + "properties": { + "byte_info": { + "type": "object", + "title": "Information on sent and received bytes", + "required": [ + "payload-bytes-recv", + "payload-bytes-send", + "payload-progress-recv", + "payload-progress-send", + "total-bytes-recv", + "total-bytes-send" + ], + "properties": { + "payload-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes received" + }, + "payload-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes sent" + }, + "payload-progress-recv": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of receiving payload in percent" + }, + "payload-progress-send": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of sending payload in percent" + }, + "total-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes received" + }, + "total-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes sent" + } + } + }, + "elapsed_seconds": { + "type": "object", + "title": "Elapsed seconds until a given number or fraction of payload bytes have been received or sent, obtained from [stream-status], [stream-success], and [stream-error] log messages, only included if the measurement was a success", + "properties": { + "payload_bytes_recv": { + "type": "object", + "title": "Number of received payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_bytes_send": { + "type": "object", + "title": "Number of sent payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_recv": { + "type": "object", + "title": "Fraction of received payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_send": { + "type": "object", + "title": "Fraction of sent payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + } + } + }, + "is_complete": { + "type": "boolean", + "title": "Whether the stream finished, no matter the error state, which is always true, or otherwise the measurement would not be included here" + }, + "is_error": { + "type": "boolean", + "title": "Whether an error occurred" + }, + "is_success": { + "type": "boolean", + "title": "Whether the measurement was a success" + }, + "stream_id": { + "type": "string", + "title": "Stream identifier" + }, + "stream_info": { + "type": "object", + "title": "Information about the TGen stream", + "required": [ + "error", + "id", + "name", + "peername", + "recvsize", + "recvstate", + "sendsize", + "sendstate", + "vertexid" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "id": { + "type": "string", + "title": "Stream numerical identifier, or 0 if the stream failed" + }, + "name": { + "type": "string", + "title": "Hostname of the TGen client" + }, + "peername": { + "type": "string", + "title": "Hostname of the TGen server" + }, + "recvsize": { + "type": "string", + "title": "Number of expected payload bytes in the response" + }, + "recvstate": { + "type": "string", + "title": "Last recorded receive state of the stream, one of RECV_{NONE,AUTHENTICATE,HEADER,MODEL,PAYLOAD,CHECKSUM,SUCCESS,ERROR}" + }, + "sendsize": { + "type": "string", + "title": "Number of expected payload bytes in the request" + }, + "sendstate": { + "type": "string", + "title": "Last recorded send state of the stream, one of SEND_{NONE,COMMAND,RESPONSE,PAYLOAD,CHECKSUM,FLUSH,SUCCESS,ERROR}" + }, + "vertexid": { + "type": "string", + "title": "Vertex identifier in the TGen model" + } + } + }, + "time_info": { + "type": "object", + "title": "Elapsed time until reaching given substeps in a measurement", + "required": [ + "created-ts", + "now-ts", + "usecs-to-checksum-recv", + "usecs-to-checksum-send", + "usecs-to-command", + "usecs-to-first-byte-recv", + "usecs-to-first-byte-send", + "usecs-to-last-byte-recv", + "usecs-to-last-byte-send", + "usecs-to-proxy-choice", + "usecs-to-proxy-init", + "usecs-to-proxy-request", + "usecs-to-proxy-response", + "usecs-to-response", + "usecs-to-socket-connect", + "usecs-to-socket-create" + ], + "properties": { + "created-ts": { + "type": "string", + "title": "Montonic system time when TGen created this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "now-ts": { + "type": "string", + "title": "Montonic system time when TGen computed elapsed microseconds for this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "usecs-to-checksum-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the checksum from the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-checksum-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the checksum to the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-command": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the command to the TGen server, or -1 if missing (step 7)" + }, + "usecs-to-first-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-first-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-last-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-last-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-proxy-choice": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS choice from the Tor client, or -1 if missing (step 4)" + }, + "usecs-to-proxy-init": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS initialization to the Tor client, or -1 if missing (step 3)" + }, + "usecs-to-proxy-request": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS request to the Tor client, or -1 if missing (step 5)" + }, + "usecs-to-proxy-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS response from the Tor client, or -1 if missing (step 6)" + }, + "usecs-to-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the command from the TGen server, or -1 if missing (step 8)" + }, + "usecs-to-socket-connect": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has connected to the Tor client's SOCKS port, or -1 if missing (step 2)" + }, + "usecs-to-socket-create": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has opened a TCP connection to the Tor client's SOCKS port, or -1 if missing (step 1)" + } + } + }, + "transport_info": { + "type": "object", + "title": "Information about the TGen transport", + "required": [ + "error", + "fd", + "local", + "proxy", + "remote", + "state" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "fd": { + "type": "string", + "title": "File descriptor" + }, + "local": { + "type": "string", + "title": "Local host name, IP address, and TCP port" + }, + "proxy": { + "type": "string", + "title": "Proxy host name, IP address, and TCP port" + }, + "remote": { + "type": "string", + "title": "Remote host name, IP address, and TCP port" + }, + "state": { + "type": "string", + "title": "Last recorded state of the transport, one of CONNECT,INIT,CHOICE,REQUEST,AUTH_{REQUEST,RESPONSE},RESPONSE_{STATUS,TYPE,IPV4,NAMELEN,NAME},SUCCESS_{OPEN,EOF},ERROR" + } + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the measurement, obtained from the log time of the [stream-success] or [stream-error] log message, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the measurement, obtained by subtracting the largest number of elapsed microseconds in time_info from unix_ts_end, given in seconds since the epoch" + } + } + } + } + } + }, + "tor": { + "type": "object", + "title": "Metadata obtained from client-side Tor controller logs", + "required": [ + "circuits", + "streams" + ], + "properties": { + "circuits": { + "type": "object", + "title": "Information about Tor circuits, by circuit identifier, obtained from CIRC and CIRC_MINOR events, for all circuits created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor circuit", + "required": [ + "circuit_id", + "elapsed_seconds", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "build_quantile": { + "type": "number", + "title": "Circuit build time quantile, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC LAUNCHED event" + }, + "build_timeout": { + "type": "integer", + "title": "Circuit build time in milliseconds, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC event with status LAUNCHED" + }, + "buildtime_seconds": { + "type": "number", + "title": "Build time in seconds, computed as time elapsed between CIRC LAUNCHED and CIRC BUILT events" + }, + "circuit_id": { + "type": "integer", + "title": "Circuit identifier, obtained from CIRC and CIRC_MINOR events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging CIRC and CIRC_MINOR events", + "items": { + "type": "array", + "title": "Elapsed seconds until reaching a given circuit status change", + "items": [ + { + "type": "string", + "title": "Circuit status change" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from CIRC FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from CIRC FAILED events" + }, + "filtered_out": { + "type": "boolean", + "title": "Whether this circuit has been filtered out when applying filters in `onionperf filter`." + }, + "path": { + "type": "array", + "title": "Path information", + "items": { + "type": "array", + "title": "Elapsed seconds until extending the circuit to a given relay", + "items": [ + { + "type": "string", + "pattern": "^\\$[0-9A-Z]{40}~[0-9a-zA-Z]{1,19}$", + "title": "Relay fingerprint and nickname" + }, + { + "type": "number", + "minimum": 0, + "title": "Elapsed seconds" + } + ] + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the circuit, obtained from the log time of the last CIRC CLOSED or CIRC FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the circuit, obtained from the log time of the CIRC LAUNCHED event, given in seconds since the epoch" + } + } + } + }, + "streams": { + "type": "object", + "title": "Information about Tor stream, by stream identifier, obtained from STREAM events, for all streams created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor stream", + "required": [ + "circuit_id", + "elapsed_seconds", + "stream_id", + "target", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "circuit_id": { + "title": "Circuit identifier, obtained from STREAM events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging STREAM events", + "items": { + "type": "array", + "items": [ + { + "type": "string", + "title": "Stream purpose and STREAM event status" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from STREAM FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from STREAM FAILED events" + }, + "source": { + "type": "string", + "title": "Stream source IP address and TCP port, obtained from STREAM NEW or STREAM NEWRESOLVE events" + }, + "stream_id": { + "type": "integer", + "title": "Stream identifier, unique at least for the lifetime of this stream" + }, + "target": { + "type": "string", + "title": "Stream target domain name and TCP port, obtained from STREAM events", + "examples": [ + "jzxfvaupigl7hkemf4jhfi2vrruvbb7ucyiwdolkkc2hf3xlm34f3qyd.onion:8080" + ] + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the stream, obtained from the log time of the last STREAM CLOSED or STREAM FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the stream, obtained from the log time of the first STREAM NEW or STREAM NEWRESOLVE event, given in seconds since the epoch" + } + } + } + } + } + } + } + } + }, + "filters": { + "type": "object", + "title": "Filters applied by type", + "propertyNames": { + "pattern": "^[A-Za-z/]" + }, + "additionalProperties": { + "type": "array", + "title": "Filters applied of a given type", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "title": "Filter name" + }, + "filepath": { + "type": "string", + "title": "File path" + } + } + } + } + }, + "type": { + "type": "string", + "title": "Document type", + "const": "onionperf" + }, + "version": { + "type": "string", + "title": "Document version", + "pattern": "^4\\.[0-9]+$" + } + } +}

1 0