September 2020 - tor-commits - lists.torproject.org

[onionperf/master] Fix matching of tor_circuit ids
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 26ff79a3e57f1dfb0c961cceb167534bb39d17f3 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Aug 21 02:14:32 2020 +0100 Fix matching of tor_circuit ids --- onionperf/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 435a1bc..204b161 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -73,7 +73,7 @@ class Filtering(object): for tor_stream in tor_streams_by_source_port[source_port]: if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: circuit_id = tor_stream["circuit_id"] - if circuit_id and circuit_id in tor_circuits: + if circuit_id and str(circuit_id) in tor_circuits: tor_circuit = tor_circuits[circuit_id] path = tor_circuit["path"] keep = True

1 0

[onionperf/master] Only try to load analysis files in an input directory
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit d21d41e4d504b48ac5617530830da320a89a9eed Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 15:10:32 2020 +0100 Only try to load analysis files in an input directory --- onionperf/onionperf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index 7c16aea..e3f49c8 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -458,11 +458,12 @@ def filter(args): output_dir, output_file = os.path.split(output_path) filtering.apply_filters(input_path=input_path, output_dir=output_dir, output_file=output_file) else: - for dirpath, dirnames, filenames in os.walk(input_path): - for filename in filenames: - input_file = os.path.join(dirpath, filename) - output_dir = os.path.join(output_path, os.path.relpath(dirpath, input_path)) - filtering.apply_filters(input_path=input_file, output_dir=output_dir, output_file=filename) + from onionperf import reprocessing + analyses = reprocessing.collect_logs(input_path, '*onionperf.analysis.*') + for analysis in analyses: + full_output_path = os.path.join(output_path, os.path.relpath(analysis, input_path)) + output_dir, output_file = os.path.split(full_output_path) + filtering.apply_filters(input_path=analysis, output_dir=output_dir, output_file=output_file) def visualize(args): from onionperf.visualization import TGenVisualization

1 0

[onionperf/master] Prevent incorrectly keeping transfers for empty incl./excl. lists
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit efe2ef11eb91e803e44229f5ede0209ead0aae21 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 14:18:01 2020 +0100 Prevent incorrectly keeping transfers for empty incl./excl. lists --- onionperf/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index b431eb9..9e7b34f 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -79,10 +79,10 @@ class Filtering(object): fingerprint_match = self.fingerprint_pattern.match(long_name) if fingerprint_match: fingerprint = fingerprint_match.group(1).upper() - if self.fingerprints_to_include and fingerprint not in self.fingerprints_to_include: + if self.fingerprints_to_include is not None and fingerprint not in self.fingerprints_to_include: keep = False break - if self.fingerprints_to_exclude and fingerprint in self.fingerprints_to_exclude: + if self.fingerprints_to_exclude is not None and fingerprint in self.fingerprints_to_exclude: keep = False break if keep:

1 0

[onionperf/master] Change filter mode to filter Tor circuits.
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 9d0c80561b84905e72b41758dfcb7b712f5f407f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 31 11:30:53 2020 +0200 Change filter mode to filter Tor circuits. This new filter mode removes Tor circuits that don't match the provided fingerprints and leaves TGen transfers/streams untouched. At the same time the visualize mode only includes TGen transfers/streams with an existing mapping between TGen transfers/streams and Tor streams/circuits. This patch changes the default behavior of the visualize mode. The original behavior of visualizing TGen transfers/streams *without* an existing mapping to Tor streams/circuits can be selected with the --outer-join switch, even though that's rather an edge use case. Another minor change is that the filtered analysis files is not written with sort_keys=True anymore, which would have produced a newly sorted file with keys in alphabetic order rather than the original insert order. The result is an actually useful diff. --- onionperf/analysis.py | 13 +++++------- onionperf/filtering.py | 49 ++++++++-------------------------------------- onionperf/onionperf | 24 +++++++++++++++++------ onionperf/visualization.py | 33 ++++++++++++++++++++----------- 4 files changed, 53 insertions(+), 66 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index b2f483f..49c109c 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -62,13 +62,7 @@ class OPAnalysis(Analysis): self.json_db['data'][self.nickname]["tgen"].pop("stream_summary") self.did_analysis = True - def set_tgen_transfers(self, node, tgen_transfers): - self.json_db['data'][node]['tgen']['transfers'] = tgen_transfers - - def set_tgen_streams(self, node, tgen_streams): - self.json_db['data'][node]['tgen']['streams'] = tgen_streams - - def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None): + def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None, sort_keys=True): if filename is None: base_filename = "onionperf.analysis.json.xz" if date_prefix is not None: @@ -85,7 +79,7 @@ class OPAnalysis(Analysis): logging.info("saving analysis results to {0}".format(filepath)) outf = util.FileWritable(filepath, do_compress=do_compress) - json.dump(self.json_db, outf, sort_keys=True, separators=(',', ': '), indent=2) + json.dump(self.json_db, outf, sort_keys=sort_keys, separators=(',', ': '), indent=2) outf.close() logging.info("done!") @@ -109,6 +103,9 @@ class OPAnalysis(Analysis): except: return None + def set_tor_circuits(self, node, tor_circuits): + self.json_db['data'][node]['tor']['circuits'] = tor_circuits + def get_tor_streams(self, node): try: return self.json_db['data'][node]['tor']['streams'] diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 9e7b34f..1b614d6 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -38,41 +38,11 @@ class Filtering(object): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: return for source in self.analysis.get_nodes(): - tor_streams_by_source_port = {} - tor_streams = self.analysis.get_tor_streams(source) - for tor_stream in tor_streams.values(): - if "source" in tor_stream and ":" in tor_stream["source"]: - source_port = tor_stream["source"].split(":")[1] - tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream) tor_circuits = self.analysis.get_tor_circuits(source) - tgen_streams = self.analysis.get_tgen_streams(source) - tgen_transfers = self.analysis.get_tgen_transfers(source) - retained_tgen_streams = {} - retained_tgen_transfers = {} - while tgen_streams or tgen_transfers: - stream_id = None - transfer_id = None - source_port = None - unix_ts_end = None + filtered_circuit_ids = [] + for circuit_id, tor_circuit in tor_circuits.items(): keep = False - if tgen_streams: - stream_id, stream_data = tgen_streams.popitem() - if "local" in stream_data["transport_info"] and len(stream_data["transport_info"]["local"].split(":")) > 2: - source_port = stream_data["transport_info"]["local"].split(":")[2] - if "unix_ts_end" in stream_data: - unix_ts_end = stream_data["unix_ts_end"] - elif tgen_transfers: - transfer_id, transfer_data = tgen_transfers.popitem() - if "endpoint_local" in transfer_data and len(transfer_data["endpoint_local"].split(":")) > 2: - source_port = transfer_data["endpoint_local"].split(":")[2] - if "unix_ts_end" in transfer_data: - unix_ts_end = transfer_data["unix_ts_end"] - if source_port and unix_ts_end: - for tor_stream in tor_streams_by_source_port[source_port]: - if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: - circuit_id = tor_stream["circuit_id"] - if circuit_id and str(circuit_id) in tor_circuits: - tor_circuit = tor_circuits[circuit_id] + if "path" in tor_circuit: path = tor_circuit["path"] keep = True for long_name, _ in path: @@ -85,12 +55,9 @@ class Filtering(object): if self.fingerprints_to_exclude is not None and fingerprint in self.fingerprints_to_exclude: keep = False break - if keep: - if stream_id: - retained_tgen_streams[stream_id] = stream_data - if transfer_id: - retained_tgen_transfers[transfer_id] = transfer_data - self.analysis.set_tgen_streams(source, retained_tgen_streams) - self.analysis.set_tgen_transfers(source, retained_tgen_transfers) - self.analysis.save(filename=output_file, output_prefix=output_dir) + if not keep: + filtered_circuit_ids.append(circuit_id) + for circuit_id in filtered_circuit_ids: + del(tor_circuits[circuit_id]) + self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False) diff --git a/onionperf/onionperf b/onionperf/onionperf index e3f49c8..1efa8cb 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -79,8 +79,13 @@ DESC_FILTER = """ Takes an OnionPerf analysis results file or directory as input, applies filters, and produces new OnionPerf analysis results file(s) as output. -This subcommand only filters measurements in `data/[source]/tgen/transfers` -and `data/[source]/tgen/streams`, but leaves any summaries unchanged. +The `filter` subcommand is typically used in combination with the `visualize` +subcommand. The work flow is to filter out any TGen transfers/streams or Tor +streams/circuits that are not supposed to be visualized and then visualize only +those measurements with an existing mapping between TGen transfers/streams and +Tor streams/circuits. + +This subcommand only filters individual objects and leaves summaries unchanged. """ HELP_FILTER = """ Filter OnionPerf analysis results @@ -304,15 +309,15 @@ files generated by this script will be written""", action="store", dest="input") filter_parser.add_argument('--include-fingerprints', - help="""include only measurements with known circuit path and with all + help="""include only Tor circuits with known circuit path and with all relays being contained in the fingerprints file located at PATH""", metavar="PATH", action="store", dest="include_fingerprints", default=None) filter_parser.add_argument('--exclude-fingerprints', - help="""exclude measurements without known circuit path or with any - relays being contained in the fingerprints file located at + help="""exclude Tor circuits without known circuit path or with any + relay being contained in the fingerprints file located at PATH""", metavar="PATH", action="store", dest="exclude_fingerprints", default=None) @@ -337,6 +342,13 @@ files generated by this script will be written""", required="True", action=PathStringArgsAction, dest="datasets") + visualize_parser.add_argument('--outer-join', + help="""Include measurements without an existing mapping between TGen + transfers/streams and Tor streams/circuits, which is the + equivalent of an outer join in the database sense""", + action="store_true", dest="outer_join", + default=False) + visualize_parser.add_argument('-p', '--prefix', help="a STRING filename prefix for graphs we generate", metavar="STRING", type=str, @@ -477,7 +489,7 @@ def visualize(args): if analysis is not None: analyses.append(analysis) tgen_viz.add_dataset(analyses, label) - tgen_viz.plot_all(args.prefix) + tgen_viz.plot_all(args.prefix, outer_join=args.outer_join) def type_nonnegative_integer(value): i = int(value) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 660f52e..0f69879 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -31,11 +31,11 @@ class Visualization(object, metaclass=ABCMeta): class TGenVisualization(Visualization): - def plot_all(self, output_prefix): + def plot_all(self, output_prefix, outer_join=False): if len(self.datasets) > 0: prefix = output_prefix + '.' if output_prefix is not None else '' ts = time.strftime("%Y-%m-%d_%H:%M:%S") - self.__extract_data_frame() + self.__extract_data_frame(outer_join) self.data.to_csv("{0}onionperf.viz.{1}.csv".format(prefix, ts)) sns.set_context("paper") self.page = PdfPages("{0}onionperf.viz.{1}.pdf".format(prefix, ts)) @@ -51,7 +51,7 @@ class TGenVisualization(Visualization): self.__plot_errors_time() self.page.close() - def __extract_data_frame(self): + def __extract_data_frame(self, outer_join=False): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: @@ -62,6 +62,7 @@ class TGenVisualization(Visualization): if "source" in tor_stream and ":" in tor_stream["source"]: source_port = tor_stream["source"].split(":")[1] tor_streams_by_source_port.setdefault(source_port, []).append(tor_stream) + tor_circuits = analysis.get_tor_circuits(client) tgen_streams = analysis.get_tgen_streams(client) tgen_transfers = analysis.get_tgen_transfers(client) while tgen_streams or tgen_transfers: @@ -122,20 +123,30 @@ class TGenVisualization(Visualization): unix_ts_end = transfer_data["unix_ts_end"] if "unix_ts_start" in transfer_data: stream["start"] = datetime.datetime.utcfromtimestamp(transfer_data["unix_ts_start"]) + tor_stream = None + tor_circuit = None + if source_port and unix_ts_end: + for s in tor_streams_by_source_port[source_port]: + if abs(unix_ts_end - s["unix_ts_end"]) < 150.0: + tor_stream = s + break + if tor_stream and "circuit_id" in tor_stream: + circuit_id = tor_stream["circuit_id"] + if str(circuit_id) in tor_circuits: + tor_circuit = tor_circuits[circuit_id] if error_code: if error_code == "PROXY": error_code_parts = ["TOR"] else: error_code_parts = ["TGEN", error_code] - if source_port and unix_ts_end: - for tor_stream in tor_streams_by_source_port[source_port]: - if abs(unix_ts_end - tor_stream["unix_ts_end"]) < 150.0: - if "failure_reason_local" in tor_stream: - error_code_parts.append(tor_stream["failure_reason_local"]) - if "failure_reason_remote" in tor_stream: - error_code_parts.append(tor_stream["failure_reason_remote"]) + if tor_stream: + if "failure_reason_local" in tor_stream: + error_code_parts.append(tor_stream["failure_reason_local"]) + if "failure_reason_remote" in tor_stream: + error_code_parts.append(tor_stream["failure_reason_remote"]) stream["error_code"] = "/".join(error_code_parts) - streams.append(stream) + if tor_circuit or outer_join: + streams.append(stream) self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/master] Clarify filter terminology
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit a466221c5b947bb894a36fd3e5472397f1b0b03e Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 12:41:46 2020 +0100 Clarify filter terminology --- onionperf/filtering.py | 2 +- onionperf/visualization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 7ef6168..0ba9ebc 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -64,7 +64,7 @@ class Filtering(object): keep = False break if not keep: - tor_circuits[circuit_id]["filtered"] = True + tor_circuits[circuit_id]["filtered_out"] = True tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): diff --git a/onionperf/visualization.py b/onionperf/visualization.py index f5bc03f..2cd3161 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -147,7 +147,7 @@ class TGenVisualization(Visualization): stream["error_code"] = "/".join(error_code_parts) if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: - if tor_circuit and "filtered" not in tor_circuit.keys(): + if tor_circuit and "filtered_out" not in tor_circuit.keys(): streams.append(stream) else: streams.append(stream)

1 0

[onionperf/master] Add filter section to README.md
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 52506522bc6dc679318e2af801f7c3a94b6947d0 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Aug 29 17:01:13 2020 +0100 Add filter section to README.md --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index ad53a9e..224972c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ + [Troubleshooting](#troubleshooting) * [Analysis](#analysis) + [Analyzing measurement results](#analyzing-measurement-results) + + [Filtering measurement results](#filtering-measurement-results) + [Visualizing measurement results](#visualizing-measurement-results) + [Interpreting the PDF output format](#interpreting-the-pdf-output-format) + [Interpreting the CSV output format](#interpreting-the-csv-output-format) @@ -251,6 +252,28 @@ OnionPerf's `analyze` mode has several command-line parameters for customizing t ```shell onionperf analyze --help ``` +### Filtering measurement results + +OnionPerf measurement results can be filtered based on Tor relay fingerprints. +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. + +Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. + +For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': + +```shell +onionperf filter -i onionperf.analysis.json.xz -o filtered.onionperf.analysis.json.xz --include-fingerprints fingerprints.txt +``` + +The output analysis file is written to the path specified with `-o`. If processing a directory of analysis files, its structure and filenames are preserved under the path specified with '-o'. +Note that while the subcommand filters `tgen` measurements, it leaves `tgen` and `tor` summaries in the original analysis file unchanged. + +OnionPerf's `filter` command usage can be inspected with: + +```shell +onionperf filter --help +``` + ### Visualizing measurement results

1 0

[onionperf/master] Move filters and filter metadata to analysis files
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 95b749a8fc690825c0a828b8473c58faea7ad912 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 01:51:54 2020 +0100 Move filters and filter metadata to analysis files --- onionperf/filtering.py | 25 ++++++++++++++++++------- onionperf/onionperf | 9 +-------- onionperf/visualization.py | 14 +++++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index 1b614d6..c008c03 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -7,6 +7,7 @@ import re from onionperf.analysis import OPAnalysis +from collections import defaultdict class Filtering(object): @@ -14,9 +15,11 @@ class Filtering(object): self.fingerprints_to_include = None self.fingerprints_to_exclude = None self.fingerprint_pattern = re.compile("\$?([0-9a-fA-F]{40})") + self.filters = defaultdict(list) def include_fingerprints(self, path): self.fingerprints_to_include = [] + self.fingerprints_to_include_path = path with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) @@ -26,6 +29,7 @@ class Filtering(object): def exclude_fingerprints(self, path): self.fingerprints_to_exclude = [] + self.fingerprints_to_exclude_path = path with open(path, 'rt') as f: for line in f: fingerprint_match = self.fingerprint_pattern.match(line) @@ -33,12 +37,16 @@ class Filtering(object): fingerprint = fingerprint_match.group(1).upper() self.fingerprints_to_exclude.append(fingerprint) - def apply_filters(self, input_path, output_dir, output_file): - self.analysis = OPAnalysis.load(filename=input_path) + def filter_tor_circuits(self, analysis): if self.fingerprints_to_include is None and self.fingerprints_to_exclude is None: return - for source in self.analysis.get_nodes(): - tor_circuits = self.analysis.get_tor_circuits(source) + self.filters["tor/circuits"] = [] + if self.fingerprints_to_include: + self.filters["tor/circuits"].append({"name": "include_fingerprints", "filepath": self.fingerprints_to_include_path }) + if self.fingerprints_to_exclude: + self.filters["tor/circuits"].append({"name": "exclude_fingerprints", "filepath": self.fingerprints_to_exclude_path }) + for source in analysis.get_nodes(): + tor_circuits = analysis.get_tor_circuits(source) filtered_circuit_ids = [] for circuit_id, tor_circuit in tor_circuits.items(): keep = False @@ -56,8 +64,11 @@ class Filtering(object): keep = False break if not keep: - filtered_circuit_ids.append(circuit_id) - for circuit_id in filtered_circuit_ids: - del(tor_circuits[circuit_id]) + tor_circuits[circuit_id]["filtered"] = True + + def apply_filters(self, input_path, output_dir, output_file): + self.analysis = OPAnalysis.load(filename=input_path) + self.filter_tor_circuits(self.analysis) + self.analysis.json_db["filters"] = self.filters self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False) diff --git a/onionperf/onionperf b/onionperf/onionperf index 1efa8cb..108af4e 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -342,13 +342,6 @@ files generated by this script will be written""", required="True", action=PathStringArgsAction, dest="datasets") - visualize_parser.add_argument('--outer-join', - help="""Include measurements without an existing mapping between TGen - transfers/streams and Tor streams/circuits, which is the - equivalent of an outer join in the database sense""", - action="store_true", dest="outer_join", - default=False) - visualize_parser.add_argument('-p', '--prefix', help="a STRING filename prefix for graphs we generate", metavar="STRING", type=str, @@ -489,7 +482,7 @@ def visualize(args): if analysis is not None: analyses.append(analysis) tgen_viz.add_dataset(analyses, label) - tgen_viz.plot_all(args.prefix, outer_join=args.outer_join) + tgen_viz.plot_all(args.prefix) def type_nonnegative_integer(value): i = int(value) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 0f69879..f5bc03f 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -31,11 +31,11 @@ class Visualization(object, metaclass=ABCMeta): class TGenVisualization(Visualization): - def plot_all(self, output_prefix, outer_join=False): + def plot_all(self, output_prefix): if len(self.datasets) > 0: prefix = output_prefix + '.' if output_prefix is not None else '' ts = time.strftime("%Y-%m-%d_%H:%M:%S") - self.__extract_data_frame(outer_join) + self.__extract_data_frame() self.data.to_csv("{0}onionperf.viz.{1}.csv".format(prefix, ts)) sns.set_context("paper") self.page = PdfPages("{0}onionperf.viz.{1}.pdf".format(prefix, ts)) @@ -51,7 +51,7 @@ class TGenVisualization(Visualization): self.__plot_errors_time() self.page.close() - def __extract_data_frame(self, outer_join=False): + def __extract_data_frame(self): streams = [] for (analyses, label) in self.datasets: for analysis in analyses: @@ -145,8 +145,12 @@ class TGenVisualization(Visualization): if "failure_reason_remote" in tor_stream: error_code_parts.append(tor_stream["failure_reason_remote"]) stream["error_code"] = "/".join(error_code_parts) - if tor_circuit or outer_join: - streams.append(stream) + + if "filters" in analysis.json_db.keys() and analysis.json_db["filters"]["tor/circuits"]: + if tor_circuit and "filtered" not in tor_circuit.keys(): + streams.append(stream) + else: + streams.append(stream) self.data = pd.DataFrame.from_records(streams, index="id") def __plot_firstbyte_ecdf(self):

1 0

[onionperf/master] Update README and CHANGELOG
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 1b638e34cb0476148dcf93b43b58b54a30c27428 Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 14:46:10 2020 +0100 Update README and CHANGELOG --- CHANGELOG.md | 5 +++++ README.md | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd43e1..a7d5005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ - Add a new `onionperf filter` mode that takes an OnionPerf analysis results file or directory as input, applies filters, and produces new OnionPerf analysis results file(s) as output. + - Bump the analysis version number to 4.0 for new analyses and analyses + produced by the `onionperf filter` mode + - Analyses produced by the `onionperf filter` mode have additional filter + metadata defined in a new 'filters' field, and an optional 'filtered\_out' + field per tor circuit # Changes in version 0.6 - 2020-08-08 diff --git a/README.md b/README.md index 224972c..13a9eee 100644 --- a/README.md +++ b/README.md @@ -254,10 +254,12 @@ onionperf analyze --help ``` ### Filtering measurement results -OnionPerf measurement results can be filtered based on Tor relay fingerprints. -The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files containing only the `tgen` results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. +The `filter` subcommand is typically used in combination with the `visualize` subcommand. The workflow is to filter out any Tor streams/circuits that are not desired then visualize only those measurements with an existing mapping between TGen transfers/streams and Tor streams/circuits. -Where excluding fingerprints, if ANY relay fingerprint is matched, the measurement is discarded. Where including fingerprints, ALL relay fingerprints in a path must match for the measurement to be retained. +Currently, OnionPerf measurement results can be filtered based on Tor relay fingerprints, although support for filtering TGen based on transfers/streams may be added in the future. + +The `filter` mode takes a list of fingerprints and one or more existing analysis files as inputs, and outputs new analysis files which include, unchanged, the Tor results obtained over a Tor circuit path which includes or excludes fingerprints in the input list. All other Tor results are also included in the file, but are marked as 'filtered\_out'. +Filter metadata detailing the filter type and path to the input list used is also included in the analysis file. For example, the analysis file produced above can be filtered with the following command, which retains measurements based on fingerprints contained in the file 'fingerprints.txt': @@ -290,6 +292,8 @@ As a result, two files are written to the current working directory: - `onionperf.viz.$datetime.csv` contains visualized data in a CSV file format; and - `onionperf.viz.$datetime.pdf` contains visualizations in a PDF file format. +For analysis files containing tor circuit filters, only measurements with an existing mapping between TGen transfers/streams Tor streams/circuits which have not been marked as 'filtered\_out' are visualized. + Similar to the other modes, OnionPerf's `visualize` mode has command-line parameters for customizing the visualization step: ```shell

1 0

[onionperf/master] Add JSON schema for analysis version 4.0.
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit a4a4dcc97d07b2668fbab593165119bd52f694ed Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Sep 16 11:19:58 2020 +0200 Add JSON schema for analysis version 4.0. --- schema/onionperf-4.0.json | 577 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 577 insertions(+) diff --git a/schema/onionperf-4.0.json b/schema/onionperf-4.0.json new file mode 100644 index 0000000..3c4dba3 --- /dev/null +++ b/schema/onionperf-4.0.json @@ -0,0 +1,577 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://gitlab.torproject.org/tpo/metrics/onionperf/-/raw/master/schema/oni…", + "type": "object", + "title": "OnionPerf analysis JSON file format 4.0", + "required": [ + "data", + "type", + "version" + ], + "properties": { + "data": { + "type": "object", + "title": "Measurement data by source name", + "propertyNames": { + "pattern": "^[A-Za-z0-9-]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Measurement data from a single source", + "required": [ + "measurement_ip", + "tgen", + "tor" + ], + "properties": { + "measurement_ip": { + "type": "string", + "title": "Public IP address of the measuring host." + }, + "tgen": { + "type": "object", + "title": "Measurement data obtained from client-side TGen logs", + "required": [ + "streams" + ], + "properties": { + "streams": { + "type": "object", + "title": "Measurement data, by TGen stream identifier", + "additionalProperties": { + "type": "object", + "title": "Information on a single measurement, obtained from a single [stream-success] or [stream-error] log message (except for elapsed_seconds)", + "required": [ + "byte_info", + "is_complete", + "is_error", + "is_success", + "stream_id", + "stream_info", + "time_info", + "transport_info", + "unix_ts_end", + "unix_ts_start" + ], + "properties": { + "byte_info": { + "type": "object", + "title": "Information on sent and received bytes", + "required": [ + "payload-bytes-recv", + "payload-bytes-send", + "payload-progress-recv", + "payload-progress-send", + "total-bytes-recv", + "total-bytes-send" + ], + "properties": { + "payload-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes received" + }, + "payload-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Number of payload bytes sent" + }, + "payload-progress-recv": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of receiving payload in percent" + }, + "payload-progress-send": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+%$", + "title": "Progress of sending payload in percent" + }, + "total-bytes-recv": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes received" + }, + "total-bytes-send": { + "type": "string", + "pattern": "^[0-9]+$", + "title": "Total number of bytes sent" + } + } + }, + "elapsed_seconds": { + "type": "object", + "title": "Elapsed seconds until a given number or fraction of payload bytes have been received or sent, obtained from [stream-status], [stream-success], and [stream-error] log messages, only included if the measurement was a success", + "properties": { + "payload_bytes_recv": { + "type": "object", + "title": "Number of received payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_bytes_send": { + "type": "object", + "title": "Number of sent payload bytes", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_recv": { + "type": "object", + "title": "Fraction of received payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + }, + "payload_progress_send": { + "type": "object", + "title": "Fraction of sent payload bytes", + "propertyNames": { + "pattern": "^[01]\\.[0-9]$" + }, + "additionalProperties": { + "type": "number", + "title": "Elapsed seconds" + } + } + } + }, + "is_complete": { + "type": "boolean", + "title": "Whether the stream finished, no matter the error state, which is always true, or otherwise the measurement would not be included here" + }, + "is_error": { + "type": "boolean", + "title": "Whether an error occurred" + }, + "is_success": { + "type": "boolean", + "title": "Whether the measurement was a success" + }, + "stream_id": { + "type": "string", + "title": "Stream identifier" + }, + "stream_info": { + "type": "object", + "title": "Information about the TGen stream", + "required": [ + "error", + "id", + "name", + "peername", + "recvsize", + "recvstate", + "sendsize", + "sendstate", + "vertexid" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "id": { + "type": "string", + "title": "Stream numerical identifier, or 0 if the stream failed" + }, + "name": { + "type": "string", + "title": "Hostname of the TGen client" + }, + "peername": { + "type": "string", + "title": "Hostname of the TGen server" + }, + "recvsize": { + "type": "string", + "title": "Number of expected payload bytes in the response" + }, + "recvstate": { + "type": "string", + "title": "Last recorded receive state of the stream, one of RECV_{NONE,AUTHENTICATE,HEADER,MODEL,PAYLOAD,CHECKSUM,SUCCESS,ERROR}" + }, + "sendsize": { + "type": "string", + "title": "Number of expected payload bytes in the request" + }, + "sendstate": { + "type": "string", + "title": "Last recorded send state of the stream, one of SEND_{NONE,COMMAND,RESPONSE,PAYLOAD,CHECKSUM,FLUSH,SUCCESS,ERROR}" + }, + "vertexid": { + "type": "string", + "title": "Vertex identifier in the TGen model" + } + } + }, + "time_info": { + "type": "object", + "title": "Elapsed time until reaching given substeps in a measurement", + "required": [ + "created-ts", + "now-ts", + "usecs-to-checksum-recv", + "usecs-to-checksum-send", + "usecs-to-command", + "usecs-to-first-byte-recv", + "usecs-to-first-byte-send", + "usecs-to-last-byte-recv", + "usecs-to-last-byte-send", + "usecs-to-proxy-choice", + "usecs-to-proxy-init", + "usecs-to-proxy-request", + "usecs-to-proxy-response", + "usecs-to-response", + "usecs-to-socket-connect", + "usecs-to-socket-create" + ], + "properties": { + "created-ts": { + "type": "string", + "title": "Montonic system time when TGen created this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "now-ts": { + "type": "string", + "title": "Montonic system time when TGen computed elapsed microseconds for this stream, in microseconds since some arbitrary, fixed point in the past." + }, + "usecs-to-checksum-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the checksum from the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-checksum-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the checksum to the TGen server, or -1 if missing (step 11)" + }, + "usecs-to-command": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the command to the TGen server, or -1 if missing (step 7)" + }, + "usecs-to-first-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-first-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the first payload byte, or -1 if missing (step 9)" + }, + "usecs-to-last-byte-recv": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-last-byte-send": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the last payload byte, or -1 if missing (step 10)" + }, + "usecs-to-proxy-choice": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS choice from the Tor client, or -1 if missing (step 4)" + }, + "usecs-to-proxy-init": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS initialization to the Tor client, or -1 if missing (step 3)" + }, + "usecs-to-proxy-request": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has sent the SOCKS request to the Tor client, or -1 if missing (step 5)" + }, + "usecs-to-proxy-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the SOCKS response from the Tor client, or -1 if missing (step 6)" + }, + "usecs-to-response": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has received the command from the TGen server, or -1 if missing (step 8)" + }, + "usecs-to-socket-connect": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has connected to the Tor client's SOCKS port, or -1 if missing (step 2)" + }, + "usecs-to-socket-create": { + "type": "string", + "title": "Elapsed microseconds until the TGen client has opened a TCP connection to the Tor client's SOCKS port, or -1 if missing (step 1)" + } + } + }, + "transport_info": { + "type": "object", + "title": "Information about the TGen transport", + "required": [ + "error", + "fd", + "local", + "proxy", + "remote", + "state" + ], + "properties": { + "error": { + "type": "string", + "title": "Error code, or NONE if no error occurred" + }, + "fd": { + "type": "string", + "title": "File descriptor" + }, + "local": { + "type": "string", + "title": "Local host name, IP address, and TCP port" + }, + "proxy": { + "type": "string", + "title": "Proxy host name, IP address, and TCP port" + }, + "remote": { + "type": "string", + "title": "Remote host name, IP address, and TCP port" + }, + "state": { + "type": "string", + "title": "Last recorded state of the transport, one of CONNECT,INIT,CHOICE,REQUEST,AUTH_{REQUEST,RESPONSE},RESPONSE_{STATUS,TYPE,IPV4,NAMELEN,NAME},SUCCESS_{OPEN,EOF},ERROR" + } + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the measurement, obtained from the log time of the [stream-success] or [stream-error] log message, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the measurement, obtained by subtracting the largest number of elapsed microseconds in time_info from unix_ts_end, given in seconds since the epoch" + } + } + } + } + } + }, + "tor": { + "type": "object", + "title": "Metadata obtained from client-side Tor controller logs", + "required": [ + "circuits", + "streams" + ], + "properties": { + "circuits": { + "type": "object", + "title": "Information about Tor circuits, by circuit identifier, obtained from CIRC and CIRC_MINOR events, for all circuits created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor circuit", + "required": [ + "circuit_id", + "elapsed_seconds", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "build_quantile": { + "type": "number", + "title": "Circuit build time quantile, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC LAUNCHED event" + }, + "build_timeout": { + "type": "integer", + "title": "Circuit build time in milliseconds, obtained from the most recent BUILDTIMEOUT_SET event preceding the CIRC event with status LAUNCHED" + }, + "buildtime_seconds": { + "type": "number", + "title": "Build time in seconds, computed as time elapsed between CIRC LAUNCHED and CIRC BUILT events" + }, + "circuit_id": { + "type": "integer", + "title": "Circuit identifier, obtained from CIRC and CIRC_MINOR events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging CIRC and CIRC_MINOR events", + "items": { + "type": "array", + "title": "Elapsed seconds until reaching a given circuit status change", + "items": [ + { + "type": "string", + "title": "Circuit status change" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from CIRC FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from CIRC FAILED events" + }, + "filtered_out": { + "type": "boolean", + "title": "Whether this circuit has been filtered out when applying filters in `onionperf filter`." + }, + "path": { + "type": "array", + "title": "Path information", + "items": { + "type": "array", + "title": "Elapsed seconds until extending the circuit to a given relay", + "items": [ + { + "type": "string", + "pattern": "^\\$[0-9A-Z]{40}~[0-9a-zA-Z]{1,19}$", + "title": "Relay fingerprint and nickname" + }, + { + "type": "number", + "minimum": 0, + "title": "Elapsed seconds" + } + ] + } + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the circuit, obtained from the log time of the last CIRC CLOSED or CIRC FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the circuit, obtained from the log time of the CIRC LAUNCHED event, given in seconds since the epoch" + } + } + } + }, + "streams": { + "type": "object", + "title": "Information about Tor stream, by stream identifier, obtained from STREAM events, for all streams created by the Tor client", + "propertyNames": { + "pattern": "^[0-9]+$" + }, + "additionalProperties": { + "type": "object", + "title": "Information about a Tor stream", + "required": [ + "circuit_id", + "elapsed_seconds", + "stream_id", + "target", + "unix_ts_end", + "unix_ts_start" + ], + "additionalProperties": false, + "properties": { + "circuit_id": { + "title": "Circuit identifier, obtained from STREAM events" + }, + "elapsed_seconds": { + "type": "array", + "title": "Elapsed seconds until receiving and logging STREAM events", + "items": { + "type": "array", + "items": [ + { + "type": "string", + "title": "Stream purpose and STREAM event status" + }, + { + "type": "number", + "title": "Elapsed seconds" + } + ] + } + }, + "failure_reason_local": { + "type": "string", + "title": "Local failure reason, obtained from STREAM FAILED events" + }, + "failure_reason_remote": { + "type": "string", + "title": "Remote failure reason, obtained from STREAM FAILED events" + }, + "source": { + "type": "string", + "title": "Stream source IP address and TCP port, obtained from STREAM NEW or STREAM NEWRESOLVE events" + }, + "stream_id": { + "type": "integer", + "title": "Stream identifier, unique at least for the lifetime of this stream" + }, + "target": { + "type": "string", + "title": "Stream target domain name and TCP port, obtained from STREAM events", + "examples": [ + "jzxfvaupigl7hkemf4jhfi2vrruvbb7ucyiwdolkkc2hf3xlm34f3qyd.onion:8080" + ] + }, + "unix_ts_end": { + "type": "number", + "title": "Final end time of the stream, obtained from the log time of the last STREAM CLOSED or STREAM FAILED event, given in seconds since the epoch" + }, + "unix_ts_start": { + "type": "number", + "title": "Initial start time of the stream, obtained from the log time of the first STREAM NEW or STREAM NEWRESOLVE event, given in seconds since the epoch" + } + } + } + } + } + } + } + } + }, + "filters": { + "type": "object", + "title": "Filters applied by type", + "propertyNames": { + "pattern": "^[A-Za-z/]" + }, + "additionalProperties": { + "type": "array", + "title": "Filters applied of a given type", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "title": "Filter name" + }, + "filepath": { + "type": "string", + "title": "File path" + } + } + } + } + }, + "type": { + "type": "string", + "title": "Document type", + "const": "onionperf" + }, + "version": { + "type": "string", + "title": "Document version", + "pattern": "^4\\.[0-9]+$" + } + } +}

1 0

[onionperf/master] Order analysis and filter keys, authored by Karsten
by karsten＠torproject.org 18 Sep '20

18 Sep '20

commit 7b0c91f1983ab21f39f37b83d3a58db5a4d11a8f Author: Ana Custura <ana(a)netstat.org.uk> Date: Thu Sep 10 12:37:00 2020 +0100 Order analysis and filter keys, authored by Karsten --- onionperf/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onionperf/filtering.py b/onionperf/filtering.py index c008c03..7ef6168 100644 --- a/onionperf/filtering.py +++ b/onionperf/filtering.py @@ -65,10 +65,12 @@ class Filtering(object): break if not keep: tor_circuits[circuit_id]["filtered"] = True + tor_circuits[circuit_id] = dict(sorted(tor_circuit.items())) def apply_filters(self, input_path, output_dir, output_file): self.analysis = OPAnalysis.load(filename=input_path) self.filter_tor_circuits(self.analysis) self.analysis.json_db["filters"] = self.filters + self.analysis.json_db = dict(sorted(self.analysis.json_db.items())) self.analysis.save(filename=output_file, output_prefix=output_dir, sort_keys=False)

1 0