May 2020 - tor-commits - lists.torproject.org

[tor-browser-build/master] Move ticket entry in 9.5a12 changelog
by sysrqb＠torproject.org 26 May '20

26 May '20

commit 8867e44fa82162b529d8cc2cb24462db12bc1ec7 Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Tue May 26 18:05:28 2020 +0000 Move ticket entry in 9.5a12 changelog --- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index 969203c..4a0f3fa 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -28,13 +28,13 @@ Tor Browser 9.5a12 -- May 7 2020 * Bug 33630: Remove noisebridge01 default bridge * Bug 33726: Fix patch for #23247: Communicating security expectations for .onion * Bug 34017: Bump openssl version to 1.1.1g - * Bug 33698: Update "About Tor Browser" links in Tor Browser * Windows + OS X + Linux * Update Tor Launcher to 0.2.21.6 * Translations update * Bug 33576: Update pion-webrtc version to 2.2.3 * Bug 32418: Allow updates to be disabled via an enterprise policy. * Bug 34032: Use Securedrop's Official https-everywhere ruleset + * Bug 33698: Update "About Tor Browser" links in Tor Browser * Windows * Bug 29614: Use SHA-256 algorithm for Windows timestamping * Android

1 0

[tor-browser-build/master] Small Changelog fixups
by gk＠torproject.org 26 May '20

26 May '20

commit b7f2deae207b9bb42c49b67c0e575fcc41ad2c8f Author: Georg Koppen <gk(a)torproject.org> Date: Tue May 26 10:05:24 2020 +0000 Small Changelog fixups --- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index aadb432..969203c 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -14,6 +14,7 @@ Tor Browser 9.5a13 -- May 20 2020 * Build System * All Platforms * Bump Go to 1.13.11 + * Linux * Bug 34242: Fix creation of Linux containers Tor Browser 9.5a12 -- May 7 2020 @@ -174,7 +175,7 @@ Tor Browser 9.5a5 -- February 12 2020 * Bug 32470: Backport fix for bug 1590538 * Bug 32414: Make Services.search.addEngine obey FPI * Bug 32948: Make referer behavior consistent regardless of private browing mode status - * Bug 461204: Improve the random number generator for the boundaries in multipart/form-data + * Bug 22919: Improve the random number generator for the boundaries in multipart/form-data * Windows + OS X + Linux * Update Tor to 0.4.3.2-alpha * Update Tor Launcher to 0.2.21.1

1 0

[onionperf/master] Remove some unused/commented out code.
by karsten＠torproject.org 26 May '20

26 May '20

commit 8daed2f71ae2f6e689af994d9a7206d1618fbb1d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue May 26 10:14:23 2020 +0200 Remove some unused/commented out code. --- onionperf/analysis.py | 12 ------------ onionperf/onionperf | 43 ------------------------------------------- 2 files changed, 55 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index b79eac1..59d8217 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -19,18 +19,6 @@ from stem.response import ControlMessage, convert # onionperf imports from . import util -ERRORS = { 'AUTH' : 'TGEN/AUTH', - 'READ' : 'TGEN/READ', - 'STALLOUT' : 'TGEN/STALLOUT', - 'TIMEOUT' : 'TGEN/TIMEOUT', - 'PROXY' : 'TOR', - 'PROXY_CANT_ATTACH' : 'TOR/CANT_ATTACH', - 'PROXY_DESTROY' : 'TOR/DESTROY', - 'PROXY_END_TIMEOUT' : 'TOR/END/TIMEOUT', - 'PROXY_END_CONNECTREFUSED' : 'TOR/END/CONNECTREFUSED', - 'PROXY_RESOLVEFAILED' : 'TOR/RESOLVEFAILED', - 'PROXY_TIMEOUT' : 'TOR/TIMEOUT' } - class Analysis(object): def __init__(self, nickname=None, ip_address=None): diff --git a/onionperf/onionperf b/onionperf/onionperf index 1edd5a7..38ef443 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -272,49 +272,6 @@ files generated by this script will be written""", action="store_true", dest="do_simple", default=False) -# analyze_parser = sub_parser.add_parser('analyze', description=DESC_ANALYZE, help=HELP_ANALYZE) -# analyze_parser.set_defaults(func=analyze, formatter_class=argparse.RawDescriptionHelpFormatter) -# -# analyze_parser.add_argument( -# help="""The PATH to search for tgen and tor log files, which may be '-' -# for STDIN; each log file may end in '.xz' to enable -# inline xz decompression""", -# metavar="PATH", type=type_str_path_in, -# action="store", dest="searchpath") -# -# analyze_parser.add_argument('-t', '--type', -# help="""The TYPE of Analysis to perform, which may be 'tgen', 'tor', or 'all'""", -# metavar="TYPE", type=type_supported_analysis, -# action="store", dest="atype", -# default="all") -# -# analyze_parser.add_argument('-e', '--expression', -# help="""Append a regex PATTERN to the list of strings used with -# re.search to find log file names in the search path""", -# metavar="PATTERN", type=str, -# action="append", dest="patterns", -# default=[]) -# -# analyze_parser.add_argument('-m', '--multiproc', -# help="""Enable multiprocessing with N worker process, which may be '0' -# to use the number of processor cores""", -# metavar="N", type=type_nonnegative_integer, -# action="store", dest="nprocesses", -# default=1) -# -# analyze_parser.add_argument('-p', '--prefix', -# help="""A directory PATH prefix where the processed data -# files generated by this script will be written""", -# metavar="PATH", type=type_str_dir_path_out, -# action="store", dest="prefix", -# default=os.getcwd()) -# -# analyze_parser.add_argument('-n', '--nickname', -# help="""the 'SOURCE' STRING to use in .tpf torperf stats files produced by OnionPerf""", -# metavar="STRING", type=str, -# action="store", dest="nickname", -# default=None) - # visualize visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE, formatter_class=my_formatter_class)

1 0

[onionperf/master] Remove existing Tor control log visualizations.
by karsten＠torproject.org 26 May '20

26 May '20

commit 36c9120ddc52c10ed7272ffbafa43850008e83bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue May 26 10:02:45 2020 +0200 Remove existing Tor control log visualizations. These visualizations are not really useful to us, so we can as well remove them and all remaining pylab code with it. As part of this change, output files are being renamed from tgen.onionperf.viz.yyyy-mm-dd.{csv,pdf} to just onionperf.viz.yyyy-mm-dd.{csv,pdf}. Implements #34214. --- onionperf/onionperf | 20 +----- onionperf/visualization.py | 150 ++------------------------------------------- 2 files changed, 8 insertions(+), 162 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index ee28344..1edd5a7 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -334,13 +334,6 @@ files generated by this script will be written""", action="store", dest="prefix", default=None) - visualize_parser.add_argument('-f', '--format', - help="""A comma-separated LIST of color/line format strings to cycle to - matplotlib's plot command (see matplotlib.pyplot.plot)""", - metavar="LIST", type=str, - action="store", dest="lineformats", - default=util.LINEFORMATS) - # get args and call the command handler for the chosen mode if len(sys.argv) == 1: main_parser.print_help() @@ -429,25 +422,16 @@ def analyze(args): logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed") def visualize(args): - from onionperf.visualization import TGenVisualization, TorVisualization + from onionperf.visualization import TGenVisualization from onionperf.analysis import Analysis - lflist = args.lineformats.strip().split(",") - lfcycle = cycle(lflist) - tgen_viz = TGenVisualization() - tor_viz = TorVisualization() - for (path, label) in args.datasets: - nextformat = next(lfcycle) - anal = Analysis.load(filename=path) if anal is not None: - tgen_viz.add_dataset(anal, label, nextformat) - tor_viz.add_dataset(anal, label, nextformat) + tgen_viz.add_dataset(anal, label) tgen_viz.plot_all(args.prefix) - tor_viz.plot_all(args.prefix) def type_nonnegative_integer(value): i = int(value) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 51ea59e..4547da1 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -6,138 +6,25 @@ import matplotlib; matplotlib.use('Agg') # for systems without X11 from matplotlib.backends.backend_pdf import PdfPages -import pylab, numpy, time +import time from abc import abstractmethod, ABCMeta import matplotlib.pyplot as plt import pandas as pd import seaborn as sns import datetime -''' -pylab.rcParams.update({ - 'backend': 'PDF', - 'font.size': 16, - 'figure.max_num_figures' : 50, - 'figure.figsize': (6, 4.5), - 'figure.dpi': 100.0, - 'figure.subplot.left': 0.10, - 'figure.subplot.right': 0.95, - 'figure.subplot.bottom': 0.13, - 'figure.subplot.top': 0.92, - 'grid.color': '0.1', - 'axes.grid' : True, - 'axes.titlesize' : 'small', - 'axes.labelsize' : 'small', - 'axes.formatter.limits': (-4, 4), - 'xtick.labelsize' : 'small', - 'ytick.labelsize' : 'small', - 'lines.linewidth' : 2.0, - 'lines.markeredgewidth' : 0.5, - 'lines.markersize' : 10, - 'legend.fontsize' : 'x-small', - 'legend.fancybox' : False, - 'legend.shadow' : False, - 'legend.ncol' : 1.0, - 'legend.borderaxespad' : 0.5, - 'legend.numpoints' : 1, - 'legend.handletextpad' : 0.5, - 'legend.handlelength' : 1.6, - 'legend.labelspacing' : .75, - 'legend.markerscale' : 1.0, - 'ps.useafm' : True, - 'pdf.use14corefonts' : True, - 'text.usetex' : True, -}) -''' - class Visualization(object, metaclass=ABCMeta): def __init__(self): self.datasets = [] - def add_dataset(self, analysis, label, lineformat): - self.datasets.append((analysis, label, lineformat)) + def add_dataset(self, analysis, label): + self.datasets.append((analysis, label)) @abstractmethod def plot_all(self, output_prefix): pass -class TorVisualization(Visualization): - - def plot_all(self, output_prefix, relays_only=False): - self.relays_only = relays_only - if len(self.datasets) > 0: - prefix = output_prefix + '.' if output_prefix is not None else '' - ts = time.strftime("%Y-%m-%d_%H:%M:%S") - self.page = PdfPages("{0}tor.onionperf.viz.{1}.pdf".format(prefix, ts)) - self.__plot_bytes(direction="bytes_read") - self.__plot_bytes(direction="bytes_written") - self.page.close() - - def __plot_bytes(self, direction="bytes_written"): - mafig = pylab.figure() - allcdffig = pylab.figure() - eachcdffig = pylab.figure() - - for (anal, label, lineformat) in self.datasets: - tput = {} - pertput = [] - for node in anal.get_nodes(): - if self.relays_only and 'relay' not in node and 'thority' not in node: continue - d = anal.get_tor_bandwidth_summary(node, direction) - if d is None: continue - for tstr in d: - mib = d[tstr] / 1048576.0 - t = int(tstr) - if t not in tput: tput[t] = 0 - tput[t] += mib - pertput.append(mib) - - pylab.figure(mafig.number) - x = sorted(tput.keys()) - y = [tput[t] for t in x] - y_ma = movingaverage(y, 60) - pylab.scatter(x, y, s=0.1) - pylab.plot(x, y_ma, lineformat, label=label) - - pylab.figure(allcdffig.number) - x, y = getcdf(y) - pylab.plot(x, y, lineformat, label=label) - - pylab.figure(eachcdffig.number) - x, y = getcdf(pertput) - pylab.plot(x, y, lineformat, label=label) - - pylab.figure(mafig.number) - pylab.xlabel("Tick (s)") - pylab.ylabel("Throughput (MiB/s)") - pylab.xlim(xmin=0.0) - pylab.ylim(ymin=0.0) - pylab.title("60 second moving average throughput, {0}, all relays".format("write" if direction == "bytes_written" else "read")) - pylab.legend(loc="lower right") - self.page.savefig() - pylab.close() - del(mafig) - - pylab.figure(allcdffig.number) - pylab.xlabel("Throughput (MiB/s)") - pylab.ylabel("Cumulative Fraction") - pylab.title("1 second throughput, {0}, all relays".format("write" if direction == "bytes_written" else "read")) - pylab.legend(loc="lower right") - self.page.savefig() - pylab.close() - del(allcdffig) - - pylab.figure(eachcdffig.number) - # pylab.xscale('log') - pylab.xlabel("Throughput (MiB/s)") - pylab.ylabel("Cumulative Fraction") - pylab.title("1 second throughput, {0}, each relay".format("write" if direction == "bytes_written" else "read")) - pylab.legend(loc="lower right") - self.page.savefig() - pylab.close() - del(eachcdffig) - class TGenVisualization(Visualization): def plot_all(self, output_prefix): @@ -145,9 +32,9 @@ class TGenVisualization(Visualization): prefix = output_prefix + '.' if output_prefix is not None else '' ts = time.strftime("%Y-%m-%d_%H:%M:%S") self.__extract_data_frame() - self.data.to_csv("{0}tgen.onionperf.viz.{1}.csv".format(prefix, ts)) + self.data.to_csv("{0}onionperf.viz.{1}.csv".format(prefix, ts)) sns.set_context("paper") - self.page = PdfPages("{0}tgen.onionperf.viz.{1}.pdf".format(prefix, ts)) + self.page = PdfPages("{0}onionperf.viz.{1}.pdf".format(prefix, ts)) self.__plot_firstbyte_ecdf() self.__plot_firstbyte_time() self.__plot_lastbyte_ecdf() @@ -161,7 +48,7 @@ class TGenVisualization(Visualization): def __extract_data_frame(self): transfers = [] - for (analysis, label, lineformat) in self.datasets: + for (analysis, label) in self.datasets: for client in analysis.get_nodes(): tgen_transfers = analysis.get_tgen_transfers(client) for transfer_id, transfer_data in tgen_transfers.items(): @@ -321,28 +208,3 @@ class TGenVisualization(Visualization): sns.despine() self.page.savefig() plt.close() - -# helper - compute the window_size moving average over the data in interval -def movingaverage(interval, window_size): - window = numpy.ones(int(window_size)) / float(window_size) - return numpy.convolve(interval, window, 'same') - -# # helper - cumulative fraction for y axis -def cf(d): return pylab.arange(1.0, float(len(d)) + 1.0) / float(len(d)) - -# # helper - return step-based CDF x and y values -# # only show to the 99th percentile by default -def getcdf(data, shownpercentile=0.99, maxpoints=10000.0): - data.sort() - frac = cf(data) - k = len(data) / maxpoints - x, y, lasty = [], [], 0.0 - for i in range(int(round(len(data) * shownpercentile))): - if i % k > 1.0: continue - assert not numpy.isnan(data[i]) - x.append(data[i]) - y.append(lasty) - x.append(data[i]) - y.append(frac[i]) - lasty = frac[i] - return x, y

1 0

[doctor/master] Bug 34312: Swap teor with dgoulet for notifications
by gk＠torproject.org 26 May '20

26 May '20

commit 0cefbcce9fa8280a1b587eda06f013fc7311666a Author: Georg Koppen <gk(a)torproject.org> Date: Tue May 26 06:23:20 2020 +0000 Bug 34312: Swap teor with dgoulet for notifications --- fallback_directories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fallback_directories.py b/fallback_directories.py index b63fd8d..3ff2057 100755 --- a/fallback_directories.py +++ b/fallback_directories.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2016-2019, Damian Johnson and The Tor Project +# Copyright 2016-2020, Damian Johnson and The Tor Project # See LICENSE for licensing information """ @@ -17,7 +17,7 @@ import util log = util.get_logger('fallback_directories') NOTIFICATION_THRESHOLD = 25 # send notice if this percentage of fallbacks are unusable -TO_ADDRESSES = ['tor-consensus-health(a)lists.torproject.org', 'teor(a)torproject.org', 'nickm(a)torproject.org', 'gus(a)torproject.org'] +TO_ADDRESSES = ['tor-consensus-health(a)lists.torproject.org', 'dgoulet(a)torproject.org', 'nickm(a)torproject.org', 'gus(a)torproject.org'] EMAIL_SUBJECT = 'Fallback Directory Summary (%i/%i, %i%%)' SYNOPSIS = '%i/%i (%i%%) fallback directories have become slow or unresponsive...'

1 0

[onionperf/master] Remove v2 onion service support.
by karsten＠torproject.org 25 May '20

25 May '20

commit e515e6478625c3d2e5e0615410c28d7242090b6a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri May 22 12:09:11 2020 +0200 Remove v2 onion service support. This commit touches similar places in the code as c8b0fc2 which added v3 onion service support. Implements #33434. --- onionperf/measurement.py | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 79c96ff..3da7445 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -170,7 +170,6 @@ class Measurement(object): self.nickname = nickname self.threads = None self.done_event = None - self.hs_service_id = None self.hs_v3_service_id = None self.www_docroot = "{0}/htdocs".format(self.datadir_path) self.base_config = os.environ['BASETORRC'] if "BASETORRC" in os.environ else "" @@ -226,8 +225,7 @@ class Measurement(object): general_writables.append(tor_writable) server_urls = [] - if do_onion and self.hs_service_id is not None and self.hs_v3_service_id is not None: - server_urls.append("{0}.onion:{1}".format(self.hs_service_id, client_tgen_connect_port)) + if do_onion and self.hs_v3_service_id is not None: server_urls.append("{0}.onion:{1}".format(self.hs_v3_service_id, client_tgen_connect_port)) if do_inet: connect_ip = client_tgen_connect_ip if client_tgen_connect_ip != '0.0.0.0' else util.get_ip_address() @@ -277,13 +275,6 @@ class Measurement(object): finally: logging.info("Cleaning up child processes now...") - if self.hs_service_id is not None: - try: - with Controller.from_port(port=self.hs_control_port) as torctl: - torctl.authenticate() - torctl.remove_ephemeral_hidden_service(self.hs_service_id) - except: pass # this fails to authenticate if tor proc is dead - if self.hs_v3_service_id is not None: try: with Controller.from_port(port=self.hs_v3_control_port) as torctl: @@ -291,7 +282,6 @@ class Measurement(object): torctl.remove_ephemeral_hidden_service(self.hs_v3_service_id) except: pass # this fails to authenticate if tor proc is dead - # logging.disable(logging.INFO) self.done_event.set() for t in self.threads: @@ -387,21 +377,13 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory def start_onion_service(self, control_port, hs_port_mapping, - key_path, - v3=False): - if v3: - logging.info("Creating ephemeral hidden service with v3 onions...") - else: - logging.info("Creating ephemeral hidden service with v2 onions...") + key_path): + logging.info("Creating ephemeral hidden service...") with Controller.from_port(port=control_port) as torctl: torctl.authenticate() if not os.path.exists(key_path): response = torctl.create_ephemeral_hidden_service( - hs_port_mapping, - detached=True, - await_publication=True - ) if not v3 else torctl.create_ephemeral_hidden_service( hs_port_mapping, detached=True, await_publication=True, @@ -418,12 +400,8 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory await_publication=True, key_content=key_content, key_type=key_type) - if v3: - self.hs_v3_service_id = response.service_id - self.hs_v3_control_port = control_port - else: - self.hs_service_id = response.service_id - self.hs_control_port = control_port + self.hs_v3_service_id = response.service_id + self.hs_v3_control_port = control_port logging.info("Ephemeral hidden service is available at {0}.onion".format(response.service_id)) return response.service_id @@ -437,7 +415,6 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory def __start_tor(self, name, control_port, socks_port, hs_port_mapping=None): logging.info("Starting Tor {0} process with ControlPort={1}, SocksPort={2}...".format(name, control_port, socks_port)) tor_datadir = "{0}/tor-{1}".format(self.datadir_path, name) - key_path_v2 = "{0}/os_key_v2".format(self.privatedir_path) key_path_v3 = "{0}/os_key_v3".format(self.privatedir_path) if not os.path.exists(tor_datadir): os.makedirs(tor_datadir) @@ -476,8 +453,7 @@ WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nDataDirectory self.threads.append(torctl_helper) if hs_port_mapping is not None: - self.start_onion_service(control_port, hs_port_mapping, key_path_v2) - self.start_onion_service(control_port, hs_port_mapping, key_path_v3, v3=True) + self.start_onion_service(control_port, hs_port_mapping, key_path_v3) return tor_writable, torctl_writable

1 0

[onionperf/master] Remove 50 KiB and 1 MiB downloads.
by karsten＠torproject.org 25 May '20

25 May '20

commit 3252ddfaad76f6e940afd29285b5644a8ffafcee Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri May 22 12:20:04 2020 +0200 Remove 50 KiB and 1 MiB downloads. With the new partial downloads timestamps we can easily extract time-to-first-50-KiB and time-to-first-1-MiB numbers from the 5 MiB downloads. An analysis of past measurements showed that these numbers are comparable. By removing those smaller downloads we increase the total number of actually useful measurements. Total bandwidth consumption from this change increases from 14 to 140 kbps which still seems very reasonable. Implements #34023. --- onionperf/model.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/onionperf/model.py b/onionperf/model.py index 03e7084..cd44118 100644 --- a/onionperf/model.py +++ b/onionperf/model.py @@ -73,8 +73,6 @@ class TorperfModel(GeneratableTGenModel): if self.socksproxy is not None: g.node["start"]["socksproxy"] = self.socksproxy g.add_node("pause", time="5 minutes") - g.add_node("transfer50k", type="get", protocol="tcp", size="50 KiB", timeout="270 seconds", stallout="0 seconds") - g.add_node("transfer1m", type="get", protocol="tcp", size="1 MiB", timeout="270 seconds", stallout="0 seconds") g.add_node("transfer5m", type="get", protocol="tcp", size="5 MiB", timeout="270 seconds", stallout="0 seconds") g.add_edge("start", "pause") @@ -84,9 +82,7 @@ class TorperfModel(GeneratableTGenModel): g.add_edge("pause", "pause") # these are chosen with weighted probability, change edge 'weight' attributes to adjust probability - g.add_edge("pause", "transfer50k", weight="12.0") - g.add_edge("pause", "transfer1m", weight="2.0") - g.add_edge("pause", "transfer5m", weight="1.0") + g.add_edge("pause", "transfer5m") return g

1 0

[onionperf/master] Reduce timeout and stallout values.
by karsten＠torproject.org 25 May '20

25 May '20

commit 3e79f084dfa918e06695a12106672740478a4c79 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri May 22 12:14:52 2020 +0200 Reduce timeout and stallout values. The idea is to avoid overlapping measurements. With a new measurement starting every 5:00 minutes, we time out after 4:30 minutes to make "room" for the next measurement to start. We also don't use the stallout feature anymore by setting the value to 0 seconds. The stallout would have failed a measurement after not receiving new bytes for the given number of seconds. But that's also not what we want to measure. Implements #34024. --- onionperf/model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onionperf/model.py b/onionperf/model.py index a5e0787..03e7084 100644 --- a/onionperf/model.py +++ b/onionperf/model.py @@ -73,9 +73,9 @@ class TorperfModel(GeneratableTGenModel): if self.socksproxy is not None: g.node["start"]["socksproxy"] = self.socksproxy g.add_node("pause", time="5 minutes") - g.add_node("transfer50k", type="get", protocol="tcp", size="50 KiB", timeout="295 seconds", stallout="300 seconds") - g.add_node("transfer1m", type="get", protocol="tcp", size="1 MiB", timeout="1795 seconds", stallout="1800 seconds") - g.add_node("transfer5m", type="get", protocol="tcp", size="5 MiB", timeout="3595 seconds", stallout="3600 seconds") + g.add_node("transfer50k", type="get", protocol="tcp", size="50 KiB", timeout="270 seconds", stallout="0 seconds") + g.add_node("transfer1m", type="get", protocol="tcp", size="1 MiB", timeout="270 seconds", stallout="0 seconds") + g.add_node("transfer5m", type="get", protocol="tcp", size="5 MiB", timeout="270 seconds", stallout="0 seconds") g.add_edge("start", "pause")

1 0

[onionperf/master] Update documentation after removing v2 onions.
by karsten＠torproject.org 25 May '20

25 May '20

commit b42512fec5fc8263d546e10b1948a3d28f156f37 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon May 25 16:53:24 2020 +0200 Update documentation after removing v2 onions. Still part of Implements #33434. --- onionperf/docs/onionperf.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/onionperf/docs/onionperf.rst b/onionperf/docs/onionperf.rst index 471a29a..51e30a6 100644 --- a/onionperf/docs/onionperf.rst +++ b/onionperf/docs/onionperf.rst @@ -86,9 +86,7 @@ Watch out for any projects in the log. The output of the measurement should look 2019-03-05 17:57:03 1551805023.013463 [onionperf] [INFO] Starting Tor server process with ControlPort=26984, SocksPort=17674... 2019-03-05 17:57:03 1551805023.016020 [onionperf] [INFO] Logging Tor server process output to /home/ana/onionperf-data/tor-server/onionperf.tor.log 2019-03-05 17:57:18 1551805038.183236 [onionperf] [INFO] Logging Tor server control port monitor output to /home/ana/onionperf-data/tor-server/onionperf.torctl.log - 2019-03-05 17:57:21 1551805041.193610 [onionperf] [INFO] Creating ephemeral hidden service with v2 onions... - 2019-03-05 17:57:53 1551805073.332177 [onionperf] [INFO] Ephemeral hidden service is available at t7ki27c6eratpxa2.onion - 2019-03-05 17:57:53 1551805073.334130 [onionperf] [INFO] Creating ephemeral hidden service with v3 onions... + 2019-03-05 17:57:53 1551805073.334130 [onionperf] [INFO] Creating ephemeral hidden service... 2019-03-05 17:57:55 1551805075.847095 [onionperf] [INFO] Ephemeral hidden service is available at p3d2xcwjevqkiwtyejjbjxwadp5ces7v4k4hhrsheqwbbokuismkiyad.onion 2019-03-05 17:57:55 1551805075.851187 [onionperf] [INFO] Starting Tor client process with ControlPort=18843, SocksPort=18397... 2019-03-05 17:57:55 1551805075.851770 [onionperf] [INFO] Logging Tor client process output to /home/ana/onionperf-data/tor-client/onionperf.tor.log @@ -150,7 +148,7 @@ Download files using specified configuration file for the Tor server (default: Download files using specified configuration lines (default: disabled) -By default, OnionPerf downloads files using both the Internet and Onion services, using both v2 and v3 Onion addresses. +By default, OnionPerf downloads files using both the Internet and Onion services, using a v3 Onion address. It uses publicly available relays, but by specifying additional configuration files it can be configured to run on test Tor networks, or using bridges with or without pluggable transports. :: @@ -192,7 +190,7 @@ A file PATH to a TGen binary (default: looks in $PATH) Example vanilla Tor deployment ------------------------------ -The following command will download files continuously using a Tor client through both Onion service versions (v2 + v3) and via the Internet until it is stopped: +The following command will download files continuously using a Tor client through Onion service version 3 and via the Internet until it is stopped: :: onionperf --measure @@ -200,7 +198,7 @@ The following command will download files continuously using a Tor client throug Example vanilla bridge deployment --------------------------------- -The following command will download files continuously using a Tor client through both Onion service versions (v2 + v3) and via the Internet until it is stopped. +The following command will download files continuously using a Tor client through Onion service version 3 and via the Internet until it is stopped. The Tor client will always pick one of the bridges provided in this configuration to be the first hop in the circuits it builds: ::

1 0

[metrics-web/master] Include partials in download performance graphs.
by karsten＠torproject.org 25 May '20

25 May '20

commit 110cb010bd29b24902b62273b96566bf086da270 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun May 24 08:43:10 2020 +0200 Include partials in download performance graphs. As part of #34023, we're going to give up on making 50 KiB and 1 MiB downloads in the near future. In order to continue plotting time to download these file sizes we're including partial downloads of larger file sizes. Implements #30611. --- src/main/R/rserver/rserve-init.R | 8 ++- .../torproject/metrics/stats/onionperf/Main.java | 47 +++++++++++++++++- src/main/resources/web/json/metrics.json | 3 +- .../resources/web/jsps/reproducible-metrics.jsp | 5 +- src/main/resources/web/jsps/stats.jsp | 9 ++-- src/main/sql/onionperf/init-onionperf.sql | 57 ++++++++++++++++++++++ 6 files changed, 115 insertions(+), 14 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index e1074ee..3ad89ce 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -567,7 +567,8 @@ plot_relayflags <- function(start_p, end_p, flag_p, path_p) { prepare_torperf <- function(start_p = NULL, end_p = NULL, server_p = NULL, filesize_p = NULL) { - read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), + read_csv(file = paste(stats_dir, "onionperf-including-partials.csv", + sep = ""), col_types = cols( date = col_date(format = ""), filesize = col_double(), @@ -575,10 +576,7 @@ prepare_torperf <- function(start_p = NULL, end_p = NULL, server_p = NULL, server = col_character(), q1 = col_double(), md = col_double(), - q3 = col_double(), - timeouts = col_skip(), - failures = col_skip(), - requests = col_skip())) %>% + q3 = col_double())) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% filter(if (!is.null(server_p)) server == server_p else TRUE) %>% diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 56a689b..02fa08b 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -46,6 +46,9 @@ public class Main { logger.info("Starting onionperf module."); Connection connection = connectToDatabase(); importOnionPerfFiles(connection); + writeStatistics( + new File(baseDir, "stats/onionperf-including-partials.csv").toPath(), + queryOnionperfIncludingPartials(connection)); writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(), queryOnionPerf(connection)); writeStatistics(new File(baseDir, "stats/buildtimes.csv").toPath(), @@ -83,7 +86,10 @@ public class Main { + "dataperc60, dataperc70, dataperc80, dataperc90, dataperc100, " + "launch, used_at, timeout, quantile, circ_id, used_by, " + "endpointlocal, endpointproxy, endpointremote, hostnamelocal, " - + "hostnameremote, sourceaddress) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, " + + "hostnameremote, sourceaddress, partial10240, partial20480, " + + "partial51200, partial102400, partial204800, partial512000, " + + "partial1048576, partial2097152, partial5242880) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + "?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS); @@ -195,6 +201,18 @@ public class Main { truncateString(onionPerfStrings[j], 64)); } } + int[] partialBytes = new int[] { 10240, 20480, 51200, 102400, 204800, + 512000, 1048576, 2097152, 5242880 }; + for (int i = 38, j = 0; j < partialBytes.length; i++, j++) { + if (null == tr.getPartials() + || !tr.getPartials().containsKey(partialBytes[j])) { + psMeasurementsInsert.setNull(i, Types.INTEGER); + } else { + psMeasurementsInsert.setInt(i, + (int) (tr.getPartials().get(partialBytes[j]) + - tr.getStartMillis())); + } + } psMeasurementsInsert.execute(); try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) { if (rs.next()) { @@ -238,9 +256,34 @@ public class Main { return originalString; } + static List<String> queryOnionperfIncludingPartials(Connection connection) + throws SQLException { + logger.info("Querying statistics including partials from database."); + List<String> statistics = new ArrayList<>(); + statistics + .add("date,filesize,source,server,q1,md,q3"); + Statement st = connection.createStatement(); + String queryString = "SELECT date, filesize, source, server, q1, md, q3 " + + " FROM onionperf_including_partials"; + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f", + dateFormat.format(rs.getDate("date")), + rs.getInt("filesize"), + getStringFromResultSet(rs, "source"), + getStringFromResultSet(rs, "server"), + getDoubleFromResultSet(rs, "q1"), + getDoubleFromResultSet(rs, "md"), + getDoubleFromResultSet(rs, "q3"))); + } + } + return statistics; + } + static List<String> queryOnionPerf(Connection connection) throws SQLException { - logger.info("Querying statistics from database."); + logger.info("Querying timeout and failure statistics from database."); List<String> statistics = new ArrayList<>(); statistics .add("date,filesize,source,server,q1,md,q3,timeouts,failures,requests"); diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 5d4a6c7..f2b36a5 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -274,8 +274,7 @@ "id": "torperf", "title": "Time to download files over Tor", "type": "Graph", - "description": "This graph shows overall performance when downloading static files of different sizes over Tor, either from a server on the public internet or from an onion server. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", - "function": "torperf", + "description": "This graph shows overall performance when downloading static files of different sizes over Tor, either from a server on the public internet or from an onion server. Download times include complete downloads of the shown file size as well as partial downloads of larger file sizes. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", "function": "torperf", "parameters": [ "start", "end", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 922e458..0cdd873 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -694,6 +694,7 @@ Here we explain how we evaluate Torperf/OnionPerf measurement to obtain the same <li><code>DATACOMPLETE</code>: Download end time that is only set if the request succeeded.</li> <li><code>READBYTES</code>: Total number of bytes read, which indicates whether this request succeeded (if ≥ <code>FILESIZE</code>) or failed.</li> <li><code>DIDTIMEOUT</code>: 1 if the request timed out, 0 otherwise.</li> +<li><code>PARTIAL51200</code> and <code>PARTIAL1048576</code>: Time when 51200 or 1048576 bytes were read.</li> <li><code>DATAPERCx</code>: Time when x% of expected bytes were read for x = { 10, 20, 50, 100 }.</li> <li><code>BUILDTIMES</code>: Comma-separated list of times when circuit hops were built, which includes all circuits used for making measurement requests, successful or not.</li> <li><code>ENDPOINTREMOTE</code>: Hostname, IP address, and port that was used to connect to the remote server; we use this to distinguish a request to a public server (if <code>ENDPOINTREMOTE</code> is not present or does not contain <code>".onion"</code> as substring) or to an onion server.</li> @@ -703,7 +704,9 @@ Here we explain how we evaluate Torperf/OnionPerf measurement to obtain the same Each of the measurement results parsed in the previous step constitutes a single measurement. We're first interested in statistics on download times for the <a href="/torperf.html">Time to download files over Tor</a> graph. -Therefore we consider only measurements with <code>DATACOMPLETE > START</code>, for which we calculate the download time as: <code>DATACOMPLETE - START</code>. +Therefore we consider complete downloads as well as partial downloads. +For complete downloads we calculate the download time as <code>DATACOMPLETE - START</code> for measurements with <code>DATACOMPLETE > START</code>. +For partial downloads of larger file sizes we calculate the download time as <code>PARTIAL51200 - START</code> for measurements with <code>PARTIAL51200 > START</code> and <code>FILESIZE > 51200</code>; and <code>PARTIAL1048576 - START</code> for measurements with <code>PARTIAL1048576 > START</code> and <code>FILESIZE > 1048576</code>. We then compute the 25th, 50th, and 75th percentile of download times by sorting download times, determining the percentile rank, and using linear interpolation between adjacent ranks. Next we're interested in the average throughput of measurements for the <a href="/onionperf-throughput.html">Throughput</a> graph. diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 5abbee0..ad6c566 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -56,6 +56,7 @@ https://metrics.torproject.org/identifier.csv <li>October 2, 2019: Added <a href="#webstats-tb-channel">Tor Browser updates by release channel</a> graph.</li> <li>December 18, 2019: Added <a href="#bridgedb-transport">BridgeDB requests by requested transport</a> and <a href="#bridgedb-distributor">BridgeDB requests by distributor</a> graphs.</li> <li>May 24, 2020: Included version 3 onion service measurements in <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> +<li>May 24, 2020: Included partial downloads of larger file sizes in <a href="#torperf">Time to download files over Tor</a> graph.</li> </ul> </div> @@ -549,7 +550,7 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>start: First UTC date (YYYY-MM-DD) to include in the file.</li> <li>end: Last UTC date (YYYY-MM-DD) to include in the file.</li> <li>server: Either "public" for requests to a server on the public internet, or "onion" for requests to an onion server.</li> -<li>filesize: Size of the downloaded file in bytes, with pre-defined possible values: "50kb", "1mb", or "5mb".</li> +<li>filesize: Size of the completely or partially downloaded file in bytes, with pre-defined possible values: "50kb", "1mb", or "5mb".</li> </ul> <h4>Columns</h4> @@ -559,9 +560,9 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>filesize: Size of the downloaded file in bytes.</li> <li>source: Name of the OnionPerf or Torperf service performing measurements.</li> <li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to an onion server.</li> -<li>q1: First quartile of time in milliseconds until receiving the last byte.</li> -<li>md: Median of time in milliseconds until receiving the last byte.</li> -<li>q3: Third quartile of time in milliseconds until receiving the last byte.</li> +<li>q1: First quartile of time in milliseconds until receiving the number of bytes in filesize.</li> +<li>md: Median of time in milliseconds until receiving the number of bytes in filesize.</li> +<li>q3: Third quartile of time in milliseconds until receiving the number of bytes in filesize.</li> </ul> <h3>Timeouts and failures of downloading files over Tor diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index b6b226b..ebe9f77 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -17,6 +17,15 @@ CREATE TABLE IF NOT EXISTS measurements ( writebytes INTEGER, readbytes INTEGER, didtimeout BOOLEAN, + partial10240 INTEGER, + partial20480 INTEGER, + partial51200 INTEGER, + partial102400 INTEGER, + partial204800 INTEGER, + partial512000 INTEGER, + partial1048576 INTEGER, + partial2097152 INTEGER, + partial5242880 INTEGER, dataperc0 INTEGER, dataperc10 INTEGER, dataperc20 INTEGER, @@ -53,6 +62,54 @@ CREATE TABLE IF NOT EXISTS buildtimes ( CREATE TYPE server AS ENUM ('public', 'onion'); +CREATE OR REPLACE VIEW onionperf_including_partials AS +WITH measurements_including_partials AS ( + SELECT start, + filesize, + source, + endpointremote, + datacomplete + FROM measurements + UNION + SELECT start, + 51200 AS filesize, + source, + endpointremote, + partial51200 AS datacomplete + FROM measurements + WHERE filesize > 51200 + AND partial51200 IS NOT NULL + UNION + SELECT start, + 1048576 AS filesize, + source, + endpointremote, + partial1048576 AS datacomplete + FROM measurements + WHERE filesize > 1048576 + AND partial1048576 IS NOT NULL +), grouped AS ( + SELECT DATE(start) AS date, + filesize, + source, + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' + ELSE 'public' END AS server, + CASE WHEN COUNT(*) > 0 THEN + PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) + ELSE NULL END AS q + FROM measurements_including_partials + GROUP BY date, filesize, source, server +) +SELECT date, + filesize, + source, + server, + CASE WHEN q IS NULL THEN NULL ELSE TRUNC(q[1]) END AS q1, + CASE WHEN q IS NULL THEN NULL ELSE TRUNC(q[2]) END AS md, + CASE WHEN q IS NULL THEN NULL ELSE TRUNC(q[3]) END AS q3 +FROM grouped +ORDER BY date, filesize, source, server; + CREATE OR REPLACE VIEW onionperf AS SELECT date, filesize,

1 0