September 2020 - tor-commits - lists.torproject.org

[onionperf/master] Merge branch 'phw-enhancement-33432-3' into develop
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit c707674ba9b0d7931038c12bbe2d01585a88eb22 Merge: dfec0b8 b8f1e5c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 27 10:38:21 2020 +0200 Merge branch 'phw-enhancement-33432-3' into develop CHANGELOG.md | 4 ++ onionperf/measurement.py | 139 +++++++++++++++++++----------------- onionperf/model.py | 98 ++++++++++++++----------- onionperf/onionperf | 60 ++++++++++++---- onionperf/tests/test_measurement.py | 12 ++-- 5 files changed, 185 insertions(+), 128 deletions(-) diff --cc CHANGELOG.md index c31a40e,ac6897b..4b7fcb2 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -1,9 -1,11 +1,13 @@@ # Changes in version 0.7 - 2020-??-?? + - Add `onionperf measure --drop-guards` parameter to use and drop + guards after a given number of hours. Implements #33399. + - Remove the `onionperf measure --oneshot` switch and replace it with + new switches `--tgen-pause-initial`, `--tgen-pause-between`, + `--tgen-transfer-size`, and `--tgen-num-transfers ` to further + configure the generated TGen model. -# Changes in version 0.6 - 2020-??-?? +# Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and update analysis results file version to 3.0. Implements #33974. diff --cc onionperf/measurement.py index 709fbc6,d699292..f198be0 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@@ -173,7 -188,7 +188,7 @@@ def logrotate_thread_task(writables, tg class Measurement(object): - def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=0): - def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False): ++ def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=0): self.tor_bin_path = tor_bin_path self.tgen_bin_path = tgen_bin_path self.datadir_path = datadir_path @@@ -189,13 -203,11 +203,12 @@@ self.torclient_conf_file = torclient_conf_file self.torserver_conf_file = torserver_conf_file self.single_onion = single_onion + self.drop_guards_interval_hours = drop_guards_interval_hours - def run(self, do_onion=True, do_inet=True, client_tgen_listen_port=58888, client_tgen_connect_ip='0.0.0.0', client_tgen_connect_port=8080, client_tor_ctl_port=59050, client_tor_socks_port=59000, - server_tgen_listen_port=8080, server_tor_ctl_port=59051, server_tor_socks_port=59001): + def run(self, do_onion=True, do_inet=True, tgen_model=None, tgen_client_conf=None, tgen_server_conf=None): ''' - only `server_tgen_listen_port` are "public" and need to be opened on the firewall. - if `client_tgen_connect_port` != `server_tgen_listen_port`, then you should have installed a forwarding rule in the firewall. + only `tgen_server_conf.listen_port` are "public" and need to be opened on the firewall. + if `tgen_client_conf.connect_port` != `tgen_server_conf.listen_port`, then you should have installed a forwarding rule in the firewall. all ports need to be unique though, and unique among multiple onionperf instances. here are some sane defaults: diff --cc onionperf/onionperf index 641db70,6a16da2..e6aa44a --- a/onionperf/onionperf +++ b/onionperf/onionperf @@@ -195,12 -190,30 +190,36 @@@ def main() action="store", dest="tgenconnectport", default=8080) + measure_parser.add_argument('--tgen-pause-initial', + help="""the number of seconds TGen should wait before walking through its action graph""", + metavar="N", type=int, + action="store", dest="tgenpauseinitial", + default=300) + + measure_parser.add_argument('--tgen-pause-between', + help="""the number of seconds TGen should wait in between two transfers""", + metavar="N", type=int, + action="store", dest="tgenpausebetween", + default=300) + + measure_parser.add_argument('--tgen-transfer-size', + help="""the size of the file transfer that TGen will perform (e.g., '5 MiB' or '10 KiB')""", + metavar="STRING", type=str, + action="store", dest="tgentransfersize", + default="5 MiB") + + measure_parser.add_argument('--tgen-num-transfers', + help="""the number of file transfers that TGen will perform""", + metavar="N", type=int, + action="store", dest="tgennumtransfers", + default=0) + + measure_parser.add_argument('--drop-guards', + help="""Use and drop guards every N > 0 hours, or do not use guards at all if N = 0""", + metavar="N", type=type_nonnegative_integer, + action="store", dest="drop_guards_interval_hours", + default=0) + onion_or_inet_only_group = measure_parser.add_mutually_exclusive_group() onion_or_inet_only_group.add_argument('-o', '--onion-only',

1 0

[onionperf/master] Tweak #33399 patch.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit dfec0b8960ac214f63985dfc317ffce750ef5922 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 20 15:40:25 2020 +0200 Tweak #33399 patch. - Add a change log entry. - Pick are more sensible default for `drop_guards_interval_hours`, also to fix unit tests. --- CHANGELOG.md | 5 +++++ onionperf/measurement.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a86ee..c31a40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# Changes in version 0.7 - 2020-??-?? + + - Add `onionperf measure --drop-guards` parameter to use and drop + guards after a given number of hours. Implements #33399. + # Changes in version 0.6 - 2020-08-08 - Update to TGen 1.0.0, use TGenTools for parsing TGen log files, and diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 1a5f3bb..709fbc6 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -173,7 +173,7 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni class Measurement(object): - def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=None): + def __init__(self, tor_bin_path, tgen_bin_path, datadir_path, privatedir_path, nickname, oneshot, additional_client_conf=None, torclient_conf_file=None, torserver_conf_file=None, single_onion=False, drop_guards_interval_hours=0): self.tor_bin_path = tor_bin_path self.tgen_bin_path = tgen_bin_path self.datadir_path = datadir_path

1 0

[onionperf/master] Tweak --drop-guards switch.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit 9d76ca4b3847a10c9565d12cf130933a1076297b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 17:02:47 2020 +0200 Tweak --drop-guards switch. This commit tweaks the recently added --drop-guards switch as follows: - Guards are dropped right at startup and then every N hours. Otherwise we might not receive the first round of GUARD NEW/UP events. It's unclear why we don't receive those events, but finding out might be time-consuming whereas dropping guards at startup is basically free. - Right after guards are dropped, circuit build timeouts are dropped, too, if supported by the Tor version. If the Tor version does not support this, there's going to be a warning, and the control log will simply not contain BUILDTIMEOUT_SET events. Still part of the reopened tpo/metrics/onionperf#33399. --- CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++++++----- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a212842..e0253c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # Changes in version 0.7 - 2020-??-?? - Add `onionperf measure --drop-guards` parameter to use and drop - guards after a given number of hours. Implements #33399. + guards and circuit build timeouts every given number of hours, if + supported by the Tor version. Implements #33399. - Remove the `onionperf measure --oneshot` switch and replace it with new switches `--tgen-pause-initial`, `--tgen-pause-between`, `--tgen-transfer-size`, and `--tgen-num-transfers ` to further diff --git a/onionperf/monitor.py b/onionperf/monitor.py index b5e6390..886a9a0 100644 --- a/onionperf/monitor.py +++ b/onionperf/monitor.py @@ -57,21 +57,23 @@ class TorMonitor(object): interval_count = 0 if newnym_interval_seconds is not None: next_newnym = newnym_interval_seconds - if drop_guards_interval_hours > 0: - next_drop_guards = drop_guards_interval_hours * 3600 + next_drop_guards = 0 while done_ev is None or not done_ev.is_set(): # if self.filepath != '-' and os.path.exists(self.filepath): # with open(self.filepath, 'rb') as sizef: # msg = "tor-ctl-logger[port={0}] logged {1} bytes to {2}, press CTRL-C to quit".format(self.tor_ctl_port, os.fstat(sizef.fileno()).st_size, self.filepath) # logging.info(msg) + if drop_guards_interval_hours > 0 and interval_count >= next_drop_guards: + next_drop_guards += drop_guards_interval_hours * 3600 + torctl.drop_guards() + drop_timeouts_response = torctl.msg("DROPTIMEOUTS") + if not drop_timeouts_response.is_ok(): + self.__log(self.writable, "[WARNING] unrecognized command DROPTIMEOUTS in tor\n") sleep(1) interval_count += 1 if newnym_interval_seconds is not None and interval_count >= next_newnym: next_newnym += newnym_interval_seconds torctl.signal(Signal.NEWNYM) - if drop_guards_interval_hours > 0 and interval_count >= next_drop_guards: - next_drop_guards += drop_guards_interval_hours * 3600 - torctl.drop_guards() except KeyboardInterrupt: pass # the user hit ctrl+c diff --git a/onionperf/onionperf b/onionperf/onionperf index e6aa44a..d3d8e31 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -215,7 +215,7 @@ def main(): default=0) measure_parser.add_argument('--drop-guards', - help="""Use and drop guards every N > 0 hours, or do not use guards at all if N = 0""", + help="""Use and drop guards and circuit build timeouts every N > 0 hours, or do not use guards at all and never drop circuit build timeouts if N = 0""", metavar="N", type=type_nonnegative_integer, action="store", dest="drop_guards_interval_hours", default=0)

1 0

[onionperf/master] Bump version to 0.7.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit c8275b25e4afda9328634ec6be56ff46c7ee1cfe Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 22:30:12 2020 +0200 Bump version to 0.7. --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0253c5..80fb14d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 0.7 - 2020-??-?? +# Changes in version 0.7 - 2020-09-01 - Add `onionperf measure --drop-guards` parameter to use and drop guards and circuit build timeouts every given number of hours, if diff --git a/setup.py b/setup.py index 7fd7f85..d72b27a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('requirements.txt') as f: install_requires = f.readlines() setup(name='OnionPerf', - version='0.6', + version='0.7', description='A utility to monitor, measure, analyze, and visualize the performance of Tor and Onion Services', author='Rob Jansen', url='https://github.com/robgjansen/onionperf/',

1 0

[onionperf/master] Add issue number to change log.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit f496edec6fec8adc0b4e5c0454989c2bc06312e3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 27 11:09:07 2020 +0200 Add issue number to change log. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7fcb2..a212842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Remove the `onionperf measure --oneshot` switch and replace it with new switches `--tgen-pause-initial`, `--tgen-pause-between`, `--tgen-transfer-size`, and `--tgen-num-transfers ` to further - configure the generated TGen model. + configure the generated TGen model. Implemets #33432. # Changes in version 0.6 - 2020-08-08

1 0

[onionperf/master] Merge remote-tracking branch 'karsten/task-33399-2' into develop
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit 8fe84702dea4b2e4640c0817e505e6c449e7b9a8 Merge: f496ede 9d76ca4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 22:28:52 2020 +0200 Merge remote-tracking branch 'karsten/task-33399-2' into develop CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++++++----- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-)

1 0

[onionperf/develop] Tweak --drop-guards switch.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit 9d76ca4b3847a10c9565d12cf130933a1076297b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 17:02:47 2020 +0200 Tweak --drop-guards switch. This commit tweaks the recently added --drop-guards switch as follows: - Guards are dropped right at startup and then every N hours. Otherwise we might not receive the first round of GUARD NEW/UP events. It's unclear why we don't receive those events, but finding out might be time-consuming whereas dropping guards at startup is basically free. - Right after guards are dropped, circuit build timeouts are dropped, too, if supported by the Tor version. If the Tor version does not support this, there's going to be a warning, and the control log will simply not contain BUILDTIMEOUT_SET events. Still part of the reopened tpo/metrics/onionperf#33399. --- CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++++++----- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a212842..e0253c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # Changes in version 0.7 - 2020-??-?? - Add `onionperf measure --drop-guards` parameter to use and drop - guards after a given number of hours. Implements #33399. + guards and circuit build timeouts every given number of hours, if + supported by the Tor version. Implements #33399. - Remove the `onionperf measure --oneshot` switch and replace it with new switches `--tgen-pause-initial`, `--tgen-pause-between`, `--tgen-transfer-size`, and `--tgen-num-transfers ` to further diff --git a/onionperf/monitor.py b/onionperf/monitor.py index b5e6390..886a9a0 100644 --- a/onionperf/monitor.py +++ b/onionperf/monitor.py @@ -57,21 +57,23 @@ class TorMonitor(object): interval_count = 0 if newnym_interval_seconds is not None: next_newnym = newnym_interval_seconds - if drop_guards_interval_hours > 0: - next_drop_guards = drop_guards_interval_hours * 3600 + next_drop_guards = 0 while done_ev is None or not done_ev.is_set(): # if self.filepath != '-' and os.path.exists(self.filepath): # with open(self.filepath, 'rb') as sizef: # msg = "tor-ctl-logger[port={0}] logged {1} bytes to {2}, press CTRL-C to quit".format(self.tor_ctl_port, os.fstat(sizef.fileno()).st_size, self.filepath) # logging.info(msg) + if drop_guards_interval_hours > 0 and interval_count >= next_drop_guards: + next_drop_guards += drop_guards_interval_hours * 3600 + torctl.drop_guards() + drop_timeouts_response = torctl.msg("DROPTIMEOUTS") + if not drop_timeouts_response.is_ok(): + self.__log(self.writable, "[WARNING] unrecognized command DROPTIMEOUTS in tor\n") sleep(1) interval_count += 1 if newnym_interval_seconds is not None and interval_count >= next_newnym: next_newnym += newnym_interval_seconds torctl.signal(Signal.NEWNYM) - if drop_guards_interval_hours > 0 and interval_count >= next_drop_guards: - next_drop_guards += drop_guards_interval_hours * 3600 - torctl.drop_guards() except KeyboardInterrupt: pass # the user hit ctrl+c diff --git a/onionperf/onionperf b/onionperf/onionperf index e6aa44a..d3d8e31 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -215,7 +215,7 @@ def main(): default=0) measure_parser.add_argument('--drop-guards', - help="""Use and drop guards every N > 0 hours, or do not use guards at all if N = 0""", + help="""Use and drop guards and circuit build timeouts every N > 0 hours, or do not use guards at all and never drop circuit build timeouts if N = 0""", metavar="N", type=type_nonnegative_integer, action="store", dest="drop_guards_interval_hours", default=0)

1 0

[onionperf/develop] Merge remote-tracking branch 'karsten/task-33399-2' into develop
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit 8fe84702dea4b2e4640c0817e505e6c449e7b9a8 Merge: f496ede 9d76ca4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 22:28:52 2020 +0200 Merge remote-tracking branch 'karsten/task-33399-2' into develop CHANGELOG.md | 3 ++- onionperf/monitor.py | 12 +++++++----- onionperf/onionperf | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-)

1 0

[onionperf/develop] Bump version to 0.7.
by karsten＠torproject.org 01 Sep '20

01 Sep '20

commit c8275b25e4afda9328634ec6be56ff46c7ee1cfe Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 1 22:30:12 2020 +0200 Bump version to 0.7. --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0253c5..80fb14d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 0.7 - 2020-??-?? +# Changes in version 0.7 - 2020-09-01 - Add `onionperf measure --drop-guards` parameter to use and drop guards and circuit build timeouts every given number of hours, if diff --git a/setup.py b/setup.py index 7fd7f85..d72b27a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('requirements.txt') as f: install_requires = f.readlines() setup(name='OnionPerf', - version='0.6', + version='0.7', description='A utility to monitor, measure, analyze, and visualize the performance of Tor and Onion Services', author='Rob Jansen', url='https://github.com/robgjansen/onionperf/',

1 0

[tor-browser-build/master] Downgrade NSS and SQLCipher for 61.0.10
by sysrqb＠torproject.org 01 Sep '20

01 Sep '20

commit 933d4b890546abc07729e455c3074783c5da62f2 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Aug 21 20:14:00 2020 +0000 Downgrade NSS and SQLCipher for 61.0.10 We are about to switch to a previous `application-services` version which is the proper dependency for the `android-components` and `fenix` commits we'll use. However, that version requires an NSS and SQLCipher downgrade. The code that landed previously was based on a later `application-services` commit in the assumption that we'll use a tag which includes those changes by the time we would wire all our requirements for `fenix` together. --- projects/nss/config | 6 +++--- projects/sqlcipher/config | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/nss/config b/projects/nss/config index b260e77..2b5b4d3 100644 --- a/projects/nss/config +++ b/projects/nss/config @@ -2,8 +2,8 @@ filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' # The required versions for application-services can be found at the respective # commit in libs/build-all.sh -version: 3.54 -nspr_version: 4.26 +version: 3.53.1 +nspr_version: 4.25 var: container: use_container: 1 @@ -19,7 +19,7 @@ input_files: project: ninja - URL: 'https://ftp.mozilla.org/pub/security/nss/releases/NSS_[% c("version") | replace("\\.", "_") %]_RTM/src/nss-[% c("version") %]-with-nspr-[% c("nspr_version") %].tar.gz' name: nss - sha256sum: e0e81f0ff264d810f130d3cd9334722f7f883c752430483131d1ca5ac62d3f70 + sha256sum: 132ce527f3963e3ae5be3337ed4e1099e160b9402be177b5704de3ed2f5271d2 - filename: configure.patch - filename: config.patch - filename: bug_13028.patch diff --git a/projects/sqlcipher/config b/projects/sqlcipher/config index fb4ae3a..da78dcd 100644 --- a/projects/sqlcipher/config +++ b/projects/sqlcipher/config @@ -2,7 +2,7 @@ filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' # The required versions for application-services can be found at the respective # commit in libs/build-all.sh -version: 4.4.0 +version: 4.3.0 var: container: use_container: 1 @@ -18,4 +18,4 @@ input_files: project: nss - URL: 'https://github.com/sqlcipher/sqlcipher/archive/v[% c("version") %].tar.gz' name: sqlcipher - sha256sum: 0924b2ae1079717954498bda78a30de20ce2a6083076b16214a711567821d148 + sha256sum: fccb37e440ada898902b294d02cde7af9e8706b185d77ed9f6f4d5b18b4c305f

1 0