October 2018 - tor-commits - lists.torproject.org

[chutney/master] Debug: print tor versions when launching tor
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit bbf30360ee42dd2af05ff9c7b0e06ebdc6d62d05 Author: teor <teor(a)torproject.org> Date: Tue Oct 23 16:20:45 2018 +1000 Debug: print tor versions when launching tor And for unsupported torrc options. Also limit the number of unsupported torrc options that are printed. --- lib/chutney/TorNet.py | 103 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py index 8c6cb57..2aa7235 100644 --- a/lib/chutney/TorNet.py +++ b/lib/chutney/TorNet.py @@ -27,9 +27,13 @@ import chutney.Templating import chutney.Traffic _BASE_ENVIRON = None +_TOR_VERSIONS = None _TORRC_OPTIONS = None _THE_NETWORK = None +TORRC_OPTION_WARN_LIMIT = 10 +torrc_option_warn_count = 0 + # Get verbose tracebacks, so we can diagnose better. cgitb.enable(format="plain") @@ -172,6 +176,49 @@ def run_tor_gencert(cmdline, passphrase): assert empty_stderr is None return stdouterr +def get_tor_version(tor): + """Return the version of the tor binary. + Versions are cached for each unique tor path. + """ + # find the version of the current tor binary, and cache it + if tor not in _TOR_VERSIONS: + cmdline = [ + tor, + "--version", + ] + tor_version = run_tor(cmdline) + # clean it up a bit + tor_version = tor_version.strip() + tor_version = tor_version.replace("version ", "") + tor_version = tor_version.replace(").", ")") + # check we received a tor version, and nothing else + assert re.match(r'^[-+.() A-Za-z0-9]+$', tor_version) + # cache the version for this tor binary's path + _TOR_VERSIONS[tor] = tor_version + else: + tor_version = _TOR_VERSIONS[tor] + return tor_version + +def get_torrc_options(tor): + """Return the torrc options supported by the tor binary. + Options are cached for each unique tor path. + """ + # find the options the current tor binary supports, and cache them + if tor not in _TORRC_OPTIONS: + cmdline = [ + tor, + "--list-torrc-options", + ] + opts = run_tor(cmdline) + # check we received a list of options, and nothing else + assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE) + torrc_opts = opts.split() + # cache the options for this tor binary's path + _TORRC_OPTIONS[tor] = torrc_opts + else: + torrc_opts = _TORRC_OPTIONS[tor] + return torrc_opts + class Node(object): @@ -361,26 +408,12 @@ class LocalNodeBuilder(NodeBuilder): # now filter the options we're about to write, commenting out # the options that the current tor binary doesn't support tor = self._env['tor'] - # find the options the current tor binary supports, and cache them - if tor not in _TORRC_OPTIONS: - cmdline = [ - tor, - "--list-torrc-options", - ] - opts = run_tor(cmdline) - # check we received a list of options, and nothing else - assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE) - torrc_opts = opts.split() - # cache the options for this tor binary's path - _TORRC_OPTIONS[tor] = torrc_opts - else: - torrc_opts = _TORRC_OPTIONS[tor] + tor_version = get_tor_version(tor) + torrc_opts = get_torrc_options(tor) # check if each option is supported before writing it # Unsupported option values may need special handling. with open(fn_out, 'w') as f: # we need to do case-insensitive option comparison - # even if this is a static whitelist, - # so we convert to lowercase as close to the loop as possible lower_opts = [opt.lower() for opt in torrc_opts] # keep ends when splitting lines, so we can write them out # using writelines() without messing around with "\n"s @@ -391,17 +424,20 @@ class LocalNodeBuilder(NodeBuilder): if (len(sline) == 0 or sline[0] == '#' or sline.split()[0].lower() in lower_opts): - f.writelines([line]) + pass else: - # well, this could get spammy - # TODO: warn once per option per tor binary - # TODO: print tor version? - print(("The tor binary at %r does not support the " - "option in the torrc line:\n" - "%r") % (tor, line.strip())) - # we could decide to skip these lines entirely - # TODO: write tor version? - f.writelines(["# " + tor + " unsupported: " + line]) + warn_msg = (("The tor binary at {} does not support " + + "the option in the torrc line:\n{}") + .format(tor, line.strip())) + if torrc_option_warn_count < TORRC_OPTION_WARN_LIMIT: + print(warn_msg) + torrc_option_warn_count += 1 + else: + debug(warn_msg) + # always dump the full output to the torrc file + line = ("# {} version {} does not support: {}" + .format(tor, tor_version, line)) + f.writelines([line]) def _getTorrcTemplate(self): """Return the template used to write the torrc for this node.""" @@ -625,18 +661,21 @@ class LocalNodeController(NodeController): nick = self._env['nick'] datadir = self._env['dir'] corefile = "core.%s" % pid + tor_version = get_tor_version(self._env['tor']) if self.isRunning(pid): if listRunning: - print("%s is running with PID %s" % (nick, pid)) + print("{} is running with PID {}: {}" + .format(nick, pid, tor_version)) return True elif os.path.exists(os.path.join(datadir, corefile)): if listNonRunning: - print("%s seems to have crashed, and left core file %s" % ( - nick, corefile)) + print("{} seems to have crashed, and left core file {}: {}" + .format(nick, corefile, tor_version)) return False else: if listNonRunning: - print("%s is stopped" % nick) + print("{} is stopped: {}" + .format(nick, tor_version)) return False def hup(self): @@ -1180,9 +1219,13 @@ def parseArgs(): def main(): global _BASE_ENVIRON + global _TOR_VERSIONS global _TORRC_OPTIONS global _THE_NETWORK _BASE_ENVIRON = TorEnviron(chutney.Templating.Environ(**DEFAULTS)) + # _TOR_VERSIONS gets initialised on demand as a map of + # "/path/to/tor" => "Tor version ..." + _TOR_VERSIONS = dict() # _TORRC_OPTIONS gets initialised on demand as a map of # "/path/to/tor" => ["SupportedOption1", "SupportedOption2", ...] # Or it can be pre-populated as a static whitelist of options

1 0

[chutney/master] Logging: improve the formatting of nicknames and PIDs
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 7b9b6a6925fc9b2ce35d54dd1f7f68ce8043ba78 Author: teor <teor(a)torproject.org> Date: Tue Oct 23 16:35:10 2018 +1000 Logging: improve the formatting of nicknames and PIDs --- lib/chutney/TorNet.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py index 2aa7235..2d5b38e 100644 --- a/lib/chutney/TorNet.py +++ b/lib/chutney/TorNet.py @@ -518,9 +518,10 @@ class LocalNodeBuilder(NodeBuilder): '-m', str(lifetime), '-a', addr, ] - print("Creating identity key for {} with {}" + # nicknames are testNNNaa[OLD], but we want them to look tidy + print("Creating identity key for {:12} with {}" .format(self._env['nick'], cmdline[0])) - debug("Identity key path '{}', command {}" + debug("Identity key path '{}', command '{}'" .format(idfile, " ".join(cmdline))) run_tor_gencert(cmdline, passphrase) @@ -664,17 +665,18 @@ class LocalNodeController(NodeController): tor_version = get_tor_version(self._env['tor']) if self.isRunning(pid): if listRunning: - print("{} is running with PID {}: {}" + # PIDs are typically 65535 or less + print("{:12} is running with PID {:5}: {}" .format(nick, pid, tor_version)) return True elif os.path.exists(os.path.join(datadir, corefile)): if listNonRunning: - print("{} seems to have crashed, and left core file {}: {}" + print("{:12} seems to have crashed, and left core file {}: {}" .format(nick, corefile, tor_version)) return False else: if listNonRunning: - print("{} is stopped: {}" + print("{:12} is stopped: {}" .format(nick, tor_version)) return False @@ -683,11 +685,11 @@ class LocalNodeController(NodeController): pid = self.getPid() nick = self._env['nick'] if self.isRunning(pid): - print("Sending sighup to %s" % nick) + print("Sending sighup to {}".format(nick)) os.kill(pid, signal.SIGHUP) return True else: - print("%s is not running" % nick) + print("{:12} is not running".format(nick)) return False def start(self): @@ -695,7 +697,7 @@ class LocalNodeController(NodeController): already running, False if we failed.""" if self.isRunning(): - print("%s is already running" % self._env['nick']) + print("{:12} is already running".format(self._env['nick'])) return True tor_path = self._env['tor'] torrc = self._getTorrcFname() @@ -725,13 +727,14 @@ class LocalNodeController(NodeController): p.poll() if p.returncode is not None and p.returncode != 0: if self._env['poll_launch_time'] is None: - print("Couldn't launch {} command '{}': exit {}, output '{}'" + print(("Couldn't launch {:12} command '{}': " + + "exit {}, output '{}'") .format(self._env['nick'], " ".join(cmdline), p.returncode, stdouterr)) else: - print(("Couldn't poll {} command '{}' " + + print(("Couldn't poll {:12} command '{}' " + "after waiting {} seconds for launch: " + "exit {}").format(self._env['nick'], " ".join(cmdline), @@ -744,14 +747,14 @@ class LocalNodeController(NodeController): """Try to stop this node by sending it the signal 'sig'.""" pid = self.getPid() if not self.isRunning(pid): - print("%s is not running" % self._env['nick']) + print("{:12} is not running".format(self._env['nick'])) return os.kill(pid, sig) def cleanup_lockfile(self): lf = self._env['lockfile'] if not self.isRunning() and os.path.exists(lf): - debug('Removing stale lock file for {0} ...' + debug("Removing stale lock file for {} ..." .format(self._env['nick'])) os.remove(lf)

1 0

[chutney/master] Debug: exit test-network.sh with helpful text, not an exit status number
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 42279567829c6b6d398e8d9cc8ee37a9cc981f74 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 08:44:32 2018 +1000 Debug: exit test-network.sh with helpful text, not an exit status number Also handle chutney exiting with a non-zero exit status. --- tools/test-network.sh | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tools/test-network.sh b/tools/test-network.sh index c87b587..6f6eb91 100755 --- a/tools/test-network.sh +++ b/tools/test-network.sh @@ -167,7 +167,7 @@ do $ECHO "$UPDATE_YOUR_CHUTNEY" # continue processing arguments during a dry run if [ "$NETWORK_DRY_RUN" != true ]; then - exit 2 + exit 1 fi ;; esac @@ -324,12 +324,17 @@ if [ "$NETWORK_DRY_RUN" = true -o "$CHUTNEY_WARNINGS_ONLY" = true ]; then if [ "$CHUTNEY_WARNINGS_ONLY" = true ]; then "$WARNINGS" fi + $ECHO "Finished dry run" # This breaks sourcing this script: that is intentional, as the previous # behaviour only worked with bash as /bin/sh - exit + exit 0 fi -"$CHUTNEY_PATH/tools/bootstrap-network.sh" "$NETWORK_FLAVOUR" || exit 3 +if ! "$CHUTNEY_PATH/tools/bootstrap-network.sh" "$NETWORK_FLAVOUR"; then + "$WARNINGS" + $ECHO "bootstrap-network.sh failed" + exit 1 +fi # chutney starts verifying after 20 seconds, keeps on trying for 60 seconds, # and then stops immediately (by default) @@ -354,16 +359,17 @@ fi if [ "$CHUTNEY_BOOTSTRAP_TIME" -ge 0 ]; then # Chutney will try to verify for $CHUTNEY_BOOTSTRAP_TIME seconds each round n_rounds=0 - VERIFY_EXIT_STATUS=0 # Run CHUTNEY_ROUNDS verification rounds $ECHO "Running $CHUTNEY_ROUNDS verify rounds..." - while [ "$CHUTNEY_ROUNDS" -gt "$n_rounds" \ - -a "$VERIFY_EXIT_STATUS" -eq 0 ]; do - "$CHUTNEY" verify "$CHUTNEY_NETWORK" - VERIFY_EXIT_STATUS="$?" + while [ "$n_rounds" -lt "$CHUTNEY_ROUNDS" ]; do n_rounds=$((n_rounds+1)) + if ! "$CHUTNEY" verify "$CHUTNEY_NETWORK"; then + "$WARNINGS" + $ECHO "chutney verify $n_rounds/$CHUTNEY_ROUNDS failed" + exit 1 + fi + $ECHO "Completed $n_rounds/$CHUTNEY_ROUNDS verify rounds." done - $ECHO "Completed $n_rounds of $CHUTNEY_ROUNDS verify rounds." else $ECHO "Chutney network ready and running. To stop the network, use:" $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" @@ -378,14 +384,19 @@ if [ "$CHUTNEY_STOP_TIME" -ge 0 ]; then sleep "$CHUTNEY_STOP_TIME" # work around a bug/feature in make -j2 (or more) # where make hangs if any child processes are still alive - "$CHUTNEY" stop "$CHUTNEY_NETWORK" + if ! "$CHUTNEY" stop "$CHUTNEY_NETWORK"; then + "$WARNINGS" + $ECHO "chutney stop failed" + exit 1 + fi # Give tor time to exit gracefully sleep 3 - "$WARNINGS" - exit "$VERIFY_EXIT_STATUS" else $ECHO "Chutney network verified and running. To stop the network, use:" $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" "$WARNINGS" exit 0 fi + +"$WARNINGS" +exit 0

1 0

[chutney/master] Debug: when a subcommand fails, print all warnings
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 215d6a5ea9252dbae0560a205d6a57c4a0f4cfce Author: teor <teor(a)torproject.org> Date: Thu Oct 25 10:25:07 2018 +1000 Debug: when a subcommand fails, print all warnings --- tools/test-network.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/test-network.sh b/tools/test-network.sh index 6f6eb91..b8ee341 100755 --- a/tools/test-network.sh +++ b/tools/test-network.sh @@ -313,10 +313,11 @@ $ECHO "$myname: Using \$CHUTNEY_TOR: '$CHUTNEY_TOR' and \$CHUTNEY_TOR_GENCERT: ' export NETWORK_FLAVOUR=${NETWORK_FLAVOUR:-"bridges+hs-v2"} export CHUTNEY_NETWORK="$CHUTNEY_PATH/networks/$NETWORK_FLAVOUR" +WARNING_COMMAND="$CHUTNEY_PATH/tools/warnings.sh" if [ "$CHUTNEY_WARNINGS_SKIP" = true ]; then WARNINGS=true else - WARNINGS="$CHUTNEY_PATH/tools/warnings.sh" + WARNINGS="$WARNING_COMMAND" fi # And finish up if we're doing a dry run @@ -331,7 +332,8 @@ if [ "$NETWORK_DRY_RUN" = true -o "$CHUTNEY_WARNINGS_ONLY" = true ]; then fi if ! "$CHUTNEY_PATH/tools/bootstrap-network.sh" "$NETWORK_FLAVOUR"; then - "$WARNINGS" + CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ + "$WARNING_COMMAND" $ECHO "bootstrap-network.sh failed" exit 1 fi @@ -364,7 +366,9 @@ if [ "$CHUTNEY_BOOTSTRAP_TIME" -ge 0 ]; then while [ "$n_rounds" -lt "$CHUTNEY_ROUNDS" ]; do n_rounds=$((n_rounds+1)) if ! "$CHUTNEY" verify "$CHUTNEY_NETWORK"; then - "$WARNINGS" + CHUTNEY_WARNINGS_IGNORE_EXPECTED=false \ + CHUTNEY_WARNINGS_SUMMARY=false \ + "$WARNING_COMMAND" $ECHO "chutney verify $n_rounds/$CHUTNEY_ROUNDS failed" exit 1 fi @@ -385,7 +389,8 @@ if [ "$CHUTNEY_STOP_TIME" -ge 0 ]; then # work around a bug/feature in make -j2 (or more) # where make hangs if any child processes are still alive if ! "$CHUTNEY" stop "$CHUTNEY_NETWORK"; then - "$WARNINGS" + CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ + "$WARNING_COMMAND" $ECHO "chutney stop failed" exit 1 fi

1 0

[chutney/master] Debug: give better bootstrap error messages
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit ef92c76684189c4bf4049e10da0eee4fc4836009 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 09:57:28 2018 +1000 Debug: give better bootstrap error messages --- lib/chutney/TorNet.py | 3 ++- tools/bootstrap-network.sh | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py index 2d5b38e..cc86960 100644 --- a/lib/chutney/TorNet.py +++ b/lib/chutney/TorNet.py @@ -1092,7 +1092,8 @@ class Network(object): b.postConfig(network) def status(self): - statuses = [n.getController().check() for n in self._nodes] + statuses = [n.getController().check(listNonRunning=True) + for n in self._nodes] n_ok = len([x for x in statuses if x]) print("%d/%d nodes are running" % (n_ok, len(self._nodes))) return n_ok == len(self._nodes) diff --git a/tools/bootstrap-network.sh b/tools/bootstrap-network.sh index b18165e..27e8122 100755 --- a/tools/bootstrap-network.sh +++ b/tools/bootstrap-network.sh @@ -56,5 +56,10 @@ echo "$myname: bootstrapping network: $flavour" "$CHUTNEY" start "$CHUTNEY_NETWORK" sleep 1 -"$CHUTNEY" status "$CHUTNEY_NETWORK" -#echo "tail -F net/nodes/*/notice.log" +if ! "$CHUTNEY" status "$CHUTNEY_NETWORK"; then + # Try to work out why the start or status command is failing + CHUTNEY_DEBUG=1 "$CHUTNEY" start "$CHUTNEY_NETWORK" + # Wait a little longer, just in case + sleep 5 + CHUTNEY_DEBUG=1 "$CHUTNEY" status "$CHUTNEY_NETWORK" +fi

1 0

[chutney/master] Remove an unused variable
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 0d882aeb030d9c4584a37329bfa1a464ce9c9df6 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 09:58:00 2018 +1000 Remove an unused variable --- tools/bootstrap-network.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/bootstrap-network.sh b/tools/bootstrap-network.sh index 27e8122..addf991 100755 --- a/tools/bootstrap-network.sh +++ b/tools/bootstrap-network.sh @@ -34,7 +34,6 @@ if [ -d "$PWD/$CHUTNEY_DATA_DIR" ]; then export CHUTNEY_DATA_DIR="$PWD/$CHUTNEY_DATA_DIR" fi -VOTING_OFFSET=6 CHUTNEY="$CHUTNEY_PATH/chutney" myname=$(basename "$0")

1 0

[chutney/master] Consistely use 4 space indents in tools/
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 3a9e7f87c2e7e4cca8ee9624bfcc09b8a6991338 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 11:20:47 2018 +1000 Consistely use 4 space indents in tools/ Not tabs, not 2 spaces. --- tools/bootstrap-network.sh | 2 +- tools/test-network.sh | 396 ++++++++++++++++++++++----------------------- 2 files changed, 199 insertions(+), 199 deletions(-) diff --git a/tools/bootstrap-network.sh b/tools/bootstrap-network.sh index addf991..404ddb4 100755 --- a/tools/bootstrap-network.sh +++ b/tools/bootstrap-network.sh @@ -46,7 +46,7 @@ flavour=basic; [ -n "$1" ] && { flavour=$1; shift; } export CHUTNEY_NETWORK="$CHUTNEY_PATH/networks/$NETWORK_FLAVOUR" [ -e "$CHUTNEY_NETWORK" ] || \ - { echo "$myname: missing network file: $CHUTNEY_NETWORK"; exit 1; } + { echo "$myname: missing network file: $CHUTNEY_NETWORK"; exit 1; } "$CHUTNEY" stop "$CHUTNEY_NETWORK" diff --git a/tools/test-network.sh b/tools/test-network.sh index b8ee341..fabfe68 100755 --- a/tools/test-network.sh +++ b/tools/test-network.sh @@ -24,160 +24,160 @@ UPDATE_YOUR_CHUTNEY="Please update your chutney using 'git pull'." until [ -z "$1" ] do - case "$1" in - # the path to the chutney directory - --chutney-path) - export CHUTNEY_PATH="$2" - shift - ;; - --tor-path) - # the path of a tor build directory - # --tor-path overrides --tor and --tor-gencert - export TOR_DIR="$2" - shift - ;; - --tor) - # the name or path of a tor binary - export CHUTNEY_TOR="$2" - shift - ;; - --tor-gencert) - # the name or path of a tor-gencert binary - export CHUTNEY_TOR_GENCERT="$2" - shift - ;; - --debug) - export CHUTNEY_DEBUG="yes" - ;; - --flavor|--flavour|--network-flavor|--network-flavour) - export NETWORK_FLAVOUR="$2" - shift - ;; - # The amount of time chutney will wait before starting to verify - # If negative, chutney exits straight after launching the network - --start-time) - export CHUTNEY_START_TIME="$2" - shift - ;; - # The amount of time chutney will try to verify, before failing - # If negative, chutney exits without verifying - --delay|--sleep|--bootstrap-time|--time|--verify-time) - # This isn't the best name for this variable, but we kept it the same - # for backwards compatibility - export CHUTNEY_BOOTSTRAP_TIME="$2" - shift - ;; - # The amount of time chutney will wait after successfully verifying - # If negative, chutney exits without stopping - --stop-time) - export CHUTNEY_STOP_TIME="$2" - shift - ;; - # If all of the CHUTNEY_*_TIME options are positive, chutney will ask tor - # to exit when this PID exits. Set to 1 or lower to disable. - --controlling-pid) - export CHUTNEY_CONTROLLING_PID="$2" - shift - ;; - # Environmental variables used by chutney verify performance tests - # Send this many bytes per client connection (10 KBytes) - --data|--data-bytes|--data-byte|--bytes|--byte) - export CHUTNEY_DATA_BYTES="$2" - shift - ;; - # Make this many simultaneous connections per client (1) - --connections|--connection|--connection-count|--count) - export CHUTNEY_CONNECTIONS="$2" - shift - ;; - # Run this many verification rounds (1) - --rounds) - export CHUTNEY_ROUNDS="$2" - shift - ;; - # Make each client connect to each HS (0) - # 0 means a single client connects to each HS - # 1 means every client connects to every HS - --hs-multi-client|--hs-multi-clients|--hs-client|--hs-clients) - export CHUTNEY_HS_MULTI_CLIENT="$2" - shift - ;; - # The IPv4 address to bind to, defaults to 127.0.0.1 - --ipv4|--v4|-4|--ip) - export CHUTNEY_LISTEN_ADDRESS="$2" - shift - ;; - # The IPv6 address to bind to, default is not to bind to an IPv6 address - --ipv6|--v6|-6) - export CHUTNEY_LISTEN_ADDRESS_V6="$2" - shift - ;; - # The DNS server config for Tor Exits. Chutney's default is - # /etc/resolv.conf, even if tor's compile time default is different. - --dns-conf) - export CHUTNEY_DNS_CONF="$2" - shift - ;; - # Do not make any DNS queries. This is incompatible with external - # controllers that use SETCONF. - --offline) - export CHUTNEY_DNS_CONF="/dev/null" - ;; - # Use tor's compile-time default for ServerDNSResolvConfFile. - --dns-conf-default) - export CHUTNEY_DNS_CONF="" - ;; - # Warning Options - # we summarise unexpected warnings by default - # this shows all warnings per-node - --all-warnings) - export CHUTNEY_WARNINGS_IGNORE_EXPECTED=false - export CHUTNEY_WARNINGS_SUMMARY=false - ;; - # this doesn't run chutney, and only logs warnings - --only-warnings) - export CHUTNEY_WARNINGS_ONLY=true - ;; - # this skips warnings entirely - --no-warnings) - export CHUTNEY_WARNINGS_SKIP=true - ;; - # Expert options - # Code Coverage Binary - --coverage) - export USE_COVERAGE_BINARY=true - ;; - # Do Nothing (but process arguments and set environmental variables) - --dry-run) - # process arguments, but don't call any other scripts - export NETWORK_DRY_RUN=true - ;; - # The net directory, usually chutney/net - --net-dir) - export CHUTNEY_DATA_DIR="$2" - shift - ;; - # Try not to say anything (applies only to this script) - --quiet) - export ECHO=true - ;; - # Oops - *) - $ECHO "$myname: Sorry, I don't know what to do with '$1'." - $ECHO "$UPDATE_YOUR_CHUTNEY" - # continue processing arguments during a dry run - if [ "$NETWORK_DRY_RUN" != true ]; then - exit 1 - fi - ;; - esac - shift + case "$1" in + # the path to the chutney directory + --chutney-path) + export CHUTNEY_PATH="$2" + shift + ;; + --tor-path) + # the path of a tor build directory + # --tor-path overrides --tor and --tor-gencert + export TOR_DIR="$2" + shift + ;; + --tor) + # the name or path of a tor binary + export CHUTNEY_TOR="$2" + shift + ;; + --tor-gencert) + # the name or path of a tor-gencert binary + export CHUTNEY_TOR_GENCERT="$2" + shift + ;; + --debug) + export CHUTNEY_DEBUG="yes" + ;; + --flavor|--flavour|--network-flavor|--network-flavour) + export NETWORK_FLAVOUR="$2" + shift + ;; + # The amount of time chutney will wait before starting to verify + # If negative, chutney exits straight after launching the network + --start-time) + export CHUTNEY_START_TIME="$2" + shift + ;; + # The amount of time chutney will try to verify, before failing + # If negative, chutney exits without verifying + --delay|--sleep|--bootstrap-time|--time|--verify-time) + # This isn't the best name for this variable, but we kept it the same + # for backwards compatibility + export CHUTNEY_BOOTSTRAP_TIME="$2" + shift + ;; + # The amount of time chutney will wait after successfully verifying + # If negative, chutney exits without stopping + --stop-time) + export CHUTNEY_STOP_TIME="$2" + shift + ;; + # If all of the CHUTNEY_*_TIME options are positive, chutney will ask tor + # to exit when this PID exits. Set to 1 or lower to disable. + --controlling-pid) + export CHUTNEY_CONTROLLING_PID="$2" + shift + ;; + # Environmental variables used by chutney verify performance tests + # Send this many bytes per client connection (10 KBytes) + --data|--data-bytes|--data-byte|--bytes|--byte) + export CHUTNEY_DATA_BYTES="$2" + shift + ;; + # Make this many simultaneous connections per client (1) + --connections|--connection|--connection-count|--count) + export CHUTNEY_CONNECTIONS="$2" + shift + ;; + # Run this many verification rounds (1) + --rounds) + export CHUTNEY_ROUNDS="$2" + shift + ;; + # Make each client connect to each HS (0) + # 0 means a single client connects to each HS + # 1 means every client connects to every HS + --hs-multi-client|--hs-multi-clients|--hs-client|--hs-clients) + export CHUTNEY_HS_MULTI_CLIENT="$2" + shift + ;; + # The IPv4 address to bind to, defaults to 127.0.0.1 + --ipv4|--v4|-4|--ip) + export CHUTNEY_LISTEN_ADDRESS="$2" + shift + ;; + # The IPv6 address to bind to, default is not to bind to an IPv6 address + --ipv6|--v6|-6) + export CHUTNEY_LISTEN_ADDRESS_V6="$2" + shift + ;; + # The DNS server config for Tor Exits. Chutney's default is + # /etc/resolv.conf, even if tor's compile time default is different. + --dns-conf) + export CHUTNEY_DNS_CONF="$2" + shift + ;; + # Do not make any DNS queries. This is incompatible with external + # controllers that use SETCONF. + --offline) + export CHUTNEY_DNS_CONF="/dev/null" + ;; + # Use tor's compile-time default for ServerDNSResolvConfFile. + --dns-conf-default) + export CHUTNEY_DNS_CONF="" + ;; + # Warning Options + # we summarise unexpected warnings by default + # this shows all warnings per-node + --all-warnings) + export CHUTNEY_WARNINGS_IGNORE_EXPECTED=false + export CHUTNEY_WARNINGS_SUMMARY=false + ;; + # this doesn't run chutney, and only logs warnings + --only-warnings) + export CHUTNEY_WARNINGS_ONLY=true + ;; + # this skips warnings entirely + --no-warnings) + export CHUTNEY_WARNINGS_SKIP=true + ;; + # Expert options + # Code Coverage Binary + --coverage) + export USE_COVERAGE_BINARY=true + ;; + # Do Nothing (but process arguments and set environmental variables) + --dry-run) + # process arguments, but don't call any other scripts + export NETWORK_DRY_RUN=true + ;; + # The net directory, usually chutney/net + --net-dir) + export CHUTNEY_DATA_DIR="$2" + shift + ;; + # Try not to say anything (applies only to this script) + --quiet) + export ECHO=true + ;; + # Oops + *) + $ECHO "$myname: Sorry, I don't know what to do with '$1'." + $ECHO "$UPDATE_YOUR_CHUTNEY" + # continue processing arguments during a dry run + if [ "$NETWORK_DRY_RUN" != true ]; then + exit 1 + fi + ;; + esac + shift done # If the DNS server doesn't work, tor exits may reject all exit traffic, and # chutney may fail if [ "$CHUTNEY_WARNINGS_ONLY" != true ]; then - $ECHO "$myname: using CHUTNEY_DNS_CONF '$CHUTNEY_DNS_CONF'" + $ECHO "$myname: using CHUTNEY_DNS_CONF '$CHUTNEY_DNS_CONF'" fi # optional: $TOR_DIR is the tor build directory @@ -231,7 +231,7 @@ fi # make TOR_DIR and TOR_APP_DIR absolute if [ -d "$PWD/$TOR_DIR" -a -d "$PWD/$TOR_APP_DIR" -a \ - -d "$PWD/$TOR_DIR/src/tools" ]; then + -d "$PWD/$TOR_DIR/src/tools" ]; then export TOR_DIR="$PWD/$TOR_DIR" export TOR_APP_DIR="$PWD/$TOR_APP_DIR" fi @@ -247,13 +247,13 @@ if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" -o \ $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD" export CHUTNEY_PATH="$PWD" elif [ -d "`dirname \"$0\"`/.." -a \ - -x "`dirname \"$0\"`/../chutney" -a \ - -f "`dirname \"$0\"`/../chutney" ]; then + -x "`dirname \"$0\"`/../chutney" -a \ + -f "`dirname \"$0\"`/../chutney" ]; then $ECHO "$myname: \$CHUTNEY_PATH not valid, using this script's location" export CHUTNEY_PATH="`dirname \"$0\"`/.." elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \ -x "$TOR_DIR/../chutney/chutney" -a \ - -f "$TOR_DIR/../chutney/chutney" ]; then + -f "$TOR_DIR/../chutney/chutney" ]; then $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney" export CHUTNEY_PATH="$TOR_DIR/../chutney" else @@ -315,9 +315,9 @@ export CHUTNEY_NETWORK="$CHUTNEY_PATH/networks/$NETWORK_FLAVOUR" WARNING_COMMAND="$CHUTNEY_PATH/tools/warnings.sh" if [ "$CHUTNEY_WARNINGS_SKIP" = true ]; then - WARNINGS=true + WARNINGS=true else - WARNINGS="$WARNING_COMMAND" + WARNINGS="$WARNING_COMMAND" fi # And finish up if we're doing a dry run @@ -349,58 +349,58 @@ export CHUTNEY_STOP_TIME=${CHUTNEY_STOP_TIME:-0} CHUTNEY="$CHUTNEY_PATH/chutney" if [ "$CHUTNEY_START_TIME" -ge 0 ]; then - $ECHO "Waiting $CHUTNEY_START_TIME seconds for a consensus containing relays to be generated..." - sleep "$CHUTNEY_START_TIME" + $ECHO "Waiting $CHUTNEY_START_TIME seconds for a consensus containing relays to be generated..." + sleep "$CHUTNEY_START_TIME" else - $ECHO "Chutney network launched and running. To stop the network, use:" - $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" - "$WARNINGS" - exit 0 + $ECHO "Chutney network launched and running. To stop the network, use:" + $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" + "$WARNINGS" + exit 0 fi if [ "$CHUTNEY_BOOTSTRAP_TIME" -ge 0 ]; then - # Chutney will try to verify for $CHUTNEY_BOOTSTRAP_TIME seconds each round - n_rounds=0 - # Run CHUTNEY_ROUNDS verification rounds - $ECHO "Running $CHUTNEY_ROUNDS verify rounds..." - while [ "$n_rounds" -lt "$CHUTNEY_ROUNDS" ]; do - n_rounds=$((n_rounds+1)) - if ! "$CHUTNEY" verify "$CHUTNEY_NETWORK"; then - CHUTNEY_WARNINGS_IGNORE_EXPECTED=false \ - CHUTNEY_WARNINGS_SUMMARY=false \ - "$WARNING_COMMAND" - $ECHO "chutney verify $n_rounds/$CHUTNEY_ROUNDS failed" - exit 1 - fi - $ECHO "Completed $n_rounds/$CHUTNEY_ROUNDS verify rounds." - done + # Chutney will try to verify for $CHUTNEY_BOOTSTRAP_TIME seconds each round + n_rounds=0 + # Run CHUTNEY_ROUNDS verification rounds + $ECHO "Running $CHUTNEY_ROUNDS verify rounds..." + while [ "$n_rounds" -lt "$CHUTNEY_ROUNDS" ]; do + n_rounds=$((n_rounds+1)) + if ! "$CHUTNEY" verify "$CHUTNEY_NETWORK"; then + CHUTNEY_WARNINGS_IGNORE_EXPECTED=false \ + CHUTNEY_WARNINGS_SUMMARY=false \ + "$WARNING_COMMAND" + $ECHO "chutney verify $n_rounds/$CHUTNEY_ROUNDS failed" + exit 1 + fi + $ECHO "Completed $n_rounds/$CHUTNEY_ROUNDS verify rounds." + done else - $ECHO "Chutney network ready and running. To stop the network, use:" - $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" - "$WARNINGS" - exit 0 + $ECHO "Chutney network ready and running. To stop the network, use:" + $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" + "$WARNINGS" + exit 0 fi if [ "$CHUTNEY_STOP_TIME" -ge 0 ]; then - if [ "$CHUTNEY_STOP_TIME" -gt 0 ]; then - $ECHO "Waiting $CHUTNEY_STOP_TIME seconds before stopping the network..." - fi - sleep "$CHUTNEY_STOP_TIME" - # work around a bug/feature in make -j2 (or more) - # where make hangs if any child processes are still alive - if ! "$CHUTNEY" stop "$CHUTNEY_NETWORK"; then - CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ - "$WARNING_COMMAND" - $ECHO "chutney stop failed" - exit 1 - fi - # Give tor time to exit gracefully - sleep 3 + if [ "$CHUTNEY_STOP_TIME" -gt 0 ]; then + $ECHO "Waiting $CHUTNEY_STOP_TIME seconds before stopping the network..." + fi + sleep "$CHUTNEY_STOP_TIME" + # work around a bug/feature in make -j2 (or more) + # where make hangs if any child processes are still alive + if ! "$CHUTNEY" stop "$CHUTNEY_NETWORK"; then + CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ + "$WARNING_COMMAND" + $ECHO "chutney stop failed" + exit 1 + fi + # Give tor time to exit gracefully + sleep 3 else - $ECHO "Chutney network verified and running. To stop the network, use:" - $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" - "$WARNINGS" - exit 0 + $ECHO "Chutney network verified and running. To stop the network, use:" + $ECHO "$CHUTNEY stop $CHUTNEY_NETWORK" + "$WARNINGS" + exit 0 fi "$WARNINGS"

1 0

[chutney/master] Re-wrap some long lines, and fix some other wrapping
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 66bb2389764e3dc95b92f783204c1fb6dcf4a941 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 11:24:29 2018 +1000 Re-wrap some long lines, and fix some other wrapping --- tools/test-network.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/test-network.sh b/tools/test-network.sh index fabfe68..2c8f81a 100755 --- a/tools/test-network.sh +++ b/tools/test-network.sh @@ -62,8 +62,8 @@ do # The amount of time chutney will try to verify, before failing # If negative, chutney exits without verifying --delay|--sleep|--bootstrap-time|--time|--verify-time) - # This isn't the best name for this variable, but we kept it the same - # for backwards compatibility + # This isn't the best name for this variable, but we kept it the + # same for backwards compatibility export CHUTNEY_BOOTSTRAP_TIME="$2" shift ;; @@ -73,8 +73,8 @@ do export CHUTNEY_STOP_TIME="$2" shift ;; - # If all of the CHUTNEY_*_TIME options are positive, chutney will ask tor - # to exit when this PID exits. Set to 1 or lower to disable. + # If all of the CHUTNEY_*_TIME options are positive, chutney will ask + # tor to exit when this PID exits. Set to 1 or lower to disable. --controlling-pid) export CHUTNEY_CONTROLLING_PID="$2" shift @@ -107,7 +107,8 @@ do export CHUTNEY_LISTEN_ADDRESS="$2" shift ;; - # The IPv6 address to bind to, default is not to bind to an IPv6 address + # The IPv6 address to bind to, default is not to bind to an + # IPv6 address --ipv6|--v6|-6) export CHUTNEY_LISTEN_ADDRESS_V6="$2" shift @@ -260,10 +261,8 @@ if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" -o \ # TODO: work out how to package and install chutney, # so users can find it in $PATH $ECHO "$myname: missing 'chutney' in \$CHUTNEY_PATH ($CHUTNEY_PATH)" - $ECHO "$myname: Get chutney: git clone https://git.torproject.org/\ -chutney.git" - $ECHO "$myname: Set \$CHUTNEY_PATH to a non-standard location: export \ -CHUTNEY_PATH=\`pwd\`/chutney" + $ECHO "$myname: Get chutney: git clone https://git.torproject.org/chutney.git" + $ECHO "$myname: Set \$CHUTNEY_PATH to a non-standard location: export CHUTNEY_PATH=\`pwd\`/chutney" unset CHUTNEY_PATH exit 1 fi

1 0

[chutney/master] Add the missing networks/bridges+ipv6+hs-v2 network
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 8b09c95affad2ab1264efc6b0c7959bbed572997 Author: teor <teor(a)torproject.org> Date: Fri Oct 26 11:30:42 2018 +1000 Add the missing networks/bridges+ipv6+hs-v2 network It's a copy of networks/bridges+ipv6+hs, but it is explicit about using hsv2. --- networks/bridges+ipv6+hs-v2 | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/networks/bridges+ipv6+hs-v2 b/networks/bridges+ipv6+hs-v2 new file mode 100644 index 0000000..5b62b7c --- /dev/null +++ b/networks/bridges+ipv6+hs-v2 @@ -0,0 +1,23 @@ +# By default, Authorities are not configured as exits +Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl") +ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl") +Client = Node(tag="c", client=1, torrc="client.tmpl") + +BridgeAuthority = Node(tag="ba", authority=1, bridgeauthority=1, + relay=1, torrc="bridgeauthority.tmpl") +Bridge = Node(tag="br", bridge=1, relay=1, torrc="bridge.tmpl") +BridgeIPv6 = Node(tag="br", bridge=1, relay=1, ipv6_addr="[::1]", torrc="bridge-v6.tmpl") +BridgeClient = Node(tag="bc", client=1, bridgeclient=1, torrc="bridgeclient.tmpl") +HSv2 = Node(tag="h", hs=1, torrc="hs-v2.tmpl") + +# We need 5 authorities/relays/exits to ensure we can build HS connections +# We put the IPv6 bridge first to ensure that clients try IPv6 before IPv4 +# Unfortunately, this does not prevent clients falling back to IPv4 +# bridges+ipv6-min only has IPv6 bridges +NODES = Authority.getN(3) + BridgeAuthority.getN(1) + \ + ExitRelay.getN(4) + \ + BridgeIPv6.getN(1) + Bridge.getN(1) + \ + Client.getN(1) + BridgeClient.getN(1) + \ + HSv2.getN(1) + +ConfigureNodes(NODES)

1 0

[chutney/master] Debug: Print a warnings summary as the last message on failure
by teor＠torproject.org 30 Oct '18

30 Oct '18

commit 667cfe9ba3791d2a0443563452d85944f725b0e9 Author: teor <teor(a)torproject.org> Date: Thu Oct 25 12:07:32 2018 +1000 Debug: Print a warnings summary as the last message on failure --- tools/test-network.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/test-network.sh b/tools/test-network.sh index 2c8f81a..ced7455 100755 --- a/tools/test-network.sh +++ b/tools/test-network.sh @@ -333,6 +333,7 @@ fi if ! "$CHUTNEY_PATH/tools/bootstrap-network.sh" "$NETWORK_FLAVOUR"; then CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ "$WARNING_COMMAND" + "$WARNINGS" $ECHO "bootstrap-network.sh failed" exit 1 fi @@ -368,6 +369,7 @@ if [ "$CHUTNEY_BOOTSTRAP_TIME" -ge 0 ]; then CHUTNEY_WARNINGS_IGNORE_EXPECTED=false \ CHUTNEY_WARNINGS_SUMMARY=false \ "$WARNING_COMMAND" + "$WARNINGS" $ECHO "chutney verify $n_rounds/$CHUTNEY_ROUNDS failed" exit 1 fi @@ -390,6 +392,7 @@ if [ "$CHUTNEY_STOP_TIME" -ge 0 ]; then if ! "$CHUTNEY" stop "$CHUTNEY_NETWORK"; then CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \ "$WARNING_COMMAND" + "$WARNINGS" $ECHO "chutney stop failed" exit 1 fi

1 0