April 2019 - tor-commits - lists.torproject.org

[chutney/master] test-network: Try a default start time of 120 seconds
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit d40ea6d478fdde2b7a3a22e05036f6e923ed29bb Author: teor <teor(a)torproject.org> Date: Mon Apr 8 10:47:26 2019 +1000 test-network: Try a default start time of 120 seconds For Travis and other slow or loaded machines. TODO: update the README with the new default. (Conflicts with 30059.) Bugfix for 30064. --- tools/test-network-impl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test-network-impl.sh b/tools/test-network-impl.sh index 8787253..a2de002 100755 --- a/tools/test-network-impl.sh +++ b/tools/test-network-impl.sh @@ -14,7 +14,7 @@ fi # Even the fastest chutney networks take 5-10 seconds for their first consensus # and then 10 seconds after that for relays to bootstrap and upload descriptors # But chutney defaults to running a bit more slowly, so it is more reliable. -export CHUTNEY_START_TIME=${CHUTNEY_START_TIME:-60} +export CHUTNEY_START_TIME=${CHUTNEY_START_TIME:-120} export CHUTNEY_BOOTSTRAP_TIME=${CHUTNEY_BOOTSTRAP_TIME:-60} export CHUTNEY_STOP_TIME=${CHUTNEY_STOP_TIME:-0}

1 0

[chutney/master] shellcheck: Fix warnings in the chutney script
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 40fd44a8ba26144b83c0e0a8e6caff03df717ec4 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:00:45 2019 +1000 shellcheck: Fix warnings in the chutney script Part of 30065. --- chutney | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chutney b/chutney index fcd5ec5..e6fef7e 100755 --- a/chutney +++ b/chutney @@ -3,11 +3,13 @@ set -o errexit set -o nounset -export PYTHONPATH="$(dirname "${0}")/lib:$(dirname "${0}")/scripts:${PYTHONPATH-}" +PYTHONPATH="$(dirname "${0}")/lib:$(dirname "${0}")/scripts:${PYTHONPATH-}" +export PYTHONPATH # make a last-ditch effort to set CHUTNEY_PATH based on this script's location if ! test "${CHUTNEY_PATH+y}" then - export CHUTNEY_PATH="`dirname $0`" + CHUTNEY_PATH=$(dirname "$0") + export CHUTNEY_PATH fi binaries="python python3 python2"

1 0

[chutney/master] fixup! test-network: Try a default start time of 120 seconds
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 86fb60ac29ea11f16e82aaad4d165b02d1ba83b0 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 10:56:53 2019 +1000 fixup! test-network: Try a default start time of 120 seconds --- tools/test-network-impl.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/test-network-impl.sh b/tools/test-network-impl.sh index a2de002..a63fe97 100755 --- a/tools/test-network-impl.sh +++ b/tools/test-network-impl.sh @@ -11,9 +11,11 @@ fi # chutney starts verifying after CHUTNEY_START_TIME seconds, # keeps on trying for CHUTNEY_BOOTSTRAP_TIME seconds, # and then stops after CHUTNEY_STOP_TIME seconds. -# Even the fastest chutney networks take 5-10 seconds for their first consensus -# and then 10 seconds after that for relays to bootstrap and upload descriptors -# But chutney defaults to running a bit more slowly, so it is more reliable. +# Even the fastest chutney networks take 5-10 seconds for their first consensus, +# and then 10 seconds after that for relays to bootstrap and upload descriptors, +# and then 10 seconds after that for clients and onion services to bootstrap. +# But chutney defaults to running a bit more slowly, so it is more reliable, +# and we allow a bit more time, in case the machine is heavily loaded. export CHUTNEY_START_TIME=${CHUTNEY_START_TIME:-120} export CHUTNEY_BOOTSTRAP_TIME=${CHUTNEY_BOOTSTRAP_TIME:-60} export CHUTNEY_STOP_TIME=${CHUTNEY_STOP_TIME:-0}

1 0

[chutney/master] fixup! test: Add tests/unit-tests.sh
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 2a0a85c62de94e7b5d290b3bbae132e26c6f11a1 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 11:06:08 2019 +1000 fixup! test: Add tests/unit-tests.sh --- tests/unit-tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit-tests.sh b/tests/unit-tests.sh index 91a1841..5abc7a9 100755 --- a/tests/unit-tests.sh +++ b/tests/unit-tests.sh @@ -26,7 +26,7 @@ test -n "$LOG_FILE" unset CHUTNEY_DEBUG export CHUTNEY_DEBUG lib/chutney/Debug.py | tee "$LOG_FILE" -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | tr -s " " | cut -d " " -f2) +LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") test "$LOG_FILE_LINES" -eq 1 LOG_FILE=$(mktemp) @@ -35,7 +35,7 @@ test -n "$LOG_FILE" export CHUTNEY_DEBUG=1 lib/chutney/Debug.py | tee "$LOG_FILE" -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | tr -s " " | cut -d " " -f2) +LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") test "$LOG_FILE_LINES" -eq 2 unset CHUTNEY_DEBUG @@ -68,9 +68,9 @@ PYTHONPATH=$PYTHONPATH:lib lib/chutney/Traffic.py 9999 | tee "$LOG_FILE" # Traffic.py produces output with a single newline. But we don't want to get # too picky about the details: allow an extra line and a few extra chars. -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | tr -s " " | cut -d " " -f2) +LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") test "$LOG_FILE_LINES" -le 2 -LOG_FILE_CHARS=$(wc -c "$LOG_FILE" | tr -s " " | cut -d " " -f2) +LOG_FILE_CHARS=$(wc -c "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") test "$LOG_FILE_CHARS" -le 4

1 0

[chutney/master] fixup! fixup! test: Add tests/unit-tests.sh
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 65c669cc4dd5b53463553fd32a35ed533dad948e Author: teor <teor(a)torproject.org> Date: Mon Apr 8 11:50:41 2019 +1000 fixup! fixup! test: Add tests/unit-tests.sh --- tests/unit-tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit-tests.sh b/tests/unit-tests.sh index 5abc7a9..07e8b32 100755 --- a/tests/unit-tests.sh +++ b/tests/unit-tests.sh @@ -26,7 +26,7 @@ test -n "$LOG_FILE" unset CHUTNEY_DEBUG export CHUTNEY_DEBUG lib/chutney/Debug.py | tee "$LOG_FILE" -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") +LOG_FILE_LINES=$(wc -l < "$LOG_FILE") test "$LOG_FILE_LINES" -eq 1 LOG_FILE=$(mktemp) @@ -35,7 +35,7 @@ test -n "$LOG_FILE" export CHUTNEY_DEBUG=1 lib/chutney/Debug.py | tee "$LOG_FILE" -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") +LOG_FILE_LINES=$(wc -l < "$LOG_FILE") test "$LOG_FILE_LINES" -eq 2 unset CHUTNEY_DEBUG @@ -68,9 +68,9 @@ PYTHONPATH=$PYTHONPATH:lib lib/chutney/Traffic.py 9999 | tee "$LOG_FILE" # Traffic.py produces output with a single newline. But we don't want to get # too picky about the details: allow an extra line and a few extra chars. -LOG_FILE_LINES=$(wc -l "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") +LOG_FILE_LINES=$(wc -l < "$LOG_FILE") test "$LOG_FILE_LINES" -le 2 -LOG_FILE_CHARS=$(wc -c "$LOG_FILE" | sed -e "s/[^0-9]*$[0-9]*$.*/\1/") +LOG_FILE_CHARS=$(wc -c < "$LOG_FILE") test "$LOG_FILE_CHARS" -le 4

1 0

[chutney/master] shellcheck: fix warnings in the bootstrap-network.sh script
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit d6df03566a138d6934abedf94bdff4c08d64f83b Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:01:26 2019 +1000 shellcheck: fix warnings in the bootstrap-network.sh script Part of 30065. --- tools/bootstrap-network.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/bootstrap-network.sh b/tools/bootstrap-network.sh index bea8cee..9117291 100755 --- a/tools/bootstrap-network.sh +++ b/tools/bootstrap-network.sh @@ -14,12 +14,13 @@ # # Get a working chutney path -if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then +if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then # looks like a broken path: use the path to this tool instead - TOOLS_PATH=`dirname "$0"` - export CHUTNEY_PATH=`dirname "$TOOLS_PATH"` + TOOLS_PATH=$(dirname "$0") + CHUTNEY_PATH=$(dirname "$TOOLS_PATH") + export CHUTNEY_PATH fi -if [ -d "$PWD/$CHUTNEY_PATH" -a -x "$PWD/$CHUTNEY_PATH/chutney" ]; then +if [ -d "$PWD/$CHUTNEY_PATH" ] && [ -x "$PWD/$CHUTNEY_PATH/chutney" ]; then # looks like a relative path: make chutney path absolute export CHUTNEY_PATH="$PWD/$CHUTNEY_PATH" fi

1 0

[chutney/master] shellcheck: fix warnings in the warnings.sh script
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 2e605ac428b9999a212df7c1e26c5493350d93c5 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:01:59 2019 +1000 shellcheck: fix warnings in the warnings.sh script Part of 30065. --- tools/warnings.sh | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/tools/warnings.sh b/tools/warnings.sh index 4ef91ee..b8c263d 100755 --- a/tools/warnings.sh +++ b/tools/warnings.sh @@ -11,12 +11,13 @@ # CHUTNEY_WARNINGS_IGNORE_EXPECTED: set to "true" to filter expected warnings # CHUTNEY_WARNINGS_SUMMARY: set to "true" to merge warnings from all instances -if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then +if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then # looks like a broken path: use the path to this tool instead - TOOLS_PATH=`dirname "$0"` - export CHUTNEY_PATH=`dirname "$TOOLS_PATH"` + TOOLS_PATH=$(dirname "$0") + CHUTNEY_PATH=$(dirname "$TOOLS_PATH") + export CHUTNEY_PATH fi -if [ -d "$PWD/$CHUTNEY_PATH" -a -x "$PWD/$CHUTNEY_PATH/chutney" ]; then +if [ -d "$PWD/$CHUTNEY_PATH" ] && [ -x "$PWD/$CHUTNEY_PATH/chutney" ]; then # looks like a relative path: make chutney path absolute export CHUTNEY_PATH="$PWD/$CHUTNEY_PATH" fi @@ -33,42 +34,45 @@ fi show_warnings() { # Work out the file and filter settings + LOGS=$(mktemp) if [ "$CHUTNEY_WARNINGS_SUMMARY" = true ]; then - FILE="$1/*/$LOG_FILE" + cat "$1"/*/"$LOG_FILE" > "$LOGS" else - FILE="$1/$LOG_FILE" + cat "$1/$LOG_FILE" > "$LOGS" fi - if [ "$CHUTNEY_WARNINGS_IGNORE_EXPECTED" = true -a \ - -e "$IGNORE_FILE" ]; then - CAT="grep -v -f" + FILTERED_LOGS=$(mktemp) + if [ "$CHUTNEY_WARNINGS_IGNORE_EXPECTED" = true ] && \ + [ -e "$IGNORE_FILE" ]; then + grep -v -f "$IGNORE_FILE" "$LOGS" | $SED_E "$FILTER" > "$FILTERED_LOGS" else - CAT=cat + $SED_E "$FILTER" "$LOGS" > "$FILTERED_LOGS" IGNORE_FILE= fi # Silence any messages if we are in summary mode, and there are no warnings # must be kept in sync with the filter commands below - if [ `$CAT $IGNORE_FILE $FILE | $SED_E "$FILTER" | wc -c` -eq 0 -a \ - "$CHUTNEY_WARNINGS_SUMMARY" = true ]; then - ECHO_Q=true - ECHO_A=true + if [ "$CHUTNEY_WARNINGS_SUMMARY" = true ] && \ + [ "$(wc -c < "$FILTERED_LOGS")" -eq 0 ]; \ + then + ECHO_Q="true" + ECHO_A="true" else # if there is output, always echo the detail message - ECHO_A=echo + ECHO_A="echo" fi # Give context to the warnings we're about to display if [ "$CHUTNEY_WARNINGS_SUMMARY" = true ]; then - $ECHO_Q "${GREEN}Summary: `basename $1`${NC}" + $ECHO_Q "${GREEN}Summary: $(basename "$1")${NC}" else - $ECHO_Q "${GREEN}Node: `basename $1`${NC}" + $ECHO_Q "${GREEN}Node: $(basename "$1")${NC}" fi - if [ "$CHUTNEY_WARNINGS_IGNORE_EXPECTED" = true -a \ - -e "$IGNORE_FILE" ]; then - PERMANENT_DIR=`readlink -n "$1" || echo "$1"` + if [ "$CHUTNEY_WARNINGS_IGNORE_EXPECTED" = true ] && \ + [ -e "$IGNORE_FILE" ]; then + PERMANENT_DIR=$(readlink -n "$1" || echo "$1") $ECHO_A "${GREEN}Detail: chutney/tools/warnings.sh $PERMANENT_DIR${NC}" fi # Display the warnings, after filtering and counting occurrences # must be kept in sync with the filter commands above - $CAT $IGNORE_FILE $FILE | $SED_E "$FILTER" | sort | uniq -c | \ + sort "$FILTERED_LOGS" | uniq -c | \ sed -e 's/^\s*//' -e "s/ *$[0-9][0-9]*$ *$.*$/${YELLOW}Warning:${NC} \2${YELLOW} Number: \1${NC}/" if [ "$CHUTNEY_WARNINGS_SUMMARY" != true ]; then $ECHO_Q "" @@ -86,7 +90,7 @@ if [ -t 1 ]; then YELLOW=$(tput setaf 3) GREEN=$(tput setaf 2) fi -CHUTNEY="$CHUTNEY_PATH/chutney" + NAME=$(basename "$0") DEST="$CHUTNEY_DATA_DIR/nodes" LOG_FILE=info.log

1 0

[chutney/master] tests: Add a shellcheck test
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit ef79e2143ee426eee74d322dd4140d7c076af221 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:23:48 2019 +1000 tests: Add a shellcheck test Part of 30065. --- tests/shellcheck-tests.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/shellcheck-tests.sh b/tests/shellcheck-tests.sh new file mode 100755 index 0000000..02afbaa --- /dev/null +++ b/tests/shellcheck-tests.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Exit on errors +set -e +# Verbose mode +set -v + + +# Output is prefixed with the name of the script +myname=$(basename "$0") + +echo "$myname: finding chutney directory" +TEST_DIR=$(dirname "$0") +CHUTNEY_DIR=$(dirname "$TEST_DIR") + +echo "$myname: changing to chutney directory" +cd "$CHUTNEY_DIR" + + +echo "$myname: running shellcheck tests" + +shellcheck chutney +find . -name "*.sh" -exec shellcheck {} +

1 0

[chutney/master] shellcheck: fix warnings in the hsaddress.sh script
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 899d385c700a5de3243271d0885424abf7a65204 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:02:34 2019 +1000 shellcheck: fix warnings in the hsaddress.sh script Part of 30065. --- tools/hsaddress.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/hsaddress.sh b/tools/hsaddress.sh index 00645d7..bc1d123 100755 --- a/tools/hsaddress.sh +++ b/tools/hsaddress.sh @@ -7,12 +7,13 @@ # Examples: tools/hsaddress.sh # tools/hsaddress.sh 025h -if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then +if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then # looks like a broken path: use the path to this tool instead - TOOLS_PATH=`dirname "$0"` - export CHUTNEY_PATH=`dirname "$TOOLS_PATH"` + TOOLS_PATH=$(dirname "$0") + CHUTNEY_PATH=$(dirname "$TOOLS_PATH") + export CHUTNEY_PATH fi -if [ -d "$PWD/$CHUTNEY_PATH" -a -x "$PWD/$CHUTNEY_PATH/chutney" ]; then +if [ -d "$PWD/$CHUTNEY_PATH" ] && [ -x "$PWD/$CHUTNEY_PATH/chutney" ]; then # looks like a relative path: make chutney path absolute export CHUTNEY_PATH="$PWD/$CHUTNEY_PATH" fi @@ -48,7 +49,7 @@ then do FILE="${dir}/$TARGET" [ -e "$FILE" ] || continue - echo -n "Node `basename ${dir}`: " + echo "Node $(basename "$dir"): " | tr -d "\n" show_address "$FILE" done elif [ $# -eq 1 ];

1 0

[chutney/master] Travis: test shellcheck as part of Travis
by teor＠torproject.org 12 Apr '19

12 Apr '19

commit 9ca5257386bc8f7e65a685c4e90817bf17d45183 Author: teor <teor(a)torproject.org> Date: Mon Apr 8 12:03:07 2019 +1000 Travis: test shellcheck as part of Travis Closes 30065. --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8a140dd..fd04d7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.2.9.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor env: TOR="0.2.9-nightly" - addons: @@ -43,6 +44,7 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.3.4.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor env: TOR="0.3.4-nightly" ## The current stable release is listed at the end of the file, because @@ -53,6 +55,7 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.3.5.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor env: TOR="0.3.5-nightly" ## The experimental deb line needs to be updated when 0.4.0 becomes stable @@ -62,6 +65,7 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-experimental-0.4.0.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor env: TOR="0.4.0-nightly" - addons: @@ -70,6 +74,7 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-master-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor env: TOR="master-nightly" ## Test all supported python releases @@ -148,10 +153,12 @@ addons: - sourceline: 'deb https://deb.torproject.org/torproject.org xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: + - shellcheck - tor ## (macOS only) homebrew: packages: + - shellcheck - tor install: @@ -160,9 +167,11 @@ install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then dpkg-query --show; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi - python --version + - shellcheck --version - tor --version script: + - tests/shellcheck-tests.sh - tools/test-network.sh --allow-failures 1 after_failure:

1 0