August 2020 - tor-commits - lists.torproject.org

[tor/maint-0.4.3] Improve comments in .gitlab-ci.yml
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit aeafb7f44f07136101c06ea2986b295ef9915c9c Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:20:58 2020 -0400 Improve comments in .gitlab-ci.yml --- .gitlab-ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c77e78764d..176c71f8f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,12 +58,18 @@ variables: <<: *artifacts-template variables: DEBIAN_FRONTEND: "noninteractive" + # TODO: Using "cache" in this way speeds up our downloads. It would be + # even better, though, to start with a pre-upgraded debian image. + # + # TODO: Will we have to do this differently once we have more than one + # debian version that we're using? cache: key: apt paths: - apt-cache before_script: - *apt-template + # Install patches unconditionally. - apt-get install automake build-essential @@ -77,20 +83,28 @@ variables: pkg-config python3 zlib1g-dev + # Install patches that we only need for some use cases. - if [ "$ASCIIDOC" = yes ]; then apt-get install asciidoc xmlto; fi - if [ "$DOXYGEN" = yes ]; then apt-get install doxygen; fi - if [ "$STEM" = yes ]; then apt-get install timelimit; fi - if [ "$CC" = clang ]; then apt-get install clang; fi + # TODO: This next line should not be debian-only. - if [ "$STEM" = yes ]; then git clone --depth 1 https://git.torproject.org/stem.git ; export STEM_PATH="$(pwd)/stem"; fi + # TODO: This next line should not be debian-only. - if [ "$CHUTNEY" = yes ]; then git clone --depth 1 https://git.torproject.org/chutney.git ; export CHUTNEY_PATH="$(pwd)/chutney"; fi +# Minmal check on debian: just make, make check. +# debian-minimal: image: debian:stable <<: *debian-template script: - ./scripts/ci/ci-driver.sh - +##### +# Run "make check" with a hardened clang on debian stable. This takes +# care of a hardening check, and a compile-with-clang check. +# # TODO: This will be faster once we merge #40098 and #40099. debian-hardened: image: debian:testing @@ -101,6 +115,8 @@ debian-hardened: script: - ./scripts/ci/ci-driver.sh +##### +# Distcheck on debian stable debian-distcheck: image: debian:stable <<: *debian-template @@ -110,6 +126,8 @@ debian-distcheck: script: - ./scripts/ci/ci-driver.sh +##### +# Documentation tests on debian stable: doxygen and asciidoc. debian-docs: image: debian:stable <<: *debian-template @@ -121,6 +139,13 @@ debian-docs: script: - ./scripts/ci/ci-driver.sh +##### +# Integration tests on debian stable: chutney and stem. +# +# TODO: It would be cool if this target didn't have to re-build tor, and +# could instead re-use Tor from debian-minimal. That can be done +# with the 'artifacts' mechanism, in theory, but it would be good to +# avoid having to have a system with hundreds of artifacts. debian-integration: image: debian:stable <<: *debian-template

1 0

[tor/maint-0.4.3] Merge branch 'maint-0.4.2' into maint-0.4.3
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 34919368cebb1e635d10694d8ec42911db909130 Merge: 309eafbc3c 07f0210a05 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.4.2' into maint-0.4.3 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0

[tor/maint-0.4.3] CI: Turn on stem with 044 and later.
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 41a7ab96e578e58f277b34f5735d35a2a168da09 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:13:18 2020 -0400 CI: Turn on stem with 044 and later. --- .gitlab-ci.yml | 7 +++---- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ed615687b..c77e78764d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ variables: paths: - artifacts/ +# This template .apt-template: &apt-template | export LC_ALL=C.UTF-8 echo Etc/UTC > /etc/timezone @@ -90,9 +91,7 @@ debian-minimal: - ./scripts/ci/ci-driver.sh -# TODO: This one just takes too long to finish right now! -# Maybe we need to divide the call to ./src/test/test into a few segments, -# that all end in similar amount of time? +# TODO: This will be faster once we merge #40098 and #40099. debian-hardened: image: debian:testing <<: *debian-template @@ -128,6 +127,6 @@ debian-integration: variables: CHECK: "no" CHUTNEY: "yes" - # STEM: "yes" -- currently failing on <044. + STEM: "yes" script: - ./scripts/ci/ci-driver.sh diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh index 6faffd7b40..527bba82e5 100755 --- a/scripts/ci/ci-driver.sh +++ b/scripts/ci/ci-driver.sh @@ -267,18 +267,26 @@ TOR_VERSION=$(grep -m 1 AC_INIT configure.ac | sed -e 's/.*\[//; s/\].*//;') # Use variables like these when we need to behave differently depending on # Tor version. Only create the variables we need. TOR_VER_AT_LEAST_043=no +TOR_VER_AT_LEAST_044=no # These are the currently supported Tor versions; no need to work with anything # ancient in this script. case "$TOR_VERSION" in 0.3.*) TOR_VER_AT_LEAST_043=no + TOR_VER_AT_LEAST_044=no ;; 0.4.[012].*) TOR_VER_AT_LEAST_043=no + TOR_VER_AT_LEAST_044=no + ;; + 0.4.3.*) + TOR_VER_AT_LEAST_043=yes + TOR_VER_AT_LEAST_044=no ;; *) TOR_VER_AT_LEAST_043=yes + TOR_VER_AT_LEAST_044=yes ;; esac @@ -427,21 +435,25 @@ fi if [[ "${STEM}" = "yes" ]]; then start_section "Stem" - # XXXX This shold probably be part some test-stem make target. - if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ - python3 "${STEM_PATH}/run_tests.py" \ - --tor src/app/tor \ - --integ --test control.controller \ - --test control.base_controller \ - --test process \ - --log TRACE \ - --log-file stem.log ; then - hooray "Stem tests have succeeded" - else + if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then + # XXXX This shold probably be part some test-stem make target. + if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ + python3 "${STEM_PATH}/run_tests.py" \ + --tor src/app/tor \ + --integ --test control.controller \ + --test control.base_controller \ + --test process \ + --log TRACE \ + --log-file stem.log ; then + hooray "Stem tests have succeeded" + else error "Stem output:" runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log runcmd grep -v "SocketClosed" stem.log | tail -1000 FAILED_TESTS="${FAILED_TESTS} stem" + fi + else + skipping "Stem: broken with <= 0.4.3. See bug tor#40077" fi end_section "Stem" fi

1 0

[tor/master] CI: Turn on stem with 044 and later.
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 41a7ab96e578e58f277b34f5735d35a2a168da09 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:13:18 2020 -0400 CI: Turn on stem with 044 and later. --- .gitlab-ci.yml | 7 +++---- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ed615687b..c77e78764d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ variables: paths: - artifacts/ +# This template .apt-template: &apt-template | export LC_ALL=C.UTF-8 echo Etc/UTC > /etc/timezone @@ -90,9 +91,7 @@ debian-minimal: - ./scripts/ci/ci-driver.sh -# TODO: This one just takes too long to finish right now! -# Maybe we need to divide the call to ./src/test/test into a few segments, -# that all end in similar amount of time? +# TODO: This will be faster once we merge #40098 and #40099. debian-hardened: image: debian:testing <<: *debian-template @@ -128,6 +127,6 @@ debian-integration: variables: CHECK: "no" CHUTNEY: "yes" - # STEM: "yes" -- currently failing on <044. + STEM: "yes" script: - ./scripts/ci/ci-driver.sh diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh index 6faffd7b40..527bba82e5 100755 --- a/scripts/ci/ci-driver.sh +++ b/scripts/ci/ci-driver.sh @@ -267,18 +267,26 @@ TOR_VERSION=$(grep -m 1 AC_INIT configure.ac | sed -e 's/.*\[//; s/\].*//;') # Use variables like these when we need to behave differently depending on # Tor version. Only create the variables we need. TOR_VER_AT_LEAST_043=no +TOR_VER_AT_LEAST_044=no # These are the currently supported Tor versions; no need to work with anything # ancient in this script. case "$TOR_VERSION" in 0.3.*) TOR_VER_AT_LEAST_043=no + TOR_VER_AT_LEAST_044=no ;; 0.4.[012].*) TOR_VER_AT_LEAST_043=no + TOR_VER_AT_LEAST_044=no + ;; + 0.4.3.*) + TOR_VER_AT_LEAST_043=yes + TOR_VER_AT_LEAST_044=no ;; *) TOR_VER_AT_LEAST_043=yes + TOR_VER_AT_LEAST_044=yes ;; esac @@ -427,21 +435,25 @@ fi if [[ "${STEM}" = "yes" ]]; then start_section "Stem" - # XXXX This shold probably be part some test-stem make target. - if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ - python3 "${STEM_PATH}/run_tests.py" \ - --tor src/app/tor \ - --integ --test control.controller \ - --test control.base_controller \ - --test process \ - --log TRACE \ - --log-file stem.log ; then - hooray "Stem tests have succeeded" - else + if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then + # XXXX This shold probably be part some test-stem make target. + if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ + python3 "${STEM_PATH}/run_tests.py" \ + --tor src/app/tor \ + --integ --test control.controller \ + --test control.base_controller \ + --test process \ + --log TRACE \ + --log-file stem.log ; then + hooray "Stem tests have succeeded" + else error "Stem output:" runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log runcmd grep -v "SocketClosed" stem.log | tail -1000 FAILED_TESTS="${FAILED_TESTS} stem" + fi + else + skipping "Stem: broken with <= 0.4.3. See bug tor#40077" fi end_section "Stem" fi

1 0

[tor/maint-0.4.4] Merge branch 'maint-0.3.5' into maint-0.4.2
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 07f0210a0525423f8cbbb41d00df0aa2d40fedf5 Merge: 02bb719a94 aeafb7f44f Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.3.5' into maint-0.4.2 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0

[tor/maint-0.4.4] Merge branch 'maint-0.4.3' into maint-0.4.4
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 9d77ed7be2da18bef3ab1258f277c527347a94a7 Merge: f98b1988ec 34919368ce Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.4.3' into maint-0.4.4 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0

[tor/maint-0.4.4] Improve comments in .gitlab-ci.yml
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit aeafb7f44f07136101c06ea2986b295ef9915c9c Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:20:58 2020 -0400 Improve comments in .gitlab-ci.yml --- .gitlab-ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c77e78764d..176c71f8f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,12 +58,18 @@ variables: <<: *artifacts-template variables: DEBIAN_FRONTEND: "noninteractive" + # TODO: Using "cache" in this way speeds up our downloads. It would be + # even better, though, to start with a pre-upgraded debian image. + # + # TODO: Will we have to do this differently once we have more than one + # debian version that we're using? cache: key: apt paths: - apt-cache before_script: - *apt-template + # Install patches unconditionally. - apt-get install automake build-essential @@ -77,20 +83,28 @@ variables: pkg-config python3 zlib1g-dev + # Install patches that we only need for some use cases. - if [ "$ASCIIDOC" = yes ]; then apt-get install asciidoc xmlto; fi - if [ "$DOXYGEN" = yes ]; then apt-get install doxygen; fi - if [ "$STEM" = yes ]; then apt-get install timelimit; fi - if [ "$CC" = clang ]; then apt-get install clang; fi + # TODO: This next line should not be debian-only. - if [ "$STEM" = yes ]; then git clone --depth 1 https://git.torproject.org/stem.git ; export STEM_PATH="$(pwd)/stem"; fi + # TODO: This next line should not be debian-only. - if [ "$CHUTNEY" = yes ]; then git clone --depth 1 https://git.torproject.org/chutney.git ; export CHUTNEY_PATH="$(pwd)/chutney"; fi +# Minmal check on debian: just make, make check. +# debian-minimal: image: debian:stable <<: *debian-template script: - ./scripts/ci/ci-driver.sh - +##### +# Run "make check" with a hardened clang on debian stable. This takes +# care of a hardening check, and a compile-with-clang check. +# # TODO: This will be faster once we merge #40098 and #40099. debian-hardened: image: debian:testing @@ -101,6 +115,8 @@ debian-hardened: script: - ./scripts/ci/ci-driver.sh +##### +# Distcheck on debian stable debian-distcheck: image: debian:stable <<: *debian-template @@ -110,6 +126,8 @@ debian-distcheck: script: - ./scripts/ci/ci-driver.sh +##### +# Documentation tests on debian stable: doxygen and asciidoc. debian-docs: image: debian:stable <<: *debian-template @@ -121,6 +139,13 @@ debian-docs: script: - ./scripts/ci/ci-driver.sh +##### +# Integration tests on debian stable: chutney and stem. +# +# TODO: It would be cool if this target didn't have to re-build tor, and +# could instead re-use Tor from debian-minimal. That can be done +# with the 'artifacts' mechanism, in theory, but it would be good to +# avoid having to have a system with hundreds of artifacts. debian-integration: image: debian:stable <<: *debian-template

1 0

[tor/maint-0.4.4] Merge branch 'maint-0.4.2' into maint-0.4.3
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 34919368cebb1e635d10694d8ec42911db909130 Merge: 309eafbc3c 07f0210a05 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.4.2' into maint-0.4.3 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.3.5' into maint-0.4.2
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 07f0210a0525423f8cbbb41d00df0aa2d40fedf5 Merge: 02bb719a94 aeafb7f44f Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.3.5' into maint-0.4.2 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.4.3' into maint-0.4.4
by nickm＠torproject.org 13 Aug '20

13 Aug '20

commit 9d77ed7be2da18bef3ab1258f277c527347a94a7 Merge: f98b1988ec 34919368ce Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 12 20:21:04 2020 -0400 Merge branch 'maint-0.4.3' into maint-0.4.4 .gitlab-ci.yml | 34 +++++++++++++++++++++++++++++----- scripts/ci/ci-driver.sh | 34 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 16 deletions(-)

1 0