[tor-commits] [tor/master] Add a shareable continuous-integration script.
nickm at torproject.org
nickm at torproject.org
Mon Aug 10 23:33:15 UTC 2020
commit cb027b392f0f93d543cad1689e407b5fd71c2dec
Author: Nick Mathewson <nickm at torproject.org>
Date: Mon Aug 10 15:58:28 2020 -0400
Add a shareable continuous-integration script.
Eventually this should be used by every one of our CI scripts.
---
scripts/ci/ci-driver.sh | 424 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 424 insertions(+)
diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh
new file mode 100755
index 0000000000..50728a137e
--- /dev/null
+++ b/scripts/ci/ci-driver.sh
@@ -0,0 +1,424 @@
+#!/bin/bash
+
+# This script is used to build Tor for continuous integration. It should
+# be kept the same for all supported Tor versions.
+#
+# It's subject to the regular Tor license; see LICENSE for copying
+# information.
+
+set -o errexit
+set -o nounset
+
+# Options for this script.
+DEBUG_CI="${DEBUG_CI:-no}"
+COLOR_CI="${COLOR_CI:-yes}"
+
+# Options for which CI system this is.
+ON_GITLAB="${ON_GITLAB:-yes}"
+
+# Options for how to build Tor. All should be yes/no.
+FATAL_WARNINGS="${FATAL_WARNINGS:-yes}"
+HARDENING="${HARDENING:-no}"
+COVERAGE="${COVERAGE:-no}"
+RUST="${RUST:-no}"
+DOXYGEN="${DOXYGEN:-no}"
+ASCIIDOC="${ASCIIDOC:-no}"
+
+# Options for which tests to run. All should be yes/no.
+CHECK="${CHECK:-yes}"
+STEM="${STEM:-no}"
+CHUTNEY="${CHUTNEY:-no}"
+DISTCHECK="${DISTCHECK:-no}"
+
+# Options for where the Tor source is.
+CI_SRCDIR="${CI_SRCDIR:-.}"
+
+# Options for where to build.
+CI_BUILDDIR="${CI_BUILDDIR:-./build}"
+
+# How parallel should we run make?
+MAKE_J_OPT="${MAKE_J_OPT:--j4}"
+# Should we stop after make finds an error?
+MAKE_K_OPT="${MAKE_K_OPT:--k}"
+
+# What make target should we use for chutney?
+CHUTNEY_MAKE_TARGET="${CHUTNEY_MAKE_TARGET:-test-network}"
+
+# Where do we find our additional testing tools?
+CHUTNEY_PATH="${CHUTNEY_PATH:-}"
+STEM_PATH="${STEM_PATH:-}"
+
+#############################################################################
+# Preliminary functions.
+
+# Terminal coloring/emphasis stuff.
+if [[ "${COLOR_CI}" == "yes" ]]; then
+ T_RED=$(tput setaf 1 || true)
+ T_GREEN=$(tput setaf 2 || true)
+ T_DIM=$(tput dim || true)
+ T_BOLD=$(tput bold || true)
+ T_RESET=$(tput sgr0 || true)
+else
+ T_RED=
+ T_GREEN=
+ T_DIM=
+ T_BOLD=
+ T_RESET=
+fi
+
+function error()
+{
+ echo "${T_BOLD}${T_RED}ERROR:${T_RESET} $*" 1>&2
+}
+function die()
+{
+ echo "${T_BOLD}${T_RED}FATAL ERROR:${T_RESET} $*" 1>&2
+ exit 1
+}
+function hooray()
+{
+ echo "${T_BOLD}${T_GREEN}$*${T_RESET}"
+}
+
+if [[ "${DEBUG_CI}" == "yes" ]]; then
+ function debug()
+ {
+ echo "${T_DIM}(debug): $*${T_RESET}"
+ }
+else
+ function debug()
+ {
+ :
+ }
+fi
+
+function yes_or_no()
+{
+ local varname="$1"
+ local value="${!varname}"
+ debug "${varname} is ${value}"
+ if [[ "${value}" != 'yes' && "${value}" != 'no' ]]; then
+ die "${varname} must be 'yes' or 'no'. Got unexpected value ${value}".
+ fi
+}
+
+function incompatible()
+{
+ local varname1="$1"
+ local varname2="$2"
+ local val1="${!varname1}"
+ local val2="${!varname2}"
+ if [[ "${val1}" = 'yes' && "${val2}" = 'yes' ]]; then
+ die "Cannot set both ${varname1} and ${varname2}: they are incompatible."
+ fi
+}
+
+function runcmd()
+{
+ echo "${T_BOLD}\$ $*${T_RESET}"
+ if ! "$@" ; then
+ error "command '$*' has failed."
+ return 1
+ fi
+}
+
+function show_git_version()
+{
+ local tool="$1"
+ local dir="$2"
+ local version="?????"
+ if [[ -e "$dir/.git" ]] ; then
+ version=$(cd "$dir"; git rev-parse HEAD)
+ fi
+ echo "${T_BOLD}$tool:${T_RESET} $version"
+}
+
+if [[ "${ON_GITLAB}" == "yes" ]]; then
+ function start_section()
+ {
+ local label="$1"
+ local stamp
+ stamp=$(date +%s)
+ printf "section_start:%s:%s\r\e[0K" "$stamp" "$label"
+ echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}"
+ }
+ function end_section()
+ {
+ local label="$1"
+ local stamp
+ stamp=$(date +%s)
+ printf "section_end:%s:%s\r\e[0K" "$stamp" "$label"
+ }
+else
+ function start_section()
+ {
+ true
+ }
+ function end_section()
+ {
+ true
+ }
+fi
+
+if [[ "$*" == "" ]]; then
+ RUN_STAGE_CONFIGURE="yes"
+ RUN_STAGE_BUILD="yes"
+ RUN_STAGE_TEST="yes"
+else
+ RUN_STAGE_CONFIGURE="no"
+ RUN_STAGE_BUILD="no"
+ RUN_STAGE_TEST="no"
+
+ for stage in "$@"; do
+ case "$stage" in
+ configure)
+ RUN_STAGE_CONFIGURE="yes"
+ ;;
+ build)
+ RUN_STAGE_BUILD="yes"
+ ;;
+ test)
+ RUN_STAGE_TEST="yes"
+ ;;
+ *)
+ error "Unknown stage $stage"
+ ;;
+ esac
+ done
+fi
+
+#############################################################################
+# Validate inputs.
+
+debug Validating inputs
+yes_or_no DEBUG_CI
+yes_or_no COLOR_CI
+yes_or_no ON_GITLAB
+yes_or_no FATAL_WARNINGS
+yes_or_no HARDENING
+yes_or_no COVERAGE
+yes_or_no RUST
+yes_or_no DOXYGEN
+yes_or_no ASCIIDOC
+
+yes_or_no CHECK
+yes_or_no STEM
+yes_or_no DISTCHECK
+
+incompatible DISTCHECK CHECK
+incompatible DISTCHECK CHUTNEY
+incompatible DISTCHECK STEM
+incompatible DISTCHECK COVERAGE
+incompatible DISTCHECK DOXYGEN
+
+if [[ "${CHUTNEY}" = yes && "${CHUTNEY_PATH}" = '' ]] ; then
+ die "CHUTNEY is set to 'yes', but CHUTNEY_PATH was not specified."
+fi
+
+if [[ "${STEM}" = yes && "${STEM_PATH}" = '' ]] ; then
+ die "STEM is set to 'yes', but STEM_PATH was not specified."
+fi
+
+#############################################################################
+# Set up options for make and configure.
+
+make_options=()
+if [[ "$MAKE_J_OPT" != "" ]]; then
+ make_options+=("$MAKE_J_OPT")
+fi
+if [[ "$MAKE_K_OPT" != "" ]]; then
+ make_options+=("$MAKE_K_OPT")
+fi
+
+configure_options=()
+if [[ "$FATAL_WARNINGS" == "yes" ]]; then
+ configure_options+=("--enable-fatal-warnings")
+fi
+if [[ "$HARDENING" == "yes" ]]; then
+ configure_options+=("--enable-fragile-hardening")
+fi
+if [[ "$COVERAGE" == "yes" ]]; then
+ configure_options+=("--enable-coverage")
+fi
+if [[ "$RUST" == "yes" ]]; then
+ configure_options+=("--enable-rust")
+fi
+if [[ "$ASCIIDOC" != "yes" ]]; then
+ configure_options+=("--disable-asciidoc")
+fi
+
+#############################################################################
+# Tell the user about our versions of different tools and packages.
+
+uname -a
+python -V || echo "no 'python' binary."
+python3 -V || echo "no 'pythone' binary."
+
+show_git_version Tor "${CI_SRCDIR}"
+if [[ "${STEM}" = "yes" ]]; then
+ show_git_version Stem "${STEM_PATH}"
+fi
+if [[ "${CHUTNEY}" = "yes" ]]; then
+ show_git_version Chutney "${CHUTNEY_PATH}"
+fi
+
+#############################################################################
+# Make sure the directories are all there.
+
+# Make sure CI_SRCDIR exists and has a file we expect.
+if [[ ! -d "$CI_SRCDIR" ]] ; then
+ die "CI_SRCDIR=${CI_SRCDIR} is not a directory"
+fi
+if [[ ! -f "$CI_SRCDIR/src/core/or/or.h" ]] ; then
+ die "CI_SRCDIR=${CI_SRCDIR} does not look like a Tor directory."
+fi
+
+# Make CI_SRCDIR absolute.
+CI_SRCDIR=$(cd "$CI_SRCDIR" && pwd)
+
+# Create an "artifacts" directory to copy artifacts into.
+mkdir -p ./artifacts
+
+if [[ "$RUN_STAGE_CONFIGURE" = "yes" ]]; then
+
+ start_section "Autogen"
+ runcmd cd "${CI_SRCDIR}"
+ runcmd ./autogen.sh
+ runcmd mkdir -p "${CI_BUILDDIR}"
+ runcmd cd "${CI_BUILDDIR}"
+ end_section "Autogen"
+
+ # make the builddir absolute too.
+ CI_BUILDDIR=$(pwd)
+
+ start_section "Configure"
+ if ! runcmd "${CI_SRCDIR}"/configure "${configure_options[@]}" ; then
+ error "Here is the end of config.log:"
+ runcmd tail config.log
+ die "Unable to continue"
+ fi
+ end_section "Configure"
+else
+ debug "Skipping configure stage. Making sure that ${CI_BUILDDIR}/config.log exists."
+ if [[ ! -d "${CI_BUILDDIR}" ]]; then
+ die "Build directory ${CI_BUILDDIR} did not exist!";
+ fi
+ if [[ ! -f "${CI_BUILDDIR}/config.log" ]]; then
+ die "Tor was not configured in ${CI_BUILDDIR}!";
+ fi
+
+ cp config.log "${CI_SRCDIR}"/artifacts
+
+ runcmd cd "${CI_BUILDDIR}"
+ CI_BUILDDIR=$(pwd)
+fi
+
+###############################
+# Build Tor.
+
+if [[ "$RUN_STAGE_BUILD" = "yes" ]] ; then
+ if [[ "$DISTCHECK" = "no" ]]; then
+ start_section "Build"
+ runcmd make "${make_options[@]}" all
+ cp src/app/tor "${CI_SRCDIR}"/artifacts
+ end_section "Build"
+ else
+ export DISTCHECK_CONFIGURE_FLAGS="${configure_options[*]}"
+ # XXXX Set make options?
+ start_section Distcheck
+ if runcmd make "${make_options[@]}" distcheck ; then
+ hooray "Distcheck was successful. Nothing further will be done."
+ # We have to exit early here, since we can't do any other tests.
+ cp tor-*.tar.gz "${CI_SRCDIR}"/artifacts
+ exit 0
+ else
+ error "Diagnostics:"
+ runcmd make show-distdir-testlog || true
+ runcmd make show-distdir-core || true
+ die "Unable to continue."
+ fi
+ end_section Distcheck
+ fi
+fi
+##############################
+# Run tests.
+
+if [[ "$RUN_STAGE_TEST" == "no" ]]; then
+ echo "Skipping tests. Exiting now."
+ exit 0
+fi
+
+if [[ "$RUN_STAGE_BUILD" = "no" ]] ; then
+ debug "Skipped build stage. Making sure that ./src/app/tor exists."
+ if [[ ! -f "./src/app/tor" ]]; then
+ die "$(pwd)/src/app/tor does not exist"
+ fi
+fi
+
+FAILED_TESTS=""
+
+if [[ "${DOXYGEN}" = 'yes' ]]; then
+ start_section Doxygen
+ if runcmd make doxygen; then
+ hooray "make doxygen has succeeded."
+ else
+ FAILED_TESTS="${FAILED_TESTS} doxygen"
+ fi
+ end_section Doxygen
+fi
+
+if [[ "${CHECK}" = "yes" ]]; then
+ start_section "Check"
+ if runcmd make "${make_options[@]}" check; then
+ hooray "make check has succeeded."
+ else
+ error "Here are the contents of the test suite output:"
+ runcmd cat test_suite.log || true
+ FAILED_TESTS="${FAILED_TESTS} check"
+ fi
+ end_section "Check"
+fi
+
+if [[ "${CHUTNEY}" = "yes" ]]; then
+ start_section "Chutney"
+ if runcmd make "${CHUTNEY_MAKE_TARGET}"; then
+ hooray "Chutney tests have succeeded"
+ else
+ error "Chutney says:"
+ runcmd "${CHUTNEY_PATH}"/tools/diagnostics.sh || true
+ # XXXX These next two should be part of a make target.
+ runcmd ls test_network_log || true
+ runcmd cat test_network_log || true
+ FAILED_TESTS="${FAILED_TESTS} chutney"
+ fi
+ end_section "Chutney"
+fi
+
+if [[ "${STEM}" = "yes" ]]; then
+ start_section "Stem"
+ # XXXX This shold probably be part some test-stem make target.
+ if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \
+ python3 "${STEM_PATH}/run_tests.py" \
+ --tor src/app/tor \
+ --integ --test control.controller \
+ --test control.base_controller \
+ --test process \
+ --log TRACE \
+ --log-file stem.log ; then
+ hooray "Stem tests have succeeded"
+ else
+ error "Stem output:"
+ runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log
+ runcmd grep -v "SocketClosed" stem.log | tail -1000
+ FAILED_TESTS="${FAILED_TESTS} stem"
+ fi
+ end_section "Stem"
+fi
+
+# TODO: Coverage
+
+if [[ "${FAILED_TESTS}" != "" ]]; then
+ die "Failed tests: ${FAILED_TESTS}"
+fi
+
+hooray "Everything seems fine."
More information about the tor-commits
mailing list