test-on-testmachine

#!/bin/sh

. "$(dirname "$0")"/functions
. detect-environment
. compile-options

case "$TEST_MACHINE" in
chroot)
    SCRIPT=test-chroot
    # Don't lose the trailing slash!
    CHROOT_ROOT=$HOME/testmachine-chroot/
    LOGIN_COMMAND="sudo chroot $CHROOT_ROOT /run-in-home-dir.sh"
    TESTMACHINE_URI=$CHROOT_ROOT

    log_debug "Mounting /proc filesystem..."
    sudo umount "${CHROOT_ROOT}"proc >/dev/null 2>&1 || true
    mount_procfs "${CHROOT_ROOT}"proc
    ;;
*)
    # Add VMs, etc here.
    log_error "Only chroot test machines are supported at the moment."
    exit 1
    ;;
esac

SCRIPT_BASEDIR="$(
    cd "$(dirname "$0")"
    pwd
)"                                            # /home/user/whatever/buildscripts/build-scripts
SCRIPT_BASEDIR="$(dirname "$SCRIPT_BASEDIR")" # /home/user/whatever/buildscripts
SCRIPT_BASEDIR="$(dirname "$SCRIPT_BASEDIR")" # /home/user/whatever

# We still need to perform several cleanup tasks after doing a chroot test run.
# So wrap the test execution in an if so that we can proceed with clean-up on error.
log_debug "Executing tests on test machine..."
if remote_script_general "$SCRIPT" "$LOGIN_COMMAND" "$SCRIPT_BASEDIR"; then
    return_code=0
else
    return_code=$?
    log_debug "Failed to execute tests on test machine..."
fi

# Look for lingering processes after running tests.
case "$TEST_MACHINE" in
chroot)
    # Fuser has special output. The PIDs arrive on stdout, and all the ornaments
    # arrive on stderr, so all we have to do is to grep for PID numbers.
    for pid in $(sudo "$FUSER" "$CHROOT_ROOT" 2>/dev/null | sed -e 's/[^ 0-9]//g'); do
        case "$OS" in
        rhel)
            # Minilogd may sometimes launch on RedHat without any user
            # intervention. Ignore that.
            # shellcheck disable=SC2009
            if ps -ef 2>/dev/null | grep -E "^ *[^ ]+ +$pid" | grep -q minilogd; then
                continue
            fi
            ;;
        suse)

            # gpg-agent sometimes is spawned on suse during our tests.
            # This happens only in chroot. Ignore that.
            if ps -o command --pid "$pid" 2>/dev/null | grep -q "gpg-agent --homedir /var/tmp/zypp.*zypp-trusted.*--daemon"; then
                continue
            fi
            ;;
        ubuntu)
            # gpg-agent sometimes is spawned on Ubuntu 24 during our tests.
            # This happens only in chroot. Ignore that.
            if [ "$OS_VERSION_MAJOR" = "24" ]; then
                if ps -o command --pid "$pid" 2>/dev/null | grep -q "gpg-agent --homedir /etc/apt/sources.list.d/.gnupg-temp --use-standard-socket --daemon"; then
                    continue
                fi
            fi
            ;;
        esac

        # Leaving processes behind is an error. It should never happen.
        return_code=1
        (
            # This next part is very confusing when mixed with "set -x".
            # Turn it off inside the subshell while printing this.
            set +x
            log_error "Found processes left behind in the chroot. I'm killing them, but the build will fail, so clean up your mess!"
            echo "Found PID $pid. Info from 'ps -ef'"
            echo "--------------" 1>&2
            # shellcheck disable=SC2009
            ps -ef | grep "$pid" | grep -v grep 1>&2
            echo "--------------" 1>&2
        )
    done

    # Kill the lingering process
    log_debug "Killing lingering processes in chroot..."
    sudo "$FUSER" -k "$CHROOT_ROOT" >/dev/null 2>&1 || true
    log_debug "Unmounting /proc filesystem..."
    sudo umount "${CHROOT_ROOT}"proc >/dev/null 2>&1 || true
    ;;
esac

# Collect test results.
INCLUDES='--include=test.* --include=summary.*'
# Note: Don't use sudo or "rsync -a", because we don't want root-owned files
# to show up in the results.
log_debug "Collecting test results from core repo..."
# shellcheck disable=SC2086
# > Double quote to prevent globbing and word splitting.
# We want word splitting
rsync -rv $INCLUDES --exclude="*" \
    "$TESTMACHINE_URI$BASEDIR"/core/tests/acceptance/ \
    "$BASEDIR"/core/tests/acceptance/ \
    >>/tmp/rsync.log

log_debug "Collecting test results from masterfiles repo..."
# shellcheck disable=SC2086
# > Double quote to prevent globbing and word splitting.
# We want word splitting
rsync -rv $INCLUDES --exclude="*" \
    "$TESTMACHINE_URI$BASEDIR"/masterfiles/tests/acceptance/ \
    "$BASEDIR"/masterfiles/tests/acceptance/ \
    >>/tmp/rsync.log

if [ "$PROJECT" = nova ]; then
    log_debug "Collecting test results from enterprise repo..."
    # shellcheck disable=SC2086
    # > Double quote to prevent globbing and word splitting.
    # We want word splitting
    rsync -rv $INCLUDES --exclude="*" \
        "$TESTMACHINE_URI$BASEDIR"/enterprise/tests/acceptance/ \
        "$BASEDIR"/enterprise/tests/acceptance/ \
        >>/tmp/rsync.log
fi

if [ $return_code -ne 0 ]; then
    log_error "We just finished cleaning up, but earlier command had returned: $return_code"
    exit $return_code
fi