test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 QEMU_MEM="${QEMU_MEM:-512M}"
  34 # Note that defining a different IMAGE_NAME in a test setup script will only result
  35 # in default.img being copied and renamed. It can then be extended by defining
  36 # a test_append_files() function. The $1 parameter will be the root directory.
  37 # To force creating a new image from scratch (eg: to encrypt it), also define
  38 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  39 IMAGE_NAME=${IMAGE_NAME:-default}
  40 STRIP_BINARIES="${STRIP_BINARIES:-yes}"
  41 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  42 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  43 LOOPDEV=
  44
  45 # Simple wrapper to unify boolean checks.
  46 # Note: this function needs to stay near the top of the file, so we can use it
  47 #       in code in the outermost scope.
  48 get_bool() {
  49     # Make the value lowercase to make the regex matching simpler
  50     local _bool="${1,,}"
  51
  52     # Consider empty value as "false"
  53     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  54         return 1
  55     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  56         return 0
  57     else
  58         echo >&2 "Value '$_bool' is not a valid boolean value"
  59         exit 1
  60     fi
  61 }
  62
  63 # Decide if we can (and want to) run QEMU with KVM acceleration.
  64 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  65 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  66 # running under KVM. If these conditions are met, enable KVM (and possibly
  67 # nested KVM), otherwise disable it.
  68 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && [[ "$(systemd-detect-virt -v)" != kvm ]]); then
  69     QEMU_KVM=yes
  70 else
  71     QEMU_KVM=no
  72 fi
  73
  74 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  75     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
  76     ROOTLIBDIR=/usr/lib/systemd
  77 fi
  78
  79 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
  80 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
  81 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
  82 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
  83 TOOLS_DIR="$SOURCE_DIR/tools"
  84 # These variables are used by test scripts
  85 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR TOOLS_DIR
  86
  87 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
  88 if get_bool "${NO_BUILD:=}"; then
  89     BUILD_DIR="$SOURCE_DIR"
  90 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
  91     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
  92     exit 1
  93 fi
  94
  95 PATH_TO_INIT="$ROOTLIBDIR/systemd"
  96 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
  97 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote")}"
  98 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
  99 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 100 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 101
 102 TESTFILE="${BASH_SOURCE[1]}"
 103 if [ -z "$TESTFILE" ]; then
 104     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 105     exit 1
 106 fi
 107 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 108 STATEDIR="$BUILD_DIR/test/$TESTNAME"
 109 STATEFILE="$STATEDIR/.testdir"
 110 IMAGESTATEDIR="$STATEDIR/.."
 111 TESTLOG="$STATEDIR/test.log"
 112
 113 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 114     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 115     exit 1
 116 fi
 117 TESTID="${BASH_REMATCH[1]:?}"
 118
 119 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 120     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 121     exit 1
 122 fi
 123
 124 BASICTOOLS=(
 125     awk
 126     base64
 127     basename
 128     bash
 129     capsh
 130     cat
 131     chmod
 132     chown
 133     cmp
 134     cryptsetup
 135     cut
 136     date
 137     dd
 138     diff
 139     dirname
 140     dmsetup
 141     echo
 142     env
 143     false
 144     getconf
 145     getent
 146     getfacl
 147     grep
 148     gunzip
 149     gzip
 150     head
 151     ionice
 152     ip
 153     ldd
 154     ln
 155     loadkeys
 156     login
 157     losetup
 158     lz4cat
 159     mkfifo
 160     mktemp
 161     modprobe
 162     mount
 163     mountpoint
 164     mv
 165     nc
 166     nproc
 167     readlink
 168     rev
 169     rm
 170     rmdir
 171     sed
 172     seq
 173     setfattr
 174     setfont
 175     setsid
 176     sfdisk
 177     sh
 178     sleep
 179     stat
 180     su
 181     sulogin
 182     sysctl
 183     tail
 184     tar
 185     tee
 186     test
 187     timeout
 188     touch
 189     tr
 190     true
 191     truncate
 192     umount
 193     uname
 194     unshare
 195     wc
 196     xargs
 197     xzcat
 198 )
 199
 200 DEBUGTOOLS=(
 201     cp
 202     df
 203     dhclient
 204     dmesg
 205     du
 206     find
 207     free
 208     grep
 209     hostname
 210     id
 211     less
 212     ln
 213     ls
 214     mkdir
 215     ping
 216     ps
 217     route
 218     sort
 219     strace
 220     stty
 221     tty
 222     vi
 223 )
 224
 225 is_built_with_asan() {
 226     if ! type -P objdump >/dev/null; then
 227         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 228         return 1
 229     fi
 230
 231     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 232     local _asan_calls
 233     _asan_calls="$(objdump -dC "$SYSTEMD_JOURNALD" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 234     if ((_asan_calls < 1000)); then
 235         return 1
 236     else
 237         return 0
 238     fi
 239 }
 240
 241 is_built_with_coverage() {
 242     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 243         return 1
 244     fi
 245
 246     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 247 }
 248
 249 IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
 250 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 251
 252 if get_bool "$IS_BUILT_WITH_ASAN"; then
 253     STRIP_BINARIES=no
 254     SKIP_INITRD="${SKIP_INITRD:-yes}"
 255     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
 256     QEMU_MEM="2048M"
 257     QEMU_SMP="${QEMU_SMP:-4}"
 258
 259     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 260     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 261         ASAN_COMPILER=gcc
 262         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 263     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 264         ASAN_COMPILER=clang
 265         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 266
 267         # As clang's ASan DSO is usually in a non-standard path, let's check if
 268         # the environment is set accordingly. If not, warn the user and exit.
 269         # We're not setting the LD_LIBRARY_PATH automagically here, because
 270         # user should encounter (and fix) the same issue when running the unit
 271         # tests (meson test)
 272         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 273             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 274             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 275             exit 1
 276         fi
 277     else
 278         echo >&2 "systemd is not linked against the ASan DSO"
 279         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 280         exit 1
 281     fi
 282
 283     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 284 fi
 285
 286 find_qemu_bin() {
 287     QEMU_BIN="${QEMU_BIN:-""}"
 288     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 289     if get_bool "$QEMU_KVM"; then
 290         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 291     fi
 292
 293     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 294     case $ARCH in
 295     x86_64)
 296         # QEMU's own build system calls it qemu-system-x86_64
 297         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 298         ;;
 299     i*86)
 300         # new i386 version of QEMU
 301         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 302
 303         # i386 version of QEMU
 304         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 305         ;;
 306     ppc64*)
 307         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 308         ;;
 309     esac
 310
 311     if [[ ! -e "$QEMU_BIN" ]]; then
 312         echo "Could not find a suitable QEMU binary" >&2
 313         return 1
 314     fi
 315 }
 316
 317 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 318 # returns 0 if newer or equal
 319 # returns 1 if older
 320 # returns 2 if failing
 321 qemu_min_version() {
 322     find_qemu_bin || return 2
 323
 324     # get version from binary
 325     local qemu_ver
 326     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 327
 328     # Check version string format
 329     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 330     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 331
 332     # compare as last command to return that value
 333     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 334 }
 335
 336 # Return 0 if QEMU did run (then you must check the result state/logs for actual
 337 # success), or 1 if QEMU is not available.
 338 run_qemu() {
 339     if [ -f /etc/machine-id ]; then
 340         read -r MACHINE_ID </etc/machine-id
 341         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 342             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 343         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 344             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 345     fi
 346
 347     local CONSOLE=ttyS0
 348
 349     rm -f "$initdir"/{testok,failed,skipped}
 350     # make sure the initdir is not mounted to avoid concurrent access
 351     cleanup_initdir
 352     umount_loopback
 353
 354     if [[ ! "$KERNEL_BIN" ]]; then
 355         if get_bool "$LOOKS_LIKE_ARCH"; then
 356             KERNEL_BIN=/boot/vmlinuz-linux
 357         else
 358             [ "$ARCH" ] || ARCH=$(uname -m)
 359             case $ARCH in
 360                 ppc64*)
 361                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 362                     # uses the "standard" vmlinuz- prefix
 363                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 364                     CONSOLE=hvc0
 365                     ;;
 366                 *)
 367                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 368                     ;;
 369             esac
 370         fi
 371     fi
 372
 373     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 374     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 375     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 376     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 377     if [[ ! "$INITRD" ]]; then
 378         if [[ -e "$default_fedora_initrd" ]]; then
 379             INITRD="$default_fedora_initrd"
 380         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 381             INITRD="$default_debian_initrd"
 382         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 383             INITRD="$default_arch_initrd"
 384         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 385             INITRD="$default_suse_initrd"
 386         fi
 387     fi
 388
 389     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 390     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 391     # is not installed or there's some other error when calling it, fall back
 392     # to the original value (QEMU_SMP=1).
 393     if [[ -z "${QEMU_SMP:=}" ]]; then
 394         if ! QEMU_SMP=$(nproc); then
 395             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 396             QEMU_SMP=1
 397         fi
 398     fi
 399
 400     find_qemu_bin || return 1
 401
 402     # Umount initdir to avoid concurrent access to the filesystem
 403     _umount_dir "$initdir"
 404
 405     local kernel_params=()
 406     local qemu_options=()
 407     local qemu_cmd=("$QEMU_BIN")
 408
 409     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 410         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 411     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 412         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 413     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 414         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 415     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 416         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 417         exit 1
 418     fi
 419
 420     if get_bool "$LOOKS_LIKE_SUSE"; then
 421         kernel_params+=("rd.hostonly=0")
 422     fi
 423
 424     kernel_params+=(
 425         "root=LABEL=systemd_boot"
 426         "rw"
 427         "raid=noautodetect"
 428         "rd.luks=0"
 429         "loglevel=2"
 430         "init=$PATH_TO_INIT"
 431         "console=$CONSOLE"
 432         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 433         "systemd.unit=testsuite.target"
 434         "systemd.wants=testsuite-$1.service"
 435     )
 436
 437     if ! get_bool "$INTERACTIVE_DEBUG"; then
 438         kernel_params+=("systemd.wants=end.service")
 439     fi
 440
 441     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 442     qemu_options+=(
 443         -smp "$QEMU_SMP"
 444         -net none
 445         -m "$QEMU_MEM"
 446         -nographic
 447         -kernel "$KERNEL_BIN"
 448         -drive "format=raw,cache=unsafe,file=$image"
 449     )
 450
 451     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 452         local user_qemu_options
 453         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 454         qemu_options+=("${user_qemu_options[@]}")
 455     fi
 456
 457     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 458         local user_kernel_append
 459         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 460         kernel_params+=("${user_kernel_append[@]}")
 461     fi
 462
 463     if [[ "$INITRD" ]] && ! get_bool "$SKIP_INITRD"; then
 464         qemu_options+=(-initrd "$INITRD")
 465     fi
 466
 467     # Let's use KVM if possible
 468     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 469         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 470     fi
 471
 472     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 473         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 474     fi
 475
 476     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}")
 477     rc=$?
 478     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 479         derror "Test timed out after ${QEMU_TIMEOUT}s"
 480         TIMED_OUT=1
 481     else
 482         [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
 483     fi
 484     return 0
 485 }
 486
 487 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 488 # success), or 1 if nspawn is not available.
 489 run_nspawn() {
 490     [[ -d /run/systemd/system ]] || return 1
 491     rm -f "${initdir:?}"/{testok,failed,skipped}
 492
 493     local nspawn_cmd=()
 494     local nspawn_options=(
 495         "--register=no"
 496         "--kill-signal=SIGKILL"
 497         "--directory=${1:?}"
 498         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 499     )
 500     local kernel_params=(
 501         "$PATH_TO_INIT"
 502         "systemd.unit=testsuite.target"
 503         "systemd.wants=testsuite-$2.service"
 504     )
 505
 506     if ! get_bool "$INTERACTIVE_DEBUG"; then
 507         kernel_params+=("systemd.wants=end.service")
 508     fi
 509
 510     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 511         local user_nspawn_arguments
 512         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 513         nspawn_options+=("${user_nspawn_arguments[@]}")
 514     fi
 515
 516     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 517         local user_kernel_append
 518         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 519         kernel_params+=("${user_kernel_append[@]}")
 520     fi
 521
 522     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 523         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 524         exit
 525     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 526         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 527     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 528         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 529     else
 530         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 531         exit 1
 532     fi
 533
 534     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 535         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 536     else
 537         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 538     fi
 539
 540     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}")
 541     rc=$?
 542     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 543         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 544         TIMED_OUT=1
 545     else
 546         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 547     fi
 548     return 0
 549 }
 550
 551 # Build two very minimal root images, with two units, one is the same and one is different across them
 552 install_verity_minimal() {
 553     dinfo "Set up a set of minimal images for verity verification"
 554     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 555         return
 556     fi
 557     if ! command -v mksquashfs >/dev/null 2>&1; then
 558         dfatal "mksquashfs not found"
 559         exit 1
 560     fi
 561     if ! command -v veritysetup >/dev/null 2>&1; then
 562         dfatal "veritysetup not found"
 563         exit 1
 564     fi
 565     # Local modifications of some global variables is intentional in this
 566     # subshell (SC2030)
 567     # shellcheck disable=SC2030
 568     (
 569         BASICTOOLS=(
 570             bash
 571             cat
 572             grep
 573             mount
 574             sleep
 575         )
 576         oldinitdir="$initdir"
 577         rm -rfv "$TESTDIR/minimal"
 578         export initdir="$TESTDIR/minimal"
 579         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 580         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 581         setup_basic_dirs
 582         install_basic_tools
 583         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 584         # issue, thus falsely triggering SC2030 in this case
 585         # See: koalaman/shellcheck#1409
 586         if [[ -v ASAN_RT_PATH ]]; then
 587             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 588             # into the verity images to get rid of the annoying errors about
 589             # missing $LD_PRELOAD libraries.
 590             inst_libs "$ASAN_RT_PATH"
 591             inst_library "$ASAN_RT_PATH"
 592             # Create a dummy LSan suppression file otherwise gcc's ASan
 593             # complains as it doesn't exist in the minimal image
 594             # (i.e. when running TEST-29 or TEST-50 under sanitizers)
 595             touch "$initdir/systemd-lsan.supp"
 596         fi
 597         cp "$os_release" "$initdir/usr/lib/os-release"
 598         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 599         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 600         touch "$initdir/opt/some_file"
 601         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 602         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 603         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 604 [Service]
 605 ExecStartPre=cat /usr/lib/os-release
 606 ExecStart=sleep 120
 607 EOF
 608         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 609
 610         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 611         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 612             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 613
 614         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 615         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 616         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 617
 618         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 619         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 620             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 621
 622         # Rolling distros like Arch do not set VERSION_ID
 623         local version_id=""
 624         if grep -q "^VERSION_ID=" "$os_release"; then
 625             version_id="$(grep "^VERSION_ID=" "$os_release")"
 626         fi
 627
 628         export initdir="$TESTDIR/app0"
 629         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 630         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 631         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 632         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 633 [Service]
 634 Type=oneshot
 635 RemainAfterExit=yes
 636 ExecStart=/opt/script0.sh
 637 TemporaryFileSystem=/var/lib
 638 StateDirectory=app0
 639 RuntimeDirectory=app0
 640 EOF
 641         cat >"$initdir/opt/script0.sh" <<EOF
 642 #!/bin/bash
 643 set -e
 644 test -e /usr/lib/os-release
 645 echo bar > \${STATE_DIRECTORY}/foo
 646 cat /usr/lib/extension-release.d/extension-release.app0
 647 EOF
 648         chmod +x "$initdir/opt/script0.sh"
 649         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 650         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 651
 652         export initdir="$TESTDIR/app1"
 653         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 654         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 655         ( echo "${version_id}"
 656           echo "SYSEXT_SCOPE=portable"
 657           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 658         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 659         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 660 [Service]
 661 Type=oneshot
 662 RemainAfterExit=yes
 663 ExecStart=/opt/script1.sh
 664 StateDirectory=app1
 665 RuntimeDirectory=app1
 666 EOF
 667         cat >"$initdir/opt/script1.sh" <<EOF
 668 #!/bin/bash
 669 set -e
 670 test -e /usr/lib/os-release
 671 echo baz > \${STATE_DIRECTORY}/foo
 672 cat /usr/lib/extension-release.d/extension-release.app2
 673 EOF
 674         chmod +x "$initdir/opt/script1.sh"
 675         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 676         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 677     )
 678 }
 679
 680 setup_basic_environment() {
 681     # create the basic filesystem layout
 682     setup_basic_dirs
 683
 684     install_systemd
 685     install_missing_libraries
 686     install_config_files
 687     install_zoneinfo
 688     create_rc_local
 689     install_basic_tools
 690     install_libnss
 691     install_pam
 692     install_dbus
 693     install_fonts
 694     install_keymaps
 695     install_terminfo
 696     install_execs
 697     install_fs_tools
 698     install_modules
 699     install_plymouth
 700     install_haveged
 701     install_debug_tools
 702     install_ld_so_conf
 703     install_testuser
 704     has_user_dbus_socket && install_user_dbus
 705     setup_selinux
 706     strip_binaries
 707     instmods veth
 708     install_depmod_files
 709     generate_module_dependencies
 710     if get_bool "$IS_BUILT_WITH_ASAN"; then
 711         create_asan_wrapper
 712     fi
 713     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 714         install_verity_minimal
 715     fi
 716 }
 717
 718 setup_selinux() {
 719     dinfo "Setup SELinux"
 720     # don't forget KERNEL_APPEND='... selinux=1 ...'
 721     if ! get_bool "$SETUP_SELINUX"; then
 722         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 723         return 0
 724     fi
 725
 726     local conf_dir=/etc/selinux
 727     local fixfiles_tools=(bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles)
 728
 729     # Make sure the following statement can't expand to "/" to prevent
 730     # a potential where-are-my-backups situation
 731     rm -rf "${initdir:?}/$conf_dir"
 732     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 733         dfatal "Failed to copy $conf_dir"
 734         exit 1
 735     fi
 736
 737     touch "$initdir/.autorelabel"
 738     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 739     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 740
 741     image_install "${fixfiles_tools[@]}"
 742     image_install fixfiles
 743     image_install sestatus
 744 }
 745
 746 install_valgrind() {
 747     if ! type -p valgrind; then
 748         dfatal "Failed to install valgrind"
 749         exit 1
 750     fi
 751
 752     local valgrind_bins valgrind_libs valgrind_dbg_and_supp
 753
 754     valgrind_bins="$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')"
 755     image_install "$valgrind_bins"
 756
 757     valgrind_libs="$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')"
 758     image_install "$valgrind_libs"
 759
 760     valgrind_dbg_and_supp="$(
 761         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 762         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 763     )"
 764     image_install "$valgrind_dbg_and_supp"
 765 }
 766
 767 create_valgrind_wrapper() {
 768     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 769     ddebug "Create $valgrind_wrapper"
 770     cat >"$valgrind_wrapper" <<EOF
 771 #!/usr/bin/env bash
 772
 773 mount -t proc proc /proc
 774 exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 775 EOF
 776     chmod 0755 "$valgrind_wrapper"
 777 }
 778
 779 create_asan_wrapper() {
 780     local asan_wrapper="$initdir/$ROOTLIBDIR/systemd-under-asan"
 781     dinfo "Create ASan wrapper as '$asan_wrapper'"
 782
 783     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 784
 785     # clang: install llvm-symbolizer to generate useful reports
 786     # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 787     [[ "$ASAN_COMPILER" == "clang" ]] && image_install "llvm-symbolizer"
 788
 789     cat >"$asan_wrapper" <<EOF
 790 #!/usr/bin/env bash
 791
 792 set -x
 793
 794 echo "ASan RT: $ASAN_RT_PATH"
 795 if [[ ! -e "$ASAN_RT_PATH" ]]; then
 796     echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
 797     exit 1
 798 fi
 799
 800 # Suppress certain leaks reported by LSan (either in external tools or bogus
 801 # ones)
 802 # Docs: # https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer#suppressions
 803 #
 804 # - fsck is called by systemd-homed and is reporting a leak we're not interested
 805 #   in
 806 # - libLLVM is a "side effect" caused by the previous fsck leak
 807 cat >/systemd-lsan.supp <<INNER_EOF
 808 leak:/bin/fsck$
 809 leak:/sbin/fsck$
 810 leak:/lib/libLLVM
 811 INNER_EOF
 812
 813 DEFAULT_LSAN_OPTIONS=${LSAN_OPTIONS:-}:suppressions=/systemd-lsan.supp
 814 DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
 815 DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
 816 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS LSAN_OPTIONS=\$DEFAULT_LSAN_OPTIONS"
 817
 818 # As right now bash is the PID 1, we can't expect PATH to have a sane value.
 819 # Let's make one to prevent unexpected "<bin> not found" issues in the future
 820 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 821
 822 mount -t proc proc /proc
 823 mount -t sysfs sysfs /sys
 824 mount -o remount,rw /
 825
 826 # A lot of services (most notably dbus) won't start without preloading libasan
 827 # See https://github.com/systemd/systemd/issues/5004
 828 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=$ASAN_RT_PATH"
 829
 830 if [[ "$ASAN_COMPILER" == "clang" ]]; then
 831   # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
 832   # unnecessary for gcc & libasan, however, for clang this is crucial, as its
 833   # runtime ASan DSO is in a non-standard (library) path.
 834   echo "${ASAN_RT_PATH%/*}" >/etc/ld.so.conf.d/asan-path-override.conf
 835   ldconfig
 836 fi
 837 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 838 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 839 echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
 840
 841 # ASAN and syscall filters aren't compatible with each other.
 842 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 843
 844 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 845 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 846 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 847 mkdir -p "\$JOURNALD_CONF_DIR"
 848 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 849
 850 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 851 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 852 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 853 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 854
 855 # 90s isn't enough for some services to finish when literally everything is run
 856 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 857 # Let's limit which environments such services should be executed in.
 858 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 859 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 860
 861 # Let's override another hard-coded timeout that kicks in too early
 862 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 863 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 864
 865 # D-Bus has troubles during system shutdown causing it to fail. Although it's
 866 # harmless, it causes unnecessary noise in the logs, so let's disable LSan's
 867 # at_exit check just for the dbus.service
 868 mkdir -p /etc/systemd/system/dbus.service.d
 869 printf "[Service]\nEnvironment=ASAN_OPTIONS=leak_check_at_exit=false\n" >/etc/systemd/system/dbus.service.d/disable-lsan.conf
 870
 871 # Some utilities run via IMPORT/RUN/PROGRAM udev directives fail because
 872 # they're uninstrumented (like dmsetup). Let's add a simple rule which sets
 873 # LD_PRELOAD to the ASan RT library to fix this.
 874 mkdir -p /etc/udev/rules.d
 875 cat >/etc/udev/rules.d/00-set-LD_PRELOAD.rules <<INNER_EOF
 876 SUBSYSTEM=="block", ENV{LD_PRELOAD}="$ASAN_RT_PATH"
 877 INNER_EOF
 878 chmod 0644 /etc/udev/rules.d/00-set-LD_PRELOAD.rules
 879
 880 # The 'mount' utility doesn't behave well under libasan, causing unexpected
 881 # fails during boot and subsequent test results check:
 882 # bash-5.0# mount -o remount,rw -v /
 883 # mount: /dev/sda1 mounted on /.
 884 # bash-5.0# echo \$?
 885 # 1
 886 # Let's workaround this by clearing the previously set LD_PRELOAD env variable,
 887 # so the libasan library is not loaded for this particular service
 888 unset_ld_preload() {
 889     local _dropin_dir="/etc/systemd/system/\$1.service.d"
 890     mkdir -p "\$_dropin_dir"
 891     printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$_dropin_dir/unset_ld_preload.conf"
 892 }
 893
 894 unset_ld_preload systemd-remount-fs
 895 unset_ld_preload testsuite-
 896
 897 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 898 exec "$ROOTLIBDIR/systemd" "\$@"
 899 EOF
 900
 901     chmod 0755 "$asan_wrapper"
 902 }
 903
 904 create_strace_wrapper() {
 905     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
 906     ddebug "Create $strace_wrapper"
 907     cat >"$strace_wrapper" <<EOF
 908 #!/usr/bin/env bash
 909
 910 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
 911 EOF
 912     chmod 0755 "$strace_wrapper"
 913 }
 914
 915 install_fs_tools() {
 916     dinfo "Install fsck"
 917     image_install /sbin/fsck*
 918     image_install -o /bin/fsck*
 919
 920     # fskc.reiserfs calls reiserfsck. so, install it
 921     image_install -o reiserfsck
 922
 923     # we use mkfs in system-repart tests
 924     image_install /sbin/mkfs.ext4
 925     image_install /sbin/mkfs.vfat
 926 }
 927
 928 install_modules() {
 929     dinfo "Install modules"
 930
 931     instmods loop
 932     instmods vfat
 933     instmods nls_ascii =nls
 934     instmods dummy
 935
 936     if get_bool "$LOOKS_LIKE_SUSE"; then
 937         instmods ext4
 938     fi
 939 }
 940
 941 install_dmevent() {
 942     instmods dm_crypt =crypto
 943     inst_binary dmeventd
 944     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
 945     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 946         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 947         # and since buster/bionic 95-dm-notify.rules
 948         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 949         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 950     else
 951         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 952     fi
 953     if get_bool "$LOOKS_LIKE_SUSE"; then
 954         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
 955     fi
 956 }
 957
 958 install_multipath() {
 959     instmods "=md" multipath
 960     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
 961     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
 962     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 963         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
 964     else
 965         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
 966     fi
 967     mkdir -p "${initdir:?}/etc/multipath"
 968
 969     local file
 970     while read -r file; do
 971         # Install libraries required by the given library
 972         inst_libs "$file"
 973         # Install the library itself and create necessary symlinks
 974         inst_library "$file"
 975     done < <(find /lib*/multipath -type f)
 976
 977     if get_bool "$LOOKS_LIKE_ARCH"; then
 978         # On Arch the multipath libraries are not linked against libgcc_s.so.1,
 979         # but it's still required at runtime
 980         inst_library "/lib64/libgcc_s.so.1"
 981     fi
 982 }
 983
 984 install_lvm() {
 985     image_install lvm
 986     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
 987     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
 988     image_install -o "/lib/tmpfiles.d/lvm2.conf"
 989     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 990         inst_rules 56-lvm.rules 69-lvm-metad.rules
 991     else
 992         # Support the new udev autoactivation introduced in lvm 2.03.14
 993         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
 994         # Static autoactivation (via lvm2-activation-generator) was dropped
 995         # in lvm 2.03.15
 996         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
 997         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
 998             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
 999         else
1000             image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1001             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1002             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1003         fi
1004     fi
1005     mkdir -p "${initdir:?}/etc/lvm"
1006 }
1007
1008 install_btrfs() {
1009     instmods btrfs
1010     # Not all utilities provided by btrfs-progs are listed here; extend the list
1011     # if necessary
1012     image_install btrfs btrfstune mkfs.btrfs
1013     inst_rules 64-btrfs-dm.rules
1014 }
1015
1016 install_iscsi() {
1017     # Install both client and server side stuff by default
1018     local inst="${1:-}"
1019     local file
1020
1021     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1022     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1023     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1024         image_install iscsi-iname iscsiadm iscsid iscsistart
1025         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1026         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1027         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1028         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1029         mkdir -p "${initdir:?}/etc/iscsi"
1030         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1031         inst_simple "/etc/iscsi/initiatorname.iscsi"
1032     fi
1033
1034     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1035     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1036     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1037         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1038         image_install -o /etc/sysconfig/tgtd
1039         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1040         mkdir -p "${initdir:?}/etc/tgt"
1041         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1042         # Install perl modules required by tgt-admin
1043         #
1044         # Forgive me father for I have sinned. The monstrosity below appends
1045         # a perl snippet to the `tgt-admin` perl script on the fly, which
1046         # dumps a list of files (perl modules) required by `tgt-admin` at
1047         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1048         # passed to `inst_simple` which installs the necessary files into the image
1049         #
1050         # shellcheck disable=SC2016
1051         while read -r file; do
1052             inst_simple "$file"
1053         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1054     fi
1055 }
1056
1057 install_compiled_systemd() {
1058     dinfo "Install compiled systemd"
1059
1060     local ninja_bin
1061     ninja_bin="$(type -P ninja || type -P ninja-build)"
1062     if [[ -z "$ninja_bin" ]]; then
1063         dfatal "ninja was not found"
1064         exit 1
1065     fi
1066     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1067
1068     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1069     # find them in the same directory as the runtime data counts
1070     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1071         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1072         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1073     fi
1074 }
1075
1076 install_debian_systemd() {
1077     dinfo "Install debian systemd"
1078
1079     local files
1080
1081     while read -r deb; do
1082         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1083         ddebug "Install debian files from package $deb"
1084         for file in $files; do
1085             [ -e "$file" ] || continue
1086             [ -d "$file" ] && continue
1087             inst "$file"
1088         done
1089     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1090 }
1091
1092 install_suse_systemd() {
1093     local testsdir=/usr/lib/systemd/tests
1094     local pkgs
1095
1096     dinfo "Install SUSE systemd"
1097
1098     pkgs=(
1099         systemd
1100         systemd-container
1101         systemd-coredump
1102         systemd-experimental
1103         systemd-journal-remote
1104         systemd-portable
1105         udev
1106     )
1107
1108     for p in "${pkgs[@]}"; do
1109         rpm -q "$p" &>/dev/null || continue
1110
1111         ddebug "Install files from package $p"
1112         while read -r f; do
1113             [ -e "$f" ] || continue
1114             [ -d "$f" ] && continue
1115             inst "$f"
1116         done < <(rpm -ql "$p")
1117     done
1118
1119     # we only need testsdata dir as well as the unit tests (for
1120     # TEST-02-UNITTESTS) in the image.
1121     dinfo "Install unit tests and testdata directory"
1122
1123     mkdir -p "$initdir/$testsdir"
1124     cp "$testsdir"/test-* "$initdir/$testsdir/"
1125     cp -a "$testsdir/testdata" "$initdir/$testsdir/"
1126
1127     # On openSUSE, these dirs are not created at package install for now on.
1128     mkdir -p "$initdir/var/log/journal/remote"
1129 }
1130
1131 install_distro_systemd() {
1132     dinfo "Install distro systemd"
1133
1134     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1135         install_debian_systemd
1136     elif get_bool "$LOOKS_LIKE_SUSE"; then
1137         install_suse_systemd
1138     else
1139         dfatal "NO_BUILD not supported for this distro"
1140         exit 1
1141     fi
1142 }
1143
1144 install_systemd() {
1145     dinfo "Install systemd"
1146     if get_bool "$NO_BUILD"; then
1147         install_distro_systemd
1148     else
1149         install_compiled_systemd
1150     fi
1151
1152     # remove unneeded documentation
1153     rm -fr "${initdir:?}"/usr/share/{man,doc}
1154
1155     # enable debug logging in PID1
1156     echo LogLevel=debug >>"$initdir/etc/systemd/system.conf"
1157     # store coredumps in journal
1158     echo Storage=journal >>"$initdir/etc/systemd/coredump.conf"
1159     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1160     mkdir "$initdir/etc/systemd/system/user@.service.d/"
1161     echo -e "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/override.conf"
1162
1163     # When built with gcov, disable ProtectSystem= and ProtectHome in the test
1164     # images, since it prevents gcov to write the coverage reports (*.gcda files)
1165     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1166         mkdir -p "$initdir/etc/systemd/system/service.d/"
1167         echo -e "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/gcov-override.conf"
1168     fi
1169 }
1170
1171 get_ldpath() {
1172     local rpath
1173     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1174
1175     if [ -z "$rpath" ] ; then
1176         echo "$BUILD_DIR"
1177     else
1178         echo "$rpath"
1179     fi
1180 }
1181
1182 install_missing_libraries() {
1183     dinfo "Install missing libraries"
1184     # install possible missing libraries
1185     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
1186         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1187     done
1188
1189     local lib path
1190     # A number of dependencies is now optional via dlopen, so the install
1191     # script will not pick them up, since it looks at linkage.
1192     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu libfido2 libbpf libelf libdw; do
1193         ddebug "Searching for $lib via pkg-config"
1194         if pkg-config --exists "$lib"; then
1195                 path="$(pkg-config --variable=libdir "$lib")"
1196                 if [ -z "${path}" ]; then
1197                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1198                     continue
1199                 fi
1200
1201                 if ! [[ ${lib} =~ ^lib ]]; then
1202                         lib="lib${lib}"
1203                 fi
1204                 # Some pkg-config files are broken and give out the wrong paths
1205                 # (eg: libcryptsetup), so just ignore them
1206                 inst_libs "${path}/${lib}.so" || true
1207                 inst_library "${path}/${lib}.so" || true
1208         else
1209             ddebug "$lib.pc not found, skipping"
1210             continue
1211         fi
1212     done
1213 }
1214
1215 cleanup_loopdev() {
1216     if [ -n "${LOOPDEV:=}" ]; then
1217         ddebug "losetup -d $LOOPDEV"
1218         losetup -d "${LOOPDEV}"
1219         unset LOOPDEV
1220     fi
1221 }
1222
1223 trap cleanup_loopdev EXIT INT QUIT PIPE
1224
1225 create_empty_image() {
1226     if [ -z "${IMAGE_NAME:=}" ]; then
1227         echo "create_empty_image: \$IMAGE_NAME not set"
1228         exit 1
1229     fi
1230
1231     local size=500
1232     if ! get_bool "$NO_BUILD"; then
1233         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1234             size=$((size+=200))
1235         fi
1236         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1237             size=$((size+=200))
1238         fi
1239         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1240             size=$((size+=250))
1241         fi
1242     fi
1243     if ! get_bool "$STRIP_BINARIES"; then
1244         size=$((4 * size))
1245     fi
1246
1247     echo "Setting up ${IMAGE_PUBLIC:?} (${size} MB)"
1248     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1249
1250     # Create the blank file to use as a root filesystem
1251     truncate -s "${size}M" "$IMAGE_PUBLIC"
1252
1253     LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
1254     [ -b "$LOOPDEV" ] || return 1
1255     sfdisk "$LOOPDEV" <<EOF
1256 ,$((size - 50))M
1257 ,
1258 EOF
1259
1260     udevadm settle
1261
1262     local label=(-L systemd_boot)
1263     # mkfs.reiserfs doesn't know -L. so, use --label instead
1264     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1265     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1266         dfatal "Failed to mkfs -t ${FSTYPE}"
1267         exit 1
1268     fi
1269 }
1270
1271 mount_initdir() {
1272     if [ -z "${LOOPDEV:=}" ]; then
1273         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1274         LOOPDEV="$(losetup --show -P -f "$image")"
1275         [ -b "$LOOPDEV" ] || return 1
1276
1277         udevadm settle
1278     fi
1279
1280     if ! mountpoint -q "${initdir:?}"; then
1281         mkdir -p "$initdir"
1282         mount "${LOOPDEV}p1" "$initdir"
1283         TEST_SETUP_CLEANUP_ROOTDIR=1
1284     fi
1285 }
1286
1287 cleanup_initdir() {
1288     # only umount if create_empty_image_rootdir() was called to mount it
1289     get_bool "$TEST_SETUP_CLEANUP_ROOTDIR" && _umount_dir "${initdir:?}"
1290 }
1291
1292 umount_loopback() {
1293     # unmount the loopback device from all places. Otherwise we risk file
1294     # system corruption.
1295     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1296         ddebug "Unmounting all uses of $device"
1297         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1298     done
1299 }
1300
1301 create_empty_image_rootdir() {
1302     create_empty_image
1303     mount_initdir
1304 }
1305
1306 check_asan_reports() {
1307     local ret=0
1308     local root="${1:?}"
1309
1310     if get_bool "$IS_BUILT_WITH_ASAN"; then
1311         ls -l "$root"
1312         if [[ -e "$root/systemd.asan.log.1" ]]; then
1313             cat "$root/systemd.asan.log.1"
1314             ret=$((ret+1))
1315         fi
1316
1317         journald_report="$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)"
1318         if [[ -n "$journald_report" ]]; then
1319             printf "%s\n" "$journald_report"
1320             cat "$root/systemd-journald.out" || :
1321             ret=$((ret+1))
1322         fi
1323
1324         pids="$(
1325             "$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
1326                  BEGIN {
1327                      %services_to_ignore = (
1328                          "dbus-daemon" => undef,
1329                      );
1330                  }
1331                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
1332         )"
1333         if [[ -n "$pids" ]]; then
1334             ret=$((ret+1))
1335             for pid in $pids; do
1336                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1337             done
1338         fi
1339     fi
1340
1341     return $ret
1342 }
1343
1344 check_coverage_reports() {
1345     local root="${1:?}"
1346
1347     if get_bool "$NO_BUILD"; then
1348         return 0
1349     fi
1350     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1351         return 0
1352     fi
1353
1354     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1355         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1356     else
1357         dest="${TESTDIR:?}/coverage-info"
1358     fi
1359
1360     # Create a coverage report that will later be uploaded. Remove info about
1361     # system libraries/headers, as we don't really care about them.
1362     if [[ -f "$dest" ]]; then
1363         # If the destination report file already exists, don't overwrite it, but
1364         # dump the new report in a temporary file and then merge it with the already
1365         # present one - this usually happens when running both "parts" of a test
1366         # in one run (the qemu and the nspawn part).
1367         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1368         lcov --remove "${dest}.new" -o "${dest}.new" '/usr/include/*' '/usr/lib/*'
1369         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1370         rm -f "${dest}.new"
1371     else
1372         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}"
1373         lcov --remove "${dest}" -o "${dest}" '/usr/include/*' '/usr/lib/*'
1374     fi
1375
1376     return 0
1377 }
1378
1379 save_journal() {
1380     # Default to always saving journal
1381     local save="yes"
1382
1383     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1384         save="no"
1385     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1386         save="no"
1387     fi
1388
1389     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1390         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1391     else
1392         dest="${TESTDIR:?}/system.journal"
1393     fi
1394
1395     for j in "${1:?}"/*; do
1396         if get_bool "$save"; then
1397             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1398         fi
1399
1400         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1401             echo "---- $j ----"
1402             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1403         fi
1404
1405         rm -r "$j"
1406     done
1407
1408     if ! get_bool "$save"; then
1409         return 0
1410     fi
1411
1412     if [ -n "${SUDO_USER}" ]; then
1413         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1414     fi
1415
1416     # we want to print this sometime later, so save this in a variable
1417     JOURNAL_LIST="$(ls -l "$dest"*)"
1418 }
1419
1420 check_result_common() {
1421     local workspace="${1:?}"
1422     local ret
1423
1424     if [ -s "$workspace/failed" ]; then
1425         # Non-empty …/failed has highest priority
1426         cp -a "$workspace/failed" "${TESTDIR:?}/"
1427         if [ -n "${SUDO_USER}" ]; then
1428             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1429         fi
1430         ret=1
1431     elif get_bool "$TIMED_OUT"; then
1432         echo "(timeout)" >"${TESTDIR:?}/failed"
1433         ret=2
1434     elif [ -e "$workspace/testok" ]; then
1435         # …/testok always counts (but with lower priority than …/failed)
1436         ret=0
1437     elif [ -e "$workspace/skipped" ]; then
1438         # …/skipped always counts (a message is expected)
1439         echo "${TESTNAME:?} was skipped:"
1440         cat "$workspace/skipped"
1441         ret=0
1442     else
1443         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1444         ret=3
1445     fi
1446
1447     check_asan_reports "$workspace" || ret=4
1448
1449     check_coverage_reports "$workspace" || ret=5
1450
1451     save_journal "$workspace/var/log/journal" $ret
1452
1453     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1454         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1455     fi
1456
1457     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1458         echo -n "${TESTNAME:?}: "
1459         cat "$TESTDIR/failed"
1460     fi
1461     echo "${JOURNAL_LIST:-"No journals were saved"}"
1462
1463     return ${ret:?}
1464 }
1465
1466 check_result_nspawn() {
1467     local workspace="${1:?}"
1468     local ret
1469
1470     check_result_common "${workspace}"
1471     ret=$?
1472
1473     # Run additional test-specific checks if defined by check_result_nspawn_hook()
1474     if declare -F check_result_nspawn_hook >/dev/null; then
1475         if ! check_result_nspawn_hook; then
1476             derror "check_result_nspawn_hook() returned with EC > 0"
1477             ret=4
1478         fi
1479     fi
1480
1481     _umount_dir "${initdir:?}"
1482
1483     return $ret
1484 }
1485
1486 # can be overridden in specific test
1487 check_result_qemu() {
1488     local ret
1489     mount_initdir
1490
1491     check_result_common "${initdir:?}"
1492     ret=$?
1493
1494     _umount_dir "${initdir:?}"
1495
1496     # Run additional test-specific checks if defined by check_result_qemu_hook()
1497     if declare -F check_result_qemu_hook >/dev/null; then
1498         if ! check_result_qemu_hook; then
1499             derror "check_result_qemu_hook() returned with EC > 0"
1500             ret=4
1501         fi
1502     fi
1503
1504     return $ret
1505 }
1506
1507 check_result_nspawn_unittests() {
1508     local workspace="${1:?}"
1509     local ret=1
1510
1511     [[ -e "$workspace/testok" ]] && ret=0
1512
1513     if [[ -s "$workspace/failed" ]]; then
1514         ret=$((ret + 1))
1515         echo "=== Failed test log ==="
1516         cat "$workspace/failed"
1517     else
1518         if [[ -s "$workspace/skipped" ]]; then
1519             echo "=== Skipped test log =="
1520             cat "$workspace/skipped"
1521             # We might have only skipped tests - that should not fail the job
1522             ret=0
1523         fi
1524         if [[ -s "$workspace/testok" ]]; then
1525             echo "=== Passed tests ==="
1526             cat "$workspace/testok"
1527         fi
1528     fi
1529
1530     get_bool "${TIMED_OUT:=}" && ret=1
1531     check_coverage_reports "$workspace" || ret=5
1532
1533     save_journal "$workspace/var/log/journal" $ret
1534
1535     _umount_dir "${initdir:?}"
1536
1537     return $ret
1538 }
1539
1540 check_result_qemu_unittests() {
1541     local ret=1
1542
1543     mount_initdir
1544     [[ -e "${initdir:?}/testok" ]] && ret=0
1545
1546     if [[ -s "$initdir/failed" ]]; then
1547         ret=$((ret + 1))
1548         echo "=== Failed test log ==="
1549         cat "$initdir/failed"
1550     else
1551         if [[ -s "$initdir/skipped" ]]; then
1552             echo "=== Skipped test log =="
1553             cat "$initdir/skipped"
1554             # We might have only skipped tests - that should not fail the job
1555             ret=0
1556         fi
1557         if [[ -s "$initdir/testok" ]]; then
1558             echo "=== Passed tests ==="
1559             cat "$initdir/testok"
1560         fi
1561     fi
1562
1563     get_bool "${TIMED_OUT:=}" && ret=1
1564     check_coverage_reports "$initdir" || ret=5
1565
1566     save_journal "$initdir/var/log/journal" $ret
1567
1568     _umount_dir "$initdir"
1569
1570     return $ret
1571 }
1572
1573 strip_binaries() {
1574     dinfo "Strip binaries"
1575     if ! get_bool "$STRIP_BINARIES"; then
1576         dinfo "STRIP_BINARIES == no, keeping binaries unstripped"
1577         return 0
1578     fi
1579     while read -r bin; do
1580         strip --strip-unneeded "$bin" |& grep -vi 'file format not recognized' | ddebug || :
1581     done < <(find "${initdir:?}" -executable -not -path '*/lib/modules/*.ko' -type f)
1582 }
1583
1584 create_rc_local() {
1585     dinfo "Create rc.local"
1586     mkdir -p "${initdir:?}/etc/rc.d"
1587     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1588 #!/usr/bin/env bash
1589 exit 0
1590 EOF
1591     chmod 0755 "$initdir/etc/rc.d/rc.local"
1592 }
1593
1594 install_execs() {
1595     ddebug "Install executables from the service files"
1596
1597     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1598     local systemunitdir userunitdir exe
1599     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1600     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1601     while read -r exe; do
1602         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1603         # also, plymouth is pulled in by rescue.service, but even there the exit code
1604         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1605         dinfo "Attempting to install $exe (based on unit file reference)"
1606         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1607     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1608 }
1609
1610 generate_module_dependencies() {
1611     dinfo "Generate modules dependencies"
1612     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1613         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1614             dfatal "\"depmod -a $KERNEL_VER\" failed."
1615             exit 1
1616     fi
1617 }
1618
1619 install_depmod_files() {
1620     dinfo "Install depmod files"
1621     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1622     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1623 }
1624
1625 install_plymouth() {
1626     dinfo "Install plymouth"
1627     # install plymouth, if found... else remove plymouth service files
1628     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1629     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1630     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1631     #         image_install plymouth plymouthd
1632     # else
1633         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1634     # fi
1635 }
1636
1637 install_haveged() {
1638     # If haveged is installed, it's probably included in initrd and needs to be
1639     # installed in the image too.
1640     if [ -x /usr/sbin/haveged ]; then
1641         dinfo "Install haveged files"
1642         inst /usr/sbin/haveged
1643         for u in /usr/lib/systemd/system/haveged*; do
1644             inst "$u"
1645         done
1646     fi
1647 }
1648
1649 install_ld_so_conf() {
1650     dinfo "Install /etc/ld.so.conf*"
1651     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1652     ldconfig -r "$initdir"
1653 }
1654
1655 install_testuser() {
1656     dinfo "Set up a test user"
1657     # create unprivileged user for user manager tests
1658     mkdir -p "${initdir:?}/etc/sysusers.d"
1659     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1660 u testuser    4711     "Test User" /home/testuser
1661 EOF
1662
1663     mkdir -p "$initdir/home/testuser"
1664     chmod 0700 "$initdir/home/testuser"
1665     chown 4711:4711 "$initdir/home/testuser"
1666 }
1667
1668 install_config_files() {
1669     dinfo "Install config files"
1670     inst /etc/sysconfig/init || :
1671     inst /etc/passwd
1672     inst /etc/shadow
1673     inst_any /etc/login.defs /usr/etc/login.defs
1674     inst /etc/group
1675     inst /etc/shells
1676     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
1677     inst /etc/pam.conf || :
1678     inst_any /etc/os-release /usr/lib/os-release
1679     inst /etc/localtime
1680     # we want an empty environment
1681     : >"${initdir:?}/etc/environment"
1682     : >"$initdir/etc/machine-id"
1683     : >"$initdir/etc/resolv.conf"
1684
1685     # set the hostname
1686     echo 'H' >"$initdir/etc/hostname"
1687
1688     # let's set up just one image with the traditional verbose output
1689     if [ "${IMAGE_NAME:?}" != "basic" ]; then
1690         mkdir -p "$initdir/etc/systemd/system.conf.d"
1691         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
1692     fi
1693 }
1694
1695 install_basic_tools() {
1696     dinfo "Install basic tools"
1697     image_install "${BASICTOOLS[@]}"
1698     image_install -o sushell
1699     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
1700     image_install -o ldconfig.real
1701 }
1702
1703 install_debug_tools() {
1704     dinfo "Install debug tools"
1705     image_install -o "${DEBUGTOOLS[@]}"
1706
1707     if get_bool "$INTERACTIVE_DEBUG"; then
1708         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
1709         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
1710         mkdir -p "$getty_override"
1711         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
1712
1713         cat >"$initdir/etc/motd" <<EOF
1714 To adjust the terminal size use:
1715     export COLUMNS=xx
1716     export LINES=yy
1717 or
1718     stty cols xx rows yy
1719 EOF
1720     fi
1721 }
1722
1723 install_libnss() {
1724     dinfo "Install libnss"
1725     # install libnss_files for login
1726     local NSS_LIBS
1727     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
1728     image_install "${NSS_LIBS[@]}"
1729 }
1730
1731 install_dbus() {
1732     dinfo "Install dbus"
1733     inst "${ROOTLIBDIR:?}/system/dbus.socket"
1734
1735     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1736     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
1737         inst "$ROOTLIBDIR/system/dbus-broker.service"
1738         inst_symlink /etc/systemd/system/dbus.service
1739         inst /usr/bin/dbus-broker
1740         inst /usr/bin/dbus-broker-launch
1741     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
1742         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1743         inst "$ROOTLIBDIR/system/dbus-daemon.service"
1744         # Alias symlink
1745         inst_symlink /etc/systemd/system/dbus.service
1746     else
1747         inst "$ROOTLIBDIR/system/dbus.service"
1748     fi
1749
1750     while read -r file; do
1751         inst "$file"
1752     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
1753
1754     # setup policy for Type=dbus test
1755     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
1756     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
1757 <?xml version="1.0"?>
1758 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1759         "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1760 <busconfig>
1761     <policy user="root">
1762         <allow own="systemd.test.ExecStopPost"/>
1763     </policy>
1764 </busconfig>
1765 EOF
1766 }
1767
1768 install_user_dbus() {
1769     dinfo "Install user dbus"
1770     local userunitdir
1771     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
1772         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
1773         userunitdir=/usr/lib/systemd/user
1774     fi
1775
1776     inst "$userunitdir/dbus.socket"
1777     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
1778
1779     # Append the After= dependency on dbus in case it isn't already set up
1780     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
1781     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
1782 [Unit]
1783 After=dbus.service
1784 EOF
1785
1786     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1787     if [ -f "$userunitdir/dbus-broker.service" ]; then
1788         inst "$userunitdir/dbus-broker.service"
1789         inst_symlink /etc/systemd/user/dbus.service
1790     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
1791         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1792         inst "$userunitdir/dbus-daemon.service"
1793         # Alias symlink
1794         inst_symlink /etc/systemd/user/dbus.service
1795     else
1796         inst "$userunitdir/dbus.service"
1797     fi
1798 }
1799
1800 install_pam() {
1801     dinfo "Install PAM"
1802     local paths=()
1803
1804     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
1805         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
1806     else
1807         paths+=(/lib*/security)
1808     fi
1809
1810     for d in /etc/pam.d /etc/security /usr/{etc,lib}/pam.d; do
1811         [ -d "$d" ] && paths+=("$d")
1812     done
1813
1814     while read -r file; do
1815         inst "$file"
1816     done < <(find "${paths[@]}" -xtype f)
1817
1818     # pam_unix depends on unix_chkpwd.
1819     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
1820     image_install -o unix_chkpwd
1821
1822     # set empty root password for easy debugging
1823     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
1824
1825     # And make sure pam_unix will accept it by making sure that
1826     # the PAM module has the nullok option.
1827     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
1828         [ -d "$initdir/$d" ] || continue
1829         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
1830     done
1831 }
1832
1833 # shellcheck disable=SC2120
1834 install_keymaps() {
1835     dinfo "Install keymaps"
1836     # The first three paths may be deprecated.
1837     # It seems now the last two paths are used by many distributions.
1838     for i in \
1839         /usr/lib/kbd/keymaps/include/* \
1840         /usr/lib/kbd/keymaps/i386/include/* \
1841         /usr/lib/kbd/keymaps/i386/qwerty/us.* \
1842         /usr/lib/kbd/keymaps/legacy/include/* \
1843         /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
1844             [[ -f "$i" ]] || continue
1845             inst "$i"
1846     done
1847
1848     # When it takes any argument, then install more keymaps.
1849     if [[ $# -gt 1 ]]; then
1850         for i in \
1851         /usr/lib/kbd/keymaps/i386/*/* \
1852         /usr/lib/kbd/keymaps/legacy/i386/*/*; do
1853             [[ -f "$i" ]] || continue
1854             inst "$i"
1855         done
1856     fi
1857 }
1858
1859 install_zoneinfo() {
1860     dinfo "Install time zones"
1861     inst_any /usr/share/zoneinfo/Asia/Seoul
1862     inst_any /usr/share/zoneinfo/Asia/Vladivostok
1863     inst_any /usr/share/zoneinfo/Australia/Sydney
1864     inst_any /usr/share/zoneinfo/Europe/Berlin
1865     inst_any /usr/share/zoneinfo/Europe/Dublin
1866     inst_any /usr/share/zoneinfo/Europe/Kiev
1867     inst_any /usr/share/zoneinfo/Pacific/Auckland
1868     inst_any /usr/share/zoneinfo/Pacific/Honolulu
1869     inst_any /usr/share/zoneinfo/CET
1870     inst_any /usr/share/zoneinfo/EET
1871     inst_any /usr/share/zoneinfo/UTC
1872 }
1873
1874 install_fonts() {
1875     dinfo "Install system fonts"
1876     for i in \
1877         /usr/lib/kbd/consolefonts/eurlatgr* \
1878         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
1879             [[ -f "$i" ]] || continue
1880             inst "$i"
1881     done
1882 }
1883
1884 install_terminfo() {
1885     dinfo "Install terminfo files"
1886     local terminfodir
1887     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
1888         [ -f "${terminfodir}/l/linux" ] && break
1889     done
1890     image_install -o "${terminfodir}/l/linux"
1891 }
1892
1893 has_user_dbus_socket() {
1894     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
1895         return 0
1896     else
1897         echo "Per-user instances are not supported. Skipping..."
1898         return 1
1899     fi
1900 }
1901
1902 setup_nspawn_root() {
1903     if [ -z "${initdir}" ]; then
1904         dfatal "\$initdir not defined"
1905         exit 1
1906     fi
1907
1908     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
1909
1910     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
1911         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
1912         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
1913     fi
1914 }
1915
1916 setup_basic_dirs() {
1917     mkdir -p "${initdir:?}/run"
1918     mkdir -p "$initdir/etc/systemd/system"
1919     mkdir -p "$initdir/var/log/journal"
1920
1921
1922     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
1923         if [ -L "/$d" ]; then
1924             inst_symlink "/$d"
1925         else
1926             inst_dir "/$d"
1927         fi
1928     done
1929
1930     ln -sfn /run "$initdir/var/run"
1931     ln -sfn /run/lock "$initdir/var/lock"
1932 }
1933
1934 mask_supporting_services() {
1935     # mask some services that we do not want to run in these tests
1936     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
1937     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
1938     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
1939     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
1940     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
1941 }
1942
1943 inst_libs() {
1944     local bin="${1:?}"
1945     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
1946     local file line
1947
1948     while read -r line; do
1949         [[ "$line" = 'not a dynamic executable' ]] && break
1950         # Skip a harmless error when running the tests on a system with a significantly
1951         # older systemd version (ldd tries to resolve the unprefixed RPATH for libsystemd.so.0,
1952         # which is in this case older than the already installed libsystemd.so.0 in $initdir).
1953         # The issue is triggered by installing test dependencies in install_missing_libraries().
1954         [[ "$line" =~ libsystemd.so.*:\ version\ .*\ not\ found ]] && continue
1955
1956         if [[ "$line" =~ $so_regex ]]; then
1957             file="${BASH_REMATCH[1]}"
1958             [[ -e "${initdir:?}/$file" ]] && continue
1959             inst_library "$file"
1960             continue
1961         fi
1962
1963         if [[ "$line" =~ not\ found ]]; then
1964             dfatal "Missing a shared library required by $bin."
1965             dfatal "Run \"ldd $bin\" to find out what it is."
1966             dfatal "$line"
1967             dfatal "Cannot create a test image."
1968             exit 1
1969         fi
1970     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
1971 }
1972
1973 import_testdir() {
1974     # make sure we don't get a stale LOOPDEV value from old times
1975     local _LOOPDEV="${LOOPDEV:=}"
1976     # We don't want shellcheck to follow & check the $STATEFILE
1977     # shellcheck source=/dev/null
1978     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
1979     LOOPDEV="$_LOOPDEV"
1980     if [[ ! -d "$TESTDIR" ]]; then
1981         if [[ -z "$TESTDIR" ]]; then
1982             TESTDIR="$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)"
1983         else
1984             mkdir -p "$TESTDIR"
1985         fi
1986
1987         cat >"$STATEFILE" <<EOF
1988 TESTDIR="$TESTDIR"
1989 EOF
1990         export TESTDIR
1991     fi
1992
1993     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
1994     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
1995 }
1996
1997 import_initdir() {
1998     initdir="${TESTDIR:?}/root"
1999     mkdir -p "$initdir"
2000     export initdir
2001 }
2002
2003 ## @brief Converts numeric logging level to the first letter of level name.
2004 #
2005 # @param lvl Numeric logging level in range from 1 to 6.
2006 # @retval 1 if @a lvl is out of range.
2007 # @retval 0 if @a lvl is correct.
2008 # @result Echoes first letter of level name.
2009 _lvl2char() {
2010     case "$1" in
2011         1) echo F;;
2012         2) echo E;;
2013         3) echo W;;
2014         4) echo I;;
2015         5) echo D;;
2016         6) echo T;;
2017         *) return 1;;
2018     esac
2019 }
2020
2021 ## @brief Internal helper function for _do_dlog()
2022 #
2023 # @param lvl Numeric logging level.
2024 # @param msg Message.
2025 # @retval 0 It's always returned, even if logging failed.
2026 #
2027 # @note This function is not supposed to be called manually. Please use
2028 # dtrace(), ddebug(), or others instead which wrap this one.
2029 #
2030 # This function calls _do_dlog() either with parameter msg, or if
2031 # none is given, it will read standard input and will use every line as
2032 # a message.
2033 #
2034 # This enables:
2035 # dwarn "This is a warning"
2036 # echo "This is a warning" | dwarn
2037 LOG_LEVEL="${LOG_LEVEL:-4}"
2038
2039 dlog() {
2040     local lvl lvlc
2041
2042     [ -z "$LOG_LEVEL" ] && return 0
2043     lvl="${1:?}"; shift
2044     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2045     lvlc="$(_lvl2char "$lvl")" || return 0
2046
2047     if [ $# -ge 1 ]; then
2048         echo "$lvlc: $*"
2049     else
2050         while read -r line; do
2051             echo "$lvlc: " "$line"
2052         done
2053     fi
2054 }
2055
2056 ## @brief Logs message at TRACE level (6)
2057 #
2058 # @param msg Message.
2059 # @retval 0 It's always returned, even if logging failed.
2060 dtrace() {
2061     set +x
2062     dlog 6 "$@"
2063     if get_bool "${debug:=}"; then
2064         set -x
2065     fi
2066 }
2067
2068 ## @brief Logs message at DEBUG level (5)
2069 #
2070 # @param msg Message.
2071 # @retval 0 It's always returned, even if logging failed.
2072 ddebug() {
2073     dlog 5 "$@"
2074 }
2075
2076 ## @brief Logs message at INFO level (4)
2077 #
2078 # @param msg Message.
2079 # @retval 0 It's always returned, even if logging failed.
2080 dinfo() {
2081     set +x
2082     dlog 4 "$@"
2083     if get_bool "${debug:=}"; then
2084         set -x
2085     fi
2086 }
2087
2088 ## @brief Logs message at WARN level (3)
2089 #
2090 # @param msg Message.
2091 # @retval 0 It's always returned, even if logging failed.
2092 dwarn() {
2093     set +x
2094     dlog 3 "$@"
2095     if get_bool "${debug:=}"; then
2096         set -x
2097     fi
2098 }
2099
2100 ## @brief Logs message at ERROR level (2)
2101 #
2102 # @param msg Message.
2103 # @retval 0 It's always returned, even if logging failed.
2104 derror() {
2105     dlog 2 "$@"
2106 }
2107
2108 ## @brief Logs message at FATAL level (1)
2109 #
2110 # @param msg Message.
2111 # @retval 0 It's always returned, even if logging failed.
2112 dfatal() {
2113     set +x
2114     dlog 1 "$@"
2115     if get_bool "${debug:=}"; then
2116         set -x
2117     fi
2118 }
2119
2120
2121 # Generic substring function.  If $2 is in $1, return 0.
2122 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2123
2124 # normalize_path <path>
2125 # Prints the normalized path, where it removes any duplicated
2126 # and trailing slashes.
2127 # Example:
2128 # $ normalize_path ///test/test//
2129 # /test/test
2130 normalize_path() {
2131     shopt -q -s extglob
2132     set -- "${1//+(\/)//}"
2133     shopt -q -u extglob
2134     echo "${1%/}"
2135 }
2136
2137 # convert_abs_rel <from> <to>
2138 # Prints the relative path, when creating a symlink to <to> from <from>.
2139 # Example:
2140 # $ convert_abs_rel /usr/bin/test /bin/test-2
2141 # ../../bin/test-2
2142 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2143 convert_abs_rel() {
2144     local __current __absolute __abssize __cursize __newpath
2145     local -i __i __level
2146
2147     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2148
2149     # corner case #1 - self looping link
2150     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2151
2152     # corner case #2 - own dir link
2153     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2154
2155     IFS="/" read -ra __current <<< "$1"
2156     IFS="/" read -ra __absolute <<< "$2"
2157
2158     __abssize=${#__absolute[@]}
2159     __cursize=${#__current[@]}
2160
2161     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]
2162     do
2163         (( __level++ ))
2164         if (( __level > __abssize || __level > __cursize ))
2165         then
2166             break
2167         fi
2168     done
2169
2170     for ((__i = __level; __i < __cursize-1; __i++))
2171     do
2172         if ((__i > __level))
2173         then
2174             __newpath=$__newpath"/"
2175         fi
2176         __newpath=$__newpath".."
2177     done
2178
2179     for ((__i = __level; __i < __abssize; __i++))
2180     do
2181         if [[ -n $__newpath ]]
2182         then
2183             __newpath=$__newpath"/"
2184         fi
2185         __newpath=$__newpath${__absolute[__i]}
2186     done
2187
2188     echo "$__newpath"
2189 }
2190
2191
2192 # Install a directory, keeping symlinks as on the original system.
2193 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2194 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2195 # and a symlink ${initdir}/lib -> lib64.
2196 inst_dir() {
2197     local dir="${1:?}"
2198     local part="${dir%/*}"
2199     local file
2200
2201     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2202
2203     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2204         dir="$part $dir"
2205         part="${part%/*}"
2206     done
2207
2208     # iterate over parent directories
2209     for file in $dir; do
2210         [[ -e "${initdir}/$file" ]] && continue
2211         if [[ -L $file ]]; then
2212             inst_symlink "$file"
2213         else
2214             # create directory
2215             mkdir -m 0755 "${initdir}/$file" || return 1
2216             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2217             chmod u+w "${initdir}/$file"
2218         fi
2219     done
2220 }
2221
2222 # $1 = file to copy to ramdisk
2223 # $2 (optional) Name for the file on the ramdisk
2224 # Location of the image dir is assumed to be $initdir
2225 # We never overwrite the target if it exists.
2226 inst_simple() {
2227     [[ -f "${1:?}" ]] || return 1
2228     strstr "$1" "/" || return 1
2229
2230     local src="$1"
2231     local target="${2:-$1}"
2232     if ! [[ -d ${initdir:?}/$target ]]; then
2233         [[ -e ${initdir}/$target ]] && return 0
2234         [[ -L ${initdir}/$target ]] && return 0
2235         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2236     fi
2237     # install checksum files also
2238     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2239         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2240     fi
2241     ddebug "Installing $src"
2242     cp --sparse=always -pfL "$src" "${initdir}/$target"
2243 }
2244
2245 # find symlinks linked to given library file
2246 # $1 = library file
2247 # Function searches for symlinks by stripping version numbers appended to
2248 # library filename, checks if it points to the same target and finally
2249 # prints the list of symlinks to stdout.
2250 #
2251 # Example:
2252 # rev_lib_symlinks libfoo.so.8.1
2253 # output: libfoo.so.8 libfoo.so
2254 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2255 rev_lib_symlinks() {
2256     local fn="${1:?}"
2257     local links=""
2258     local orig
2259     orig="$(readlink -f "$1")"
2260
2261     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2262
2263     until [[ "${fn##*.}" == so ]]; do
2264         fn="${fn%.*}"
2265         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2266     done
2267
2268     echo "${links}"
2269 }
2270
2271 # Same as above, but specialized to handle dynamic libraries.
2272 # It handles making symlinks according to how the original library
2273 # is referenced.
2274 inst_library() {
2275     local src="${1:?}"
2276     local dest="${2:-$1}"
2277     local reallib symlink
2278
2279     strstr "$1" "/" || return 1
2280     [[ -e ${initdir:?}/$dest ]] && return 0
2281     if [[ -L $src ]]; then
2282         # install checksum files also
2283         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2284             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2285         fi
2286         reallib="$(readlink -f "$src")"
2287         inst_simple "$reallib" "$reallib"
2288         inst_dir "${dest%/*}"
2289         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2290         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2291     else
2292         inst_simple "$src" "$dest"
2293     fi
2294
2295     # Create additional symlinks.  See rev_symlinks description.
2296     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2297         if [[ ! -e "$initdir/$symlink" ]]; then
2298             ddebug "Creating extra symlink: $symlink"
2299             inst_symlink "$symlink"
2300         fi
2301     done
2302 }
2303
2304 # find a binary.  If we were not passed the full path directly,
2305 # search in the usual places to find the binary.
2306 find_binary() {
2307     local bin="${1:?}"
2308     if [[ -z ${bin##/*} ]]; then
2309         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2310             echo "$bin"
2311             return 0
2312         fi
2313     fi
2314
2315     type -P "$bin"
2316 }
2317
2318 # Same as above, but specialized to install binary executables.
2319 # Install binary executable, and all shared library dependencies, if any.
2320 inst_binary() {
2321     local bin="${1:?}"
2322     local path target
2323
2324     # In certain cases we might attempt to install a binary which is already
2325     # present in the test image, yet it's missing from the host system.
2326     # In such cases, let's check if the binary indeed exists in the image
2327     # before doing any other chcecks. If it does, immediately return with
2328     # success.
2329     if [[ $# -eq 1 ]]; then
2330         for path in "" bin sbin usr/bin usr/sbin; do
2331             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2332         done
2333     fi
2334
2335     bin="$(find_binary "$bin")" || return 1
2336     target="${2:-$bin}"
2337     [[ -e "${initdir:?}/$target" ]] && return 0
2338     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2339
2340     local file line
2341     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2342     # I love bash!
2343     while read -r line; do
2344         [[ "$line" = 'not a dynamic executable' ]] && break
2345
2346         if [[ "$line" =~ $so_regex ]]; then
2347             file="${BASH_REMATCH[1]}"
2348             [[ -e "${initdir}/$file" ]] && continue
2349             inst_library "$file"
2350             continue
2351         fi
2352
2353         if [[ "$line" =~ not\ found ]]; then
2354             dfatal "Missing a shared library required by $bin."
2355             dfatal "Run \"ldd $bin\" to find out what it is."
2356             dfatal "$line"
2357             dfatal "Cannot create a test image."
2358             exit 1
2359         fi
2360     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2361     inst_simple "$bin" "$target"
2362 }
2363
2364 # same as above, except for shell scripts.
2365 # If your shell script does not start with shebang, it is not a shell script.
2366 inst_script() {
2367     local bin line shebang_regex
2368     bin="$(find_binary "${1:?}")" || return 1
2369     shift
2370
2371     read -r -n 80 line <"$bin"
2372     # If debug is set, clean unprintable chars to prevent messing up the term
2373     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2374     shebang_regex='(#! *)(/[^ ]+).*'
2375     [[ "$line" =~ $shebang_regex ]] || return 1
2376     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2377 }
2378
2379 # same as above, but specialized for symlinks
2380 inst_symlink() {
2381     local src="${1:?}"
2382     local target="${2:-$src}"
2383     local realsrc
2384
2385     strstr "$src" "/" || return 1
2386     [[ -L "$src" ]] || return 1
2387     [[ -L "${initdir:?}/$target" ]] && return 0
2388     realsrc="$(readlink -f "$src")"
2389     if ! [[ -e "$initdir/$realsrc" ]]; then
2390         if [[ -d "$realsrc" ]]; then
2391             inst_dir "$realsrc"
2392         else
2393             inst "$realsrc"
2394         fi
2395     fi
2396     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2397     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2398     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2399 }
2400
2401 # attempt to install any programs specified in a udev rule
2402 inst_rule_programs() {
2403     local rule="${1:?}"
2404     local prog bin
2405
2406     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2407         if [ -x "/lib/udev/$prog" ]; then
2408             bin="/lib/udev/$prog"
2409         else
2410             if ! bin="$(find_binary "$prog")"; then
2411                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2412                 continue
2413             fi
2414         fi
2415
2416         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2417         image_install "$bin"
2418     done
2419 }
2420
2421 # udev rules always get installed in the same place, so
2422 # create a function to install them to make life simpler.
2423 inst_rules() {
2424     local target=/etc/udev/rules.d
2425     local found rule
2426
2427     inst_dir "/lib/udev/rules.d"
2428     inst_dir "$target"
2429     for rule in "$@"; do
2430         if [ "${rule#/}" = "$rule" ]; then
2431             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2432                 if [[ -f "$r/$rule" ]]; then
2433                     found="$r/$rule"
2434                     inst_simple "$found"
2435                     inst_rule_programs "$found"
2436                 fi
2437             done
2438         fi
2439         for r in '' ./; do
2440             if [[ -f "${r}${rule}" ]]; then
2441                 found="${r}${rule}"
2442                 inst_simple "$found" "$target/${found##*/}"
2443                 inst_rule_programs "$found"
2444             fi
2445         done
2446         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2447         found=
2448     done
2449 }
2450
2451 # general purpose installation function
2452 # Same args as above.
2453 inst() {
2454     case $# in
2455         1) ;;
2456         2)
2457             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2458             [[ "$initdir" = "$2" ]] && set "$1"
2459             ;;
2460         3)
2461             [[ -z "$initdir" ]] && export initdir="$2"
2462             set "$1" "$3"
2463             ;;
2464         *)
2465             dfatal "inst only takes 1 or 2 or 3 arguments"
2466             exit 1
2467             ;;
2468     esac
2469
2470     local fun
2471     for fun in inst_symlink inst_script inst_binary inst_simple; do
2472         "$fun" "$@" && return 0
2473     done
2474
2475     dwarn "Failed to install '$1'"
2476     return 1
2477 }
2478
2479 # install any of listed files
2480 #
2481 # If first argument is '-d' and second some destination path, first accessible
2482 # source is installed into this path, otherwise it will installed in the same
2483 # path as source.  If none of listed files was installed, function return 1.
2484 # On first successful installation it returns with 0 status.
2485 #
2486 # Example:
2487 #
2488 # inst_any -d /bin/foo /bin/bar /bin/baz
2489 #
2490 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2491 # initramfs.
2492 inst_any() {
2493     local dest file
2494
2495     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2496
2497     for file in "$@"; do
2498         if [[ -e "$file" ]]; then
2499             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2500             inst "$file" && return 0
2501         fi
2502     done
2503
2504     return 1
2505 }
2506
2507 # image_install [-o ] <file> [<file> ... ]
2508 # Install <file> to the test image
2509 # -o optionally install the <file> and don't fail, if it is not there
2510 image_install() {
2511     local optional=no
2512     local prog="${1:?}"
2513
2514     if [[ "$prog" = '-o' ]]; then
2515         optional=yes
2516         shift
2517     fi
2518
2519     for prog in "$@"; do
2520         if ! inst "$prog" ; then
2521             if get_bool "$optional"; then
2522                 dinfo "Skipping program $prog as it cannot be found and is" \
2523                     "flagged to be optional"
2524             else
2525                 dfatal "Failed to install $prog"
2526                 exit 1
2527             fi
2528         fi
2529     done
2530 }
2531
2532 # Install a single kernel module along with any firmware it may require.
2533 # $1 = full path to kernel module to install
2534 install_kmod_with_fw() {
2535     local module="${1:?}"
2536     # no need to go further if the module is already installed
2537     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
2538     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
2539
2540     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
2541
2542     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
2543
2544     local modname="${module##*/}"
2545     local fwdir found fw
2546     modname="${modname%.ko*}"
2547
2548     while read -r fw; do
2549         found=
2550         for fwdir in /lib/firmware/updates /lib/firmware; do
2551             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
2552                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
2553                 found=yes
2554             fi
2555         done
2556         if ! get_bool "$found"; then
2557             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
2558                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
2559                     "\"${modname}.ko\""
2560             else
2561                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
2562                     "\"${modname}.ko\""
2563             fi
2564         fi
2565     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
2566     return 0
2567 }
2568
2569 # Do something with all the dependencies of a kernel module.
2570 # Note that kernel modules depend on themselves using the technique we use
2571 # $1 = function to call for each dependency we find
2572 #      It will be passed the full path to the found kernel module
2573 # $2 = module to get dependencies for
2574 # rest of args = arguments to modprobe
2575 for_each_kmod_dep() {
2576     local func="${1:?}"
2577     local kmod="${2:?}"
2578     local found=0
2579     local cmd modpath
2580     shift 2
2581
2582     while read -r cmd modpath _; do
2583         [[ "$cmd" = insmod ]] || continue
2584         "$func" "$modpath" || return $?
2585         found=1
2586     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
2587
2588     ! get_bool "$found" && return 1
2589     return 0
2590 }
2591
2592 # instmods [-c] <kernel module> [<kernel module> ... ]
2593 # instmods [-c] <kernel subsystem>
2594 # install kernel modules along with all their dependencies.
2595 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
2596 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
2597 instmods() {
2598     local check=no
2599     if [[ $# -ge 0 && "$1" = '-c' ]]; then
2600         check=yes
2601         shift
2602     fi
2603
2604     inst1mod() {
2605         local mod="${1:?}"
2606         local ret=0
2607         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
2608
2609         case "$mod" in
2610             =*)
2611                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
2612                     (
2613                         [[ "$mpargs" ]] && echo "$mpargs"
2614                         cat "${mod_dir}/modules.${mod#=}"
2615                     ) | instmods
2616                 else
2617                     (
2618                         [[ "$mpargs" ]] && echo "$mpargs"
2619                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
2620                     ) | instmods
2621                 fi
2622                 ;;
2623             --*)
2624                 mpargs+=" $mod"
2625                 ;;
2626             i2o_scsi)
2627                 # Do not load this diagnostic-only module
2628                 return
2629                 ;;
2630             *)
2631                 mod=${mod##*/}
2632                 # if we are already installed, skip this module and go on
2633                 # to the next one.
2634                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
2635
2636                 # We use '-d' option in modprobe only if modules prefix path
2637                 # differs from default '/'.  This allows us to use Dracut with
2638                 # old version of modprobe which doesn't have '-d' option.
2639                 local mod_dirname=${mod_dir%%/lib/modules/*}
2640                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
2641
2642                 # ok, load the module, all its dependencies, and any firmware
2643                 # it may require
2644                 for_each_kmod_dep install_kmod_with_fw "$mod" \
2645                     --set-version "$KERNEL_VER" \
2646                     ${mod_dirname:+"$mod_dirname"} \
2647                     ${mpargs:+"$mpargs"}
2648                 ((ret+=$?))
2649                 ;;
2650         esac
2651         return "$ret"
2652     }
2653
2654     local mod mpargs
2655
2656     if [[ $# -eq 0 ]]; then  # filenames from stdin
2657         while read -r mod; do
2658             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2659                 dfatal "Failed to install $mod"
2660                 return 1
2661             fi
2662         done
2663     fi
2664
2665     for mod in "$@"; do # filenames as arguments
2666         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2667             dfatal "Failed to install $mod"
2668             return 1
2669         fi
2670     done
2671
2672     return 0
2673 }
2674
2675 _umount_dir() {
2676     local mountpoint="${1:?}"
2677     if mountpoint -q "$mountpoint"; then
2678         ddebug "umount $mountpoint"
2679         umount "$mountpoint"
2680     fi
2681 }
2682
2683 # can be overridden in specific test
2684 test_setup_cleanup() {
2685     cleanup_initdir
2686 }
2687
2688 _test_cleanup() {
2689     # (post-test) cleanup should always ignore failure and cleanup as much as possible
2690     (
2691         set +e
2692         [[ -n "$initdir" ]] && _umount_dir "$initdir"
2693         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
2694         # If multiple setups/cleans are ran in parallel, this can cause a race
2695         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
2696             rm -vf "${IMAGESTATEDIR}/default.img"
2697         fi
2698         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
2699         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
2700     ) || :
2701 }
2702
2703 # can be overridden in specific test
2704 test_cleanup() {
2705     _test_cleanup
2706 }
2707
2708 test_cleanup_again() {
2709     [ -n "$TESTDIR" ] || return
2710     rm -rf "$TESTDIR/unprivileged-nspawn-root"
2711     [[ -n "$initdir" ]] && _umount_dir "$initdir"
2712 }
2713
2714 test_create_image() {
2715     create_empty_image_rootdir
2716
2717     # Create what will eventually be our root filesystem onto an overlay
2718     (
2719         LOG_LEVEL=5
2720         setup_basic_environment
2721     )
2722 }
2723
2724 test_setup() {
2725     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
2726             command -v meson >/dev/null && \
2727             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
2728         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
2729         exit 1
2730     fi
2731
2732     if [ -e "${IMAGE_PRIVATE:?}" ]; then
2733         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
2734         mount_initdir
2735     else
2736         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
2737             # default.img is the base that every test uses and optionally appends to
2738             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
2739                 # Create the backing public image, but then completely unmount
2740                 # it and drop the loopback device responsible for it, since we're
2741                 # going to symlink/copy the image and mount it again from
2742                 # elsewhere.
2743                 local image_old="${IMAGE_PUBLIC}"
2744                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
2745                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
2746                 fi
2747                 test_create_image
2748                 test_setup_cleanup
2749                 umount_loopback
2750                 cleanup_loopdev
2751                 IMAGE_PUBLIC="${image_old}"
2752             fi
2753             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
2754                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
2755             fi
2756         fi
2757
2758         local hook_defined
2759         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
2760
2761         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
2762         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
2763             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2764         else
2765             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2766         fi
2767
2768         mount_initdir
2769         # We want to test all services in TEST-01-BASIC, but mask them in
2770         # all other tests
2771         if [[ "${TESTID:?}" != "01" ]]; then
2772             dinfo "Masking supporting services"
2773             mask_supporting_services
2774         fi
2775
2776         if get_bool "$hook_defined"; then
2777             test_append_files "${initdir:?}"
2778         fi
2779     fi
2780
2781     setup_nspawn_root
2782 }
2783
2784 test_run() {
2785     local test_id="${1:?}"
2786     mount_initdir
2787
2788     if ! get_bool "${TEST_NO_QEMU:=}"; then
2789         if run_qemu "$test_id"; then
2790             check_result_qemu || { echo "QEMU test failed"; return 1; }
2791         else
2792             dwarn "can't run QEMU, skipping"
2793         fi
2794     fi
2795     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
2796         mount_initdir
2797         if run_nspawn "${initdir:?}" "$test_id"; then
2798             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
2799         else
2800             dwarn "can't run systemd-nspawn, skipping"
2801         fi
2802
2803         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
2804             dir="$TESTDIR/unprivileged-nspawn-root"
2805             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
2806                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
2807             else
2808                 dwarn "can't run systemd-nspawn, skipping"
2809             fi
2810         fi
2811     fi
2812     return 0
2813 }
2814
2815 do_test() {
2816     if [[ $UID != "0" ]]; then
2817         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
2818         exit 0
2819     fi
2820
2821     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
2822         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both QEMU and nspawn disabled" >&2
2823         exit 0
2824     fi
2825
2826     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2827         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: QEMU-only tests requested" >&2
2828         exit 0
2829     fi
2830
2831     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2832         TEST_NO_QEMU=1
2833     fi
2834
2835     # Detect lib paths
2836     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
2837         [[ -d $libdir ]] && libdirs+=" $libdir" && break
2838     done
2839
2840     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
2841         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
2842     done
2843
2844     mkdir -p "$STATEDIR"
2845
2846     import_testdir
2847     import_initdir
2848
2849     if [ -n "${SUDO_USER}" ]; then
2850         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
2851         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
2852     fi
2853
2854     testname="$(basename "$PWD")"
2855
2856     while (($# > 0)); do
2857         case $1 in
2858             --run)
2859                 echo "${testname} RUN: $TEST_DESCRIPTION"
2860                 test_run "$TESTID"
2861                 ret=$?
2862                 if [ $ret -eq 0 ]; then
2863                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
2864                 else
2865                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
2866                 fi
2867                 exit $ret
2868                 ;;
2869             --setup)
2870                 echo "${testname} SETUP: $TEST_DESCRIPTION"
2871                 test_setup
2872                 test_setup_cleanup
2873                 ;;
2874             --clean)
2875                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
2876                 test_cleanup
2877                 ;;
2878             --clean-again)
2879                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
2880                 test_cleanup_again
2881                 ;;
2882             --all)
2883                 ret=0
2884                 echo -n "${testname}: $TEST_DESCRIPTION "
2885                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
2886                 # and loop devices will leak
2887                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
2888                 if [ $ret -eq 0 ]; then
2889                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2890                 fi
2891                 if [ $ret -eq 0 ]; then
2892                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2893                 fi
2894                 test_cleanup
2895                 if [ $ret -eq 0 ]; then
2896                     rm "$TESTLOG"
2897                     echo "[OK]"
2898                 else
2899                     echo "[FAILED]"
2900                     echo "see $TESTLOG"
2901                 fi
2902                 exit $ret
2903                 ;;
2904             *)
2905                 break
2906                 ;;
2907         esac
2908         shift
2909     done
2910 }