test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 QEMU_MEM="${QEMU_MEM:-512M}"
  34 # Note that defining a different IMAGE_NAME in a test setup script will only result
  35 # in default.img being copied and renamed. It can then be extended by defining
  36 # a test_append_files() function. The $1 parameter will be the root directory.
  37 # To force creating a new image from scratch (eg: to encrypt it), also define
  38 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  39 IMAGE_NAME=${IMAGE_NAME:-default}
  40 STRIP_BINARIES="${STRIP_BINARIES:-yes}"
  41 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  42 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  43 LOOPDEV=
  44
  45 # Simple wrapper to unify boolean checks.
  46 # Note: this function needs to stay near the top of the file, so we can use it
  47 #       in code in the outermost scope.
  48 get_bool() {
  49     # Make the value lowercase to make the regex matching simpler
  50     local _bool="${1,,}"
  51
  52     # Consider empty value as "false"
  53     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  54         return 1
  55     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  56         return 0
  57     else
  58         echo >&2 "Value '$_bool' is not a valid boolean value"
  59         exit 1
  60     fi
  61 }
  62
  63 # Decide if we can (and want to) run qemu with KVM acceleration.
  64 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  65 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  66 # running under KVM. If these conditions are met, enable KVM (and possibly
  67 # nested KVM), otherwise disable it.
  68 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && [[ "$(systemd-detect-virt -v)" != kvm ]]); then
  69     QEMU_KVM=yes
  70 else
  71     QEMU_KVM=no
  72 fi
  73
  74 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  75     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
  76     ROOTLIBDIR=/usr/lib/systemd
  77 fi
  78
  79 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
  80 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
  81 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
  82 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
  83 TOOLS_DIR="$SOURCE_DIR/tools"
  84 # These variables are used by test scripts
  85 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR TOOLS_DIR
  86
  87 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
  88 if get_bool "${NO_BUILD:=}"; then
  89     BUILD_DIR="$SOURCE_DIR"
  90 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
  91     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
  92     exit 1
  93 fi
  94
  95 PATH_TO_INIT="$ROOTLIBDIR/systemd"
  96 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
  97 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
  98 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
  99 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 100 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 101
 102 TESTFILE="${BASH_SOURCE[1]}"
 103 if [ -z "$TESTFILE" ]; then
 104     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 105     exit 1
 106 fi
 107 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 108 STATEDIR="$BUILD_DIR/test/$TESTNAME"
 109 STATEFILE="$STATEDIR/.testdir"
 110 IMAGESTATEDIR="$STATEDIR/.."
 111 TESTLOG="$STATEDIR/test.log"
 112
 113 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 114     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 115     exit 1
 116 fi
 117 TESTID="${BASH_REMATCH[1]:?}"
 118
 119 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 120     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 121     exit 1
 122 fi
 123
 124 BASICTOOLS=(
 125     awk
 126     base64
 127     basename
 128     bash
 129     capsh
 130     cat
 131     chmod
 132     chown
 133     cmp
 134     cryptsetup
 135     cut
 136     date
 137     dd
 138     diff
 139     dirname
 140     dmsetup
 141     echo
 142     env
 143     false
 144     flock
 145     getconf
 146     getent
 147     getfacl
 148     grep
 149     gunzip
 150     gzip
 151     head
 152     ionice
 153     ip
 154     ldd
 155     ln
 156     loadkeys
 157     login
 158     losetup
 159     lz4cat
 160     mkfifo
 161     mktemp
 162     modprobe
 163     mount
 164     mountpoint
 165     mv
 166     nc
 167     nproc
 168     readlink
 169     rev
 170     rm
 171     rmdir
 172     sed
 173     seq
 174     setfattr
 175     setfont
 176     setsid
 177     sfdisk
 178     sh
 179     sleep
 180     stat
 181     su
 182     sulogin
 183     sysctl
 184     tail
 185     tar
 186     tee
 187     test
 188     timeout
 189     touch
 190     tr
 191     true
 192     truncate
 193     umount
 194     uname
 195     unshare
 196     wc
 197     xargs
 198     xzcat
 199 )
 200
 201 DEBUGTOOLS=(
 202     cp
 203     df
 204     dhclient
 205     dmesg
 206     du
 207     find
 208     free
 209     grep
 210     hostname
 211     id
 212     less
 213     ln
 214     ls
 215     mkdir
 216     ping
 217     ps
 218     route
 219     sort
 220     strace
 221     stty
 222     tty
 223     vi
 224 )
 225
 226 is_built_with_asan() {
 227     if ! type -P objdump >/dev/null; then
 228         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 229         return 1
 230     fi
 231
 232     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 233     local _asan_calls
 234     _asan_calls="$(objdump -dC "$SYSTEMD_JOURNALD" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 235     if ((_asan_calls < 1000)); then
 236         return 1
 237     else
 238         return 0
 239     fi
 240 }
 241
 242 is_built_with_coverage() {
 243     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 244         return 1
 245     fi
 246
 247     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 248 }
 249
 250 IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
 251 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 252
 253 if get_bool "$IS_BUILT_WITH_ASAN"; then
 254     STRIP_BINARIES=no
 255     SKIP_INITRD="${SKIP_INITRD:-yes}"
 256     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
 257     QEMU_MEM="2048M"
 258     QEMU_SMP="${QEMU_SMP:-4}"
 259
 260     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 261     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 262         ASAN_COMPILER=gcc
 263         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 264     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 265         ASAN_COMPILER=clang
 266         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 267
 268         # As clang's ASan DSO is usually in a non-standard path, let's check if
 269         # the environment is set accordingly. If not, warn the user and exit.
 270         # We're not setting the LD_LIBRARY_PATH automagically here, because
 271         # user should encounter (and fix) the same issue when running the unit
 272         # tests (meson test)
 273         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 274             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 275             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 276             exit 1
 277         fi
 278     else
 279         echo >&2 "systemd is not linked against the ASan DSO"
 280         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 281         exit 1
 282     fi
 283
 284     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 285 fi
 286
 287 find_qemu_bin() {
 288     QEMU_BIN="${QEMU_BIN:-""}"
 289     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 290     if get_bool "$QEMU_KVM"; then
 291         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 292     fi
 293
 294     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 295     case $ARCH in
 296     x86_64)
 297         # QEMU's own build system calls it qemu-system-x86_64
 298         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 299         ;;
 300     i*86)
 301         # new i386 version of QEMU
 302         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 303
 304         # i386 version of QEMU
 305         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 306         ;;
 307     ppc64*)
 308         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 309         ;;
 310     esac
 311
 312     if [[ ! -e "$QEMU_BIN" ]]; then
 313         echo "Could not find a suitable qemu binary" >&2
 314         return 1
 315     fi
 316 }
 317
 318 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 319 # returns 0 if newer or equal
 320 # returns 1 if older
 321 # returns 2 if failing
 322 qemu_min_version() {
 323     find_qemu_bin || return 2
 324
 325     # get version from binary
 326     local qemu_ver
 327     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 328
 329     # Check version string format
 330     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 331     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 332
 333     # compare as last command to return that value
 334     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 335 }
 336
 337 # Return 0 if qemu did run (then you must check the result state/logs for actual
 338 # success), or 1 if qemu is not available.
 339 run_qemu() {
 340     if [ -f /etc/machine-id ]; then
 341         read -r MACHINE_ID </etc/machine-id
 342         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 343             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 344         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 345             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 346     fi
 347
 348     local CONSOLE=ttyS0
 349
 350     rm -f "$initdir"/{testok,failed,skipped}
 351     # make sure the initdir is not mounted to avoid concurrent access
 352     cleanup_initdir
 353     umount_loopback
 354
 355     if [[ ! "$KERNEL_BIN" ]]; then
 356         if get_bool "$LOOKS_LIKE_ARCH"; then
 357             KERNEL_BIN=/boot/vmlinuz-linux
 358         else
 359             [ "$ARCH" ] || ARCH=$(uname -m)
 360             case $ARCH in
 361                 ppc64*)
 362                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 363                     # uses the "standard" vmlinuz- prefix
 364                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 365                     CONSOLE=hvc0
 366                     ;;
 367                 *)
 368                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 369                     ;;
 370             esac
 371         fi
 372     fi
 373
 374     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 375     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 376     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 377     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 378     if [[ ! "$INITRD" ]]; then
 379         if [[ -e "$default_fedora_initrd" ]]; then
 380             INITRD="$default_fedora_initrd"
 381         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 382             INITRD="$default_debian_initrd"
 383         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 384             INITRD="$default_arch_initrd"
 385         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 386             INITRD="$default_suse_initrd"
 387         fi
 388     fi
 389
 390     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 391     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 392     # is not installed or there's some other error when calling it, fall back
 393     # to the original value (QEMU_SMP=1).
 394     if [[ -z "${QEMU_SMP:=}" ]]; then
 395         if ! QEMU_SMP=$(nproc); then
 396             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 397             QEMU_SMP=1
 398         fi
 399     fi
 400
 401     find_qemu_bin || return 1
 402
 403     # Umount initdir to avoid concurrent access to the filesystem
 404     _umount_dir "$initdir"
 405
 406     local kernel_params=()
 407     local qemu_options=()
 408     local qemu_cmd=("$QEMU_BIN")
 409
 410     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 411         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 412     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 413         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 414     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 415         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 416     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 417         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 418         exit 1
 419     fi
 420
 421     if get_bool "$LOOKS_LIKE_SUSE"; then
 422         kernel_params+=("rd.hostonly=0")
 423     fi
 424
 425     kernel_params+=(
 426         "root=LABEL=systemd_boot"
 427         "rw"
 428         "raid=noautodetect"
 429         "rd.luks=0"
 430         "loglevel=2"
 431         "init=$PATH_TO_INIT"
 432         "console=$CONSOLE"
 433         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 434         "systemd.unit=testsuite.target"
 435         "systemd.wants=testsuite-$1.service"
 436     )
 437
 438     if ! get_bool "$INTERACTIVE_DEBUG"; then
 439         kernel_params+=("systemd.wants=end.service")
 440     fi
 441
 442     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 443     qemu_options+=(
 444         -smp "$QEMU_SMP"
 445         -net none
 446         -m "$QEMU_MEM"
 447         -nographic
 448         -kernel "$KERNEL_BIN"
 449         -drive "format=raw,cache=unsafe,file=$image"
 450     )
 451
 452     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 453         local user_qemu_options
 454         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 455         qemu_options+=("${user_qemu_options[@]}")
 456     fi
 457
 458     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 459         local user_kernel_append
 460         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 461         kernel_params+=("${user_kernel_append[@]}")
 462     fi
 463
 464     if [[ "$INITRD" ]] && ! get_bool "$SKIP_INITRD"; then
 465         qemu_options+=(-initrd "$INITRD")
 466     fi
 467
 468     # Let's use KVM if possible
 469     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 470         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 471     fi
 472
 473     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 474         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 475     fi
 476
 477     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}")
 478     rc=$?
 479     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 480         derror "Test timed out after ${QEMU_TIMEOUT}s"
 481         TIMED_OUT=1
 482     else
 483         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 484     fi
 485     return 0
 486 }
 487
 488 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 489 # success), or 1 if nspawn is not available.
 490 run_nspawn() {
 491     [[ -d /run/systemd/system ]] || return 1
 492     rm -f "${initdir:?}"/{testok,failed,skipped}
 493
 494     local nspawn_cmd=()
 495     local nspawn_options=(
 496         "--register=no"
 497         "--kill-signal=SIGKILL"
 498         "--directory=${1:?}"
 499         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 500     )
 501     local kernel_params=(
 502         "$PATH_TO_INIT"
 503         "systemd.unit=testsuite.target"
 504         "systemd.wants=testsuite-$2.service"
 505     )
 506
 507     if ! get_bool "$INTERACTIVE_DEBUG"; then
 508         kernel_params+=("systemd.wants=end.service")
 509     fi
 510
 511     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 512         local user_nspawn_arguments
 513         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 514         nspawn_options+=("${user_nspawn_arguments[@]}")
 515     fi
 516
 517     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 518         local user_kernel_append
 519         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 520         kernel_params+=("${user_kernel_append[@]}")
 521     fi
 522
 523     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 524         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 525         exit
 526     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 527         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 528     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 529         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 530     else
 531         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 532         exit 1
 533     fi
 534
 535     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 536         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 537     else
 538         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 539     fi
 540
 541     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}")
 542     rc=$?
 543     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 544         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 545         TIMED_OUT=1
 546     else
 547         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 548     fi
 549     return 0
 550 }
 551
 552 # Build two very minimal root images, with two units, one is the same and one is different across them
 553 install_verity_minimal() {
 554     dinfo "Set up a set of minimal images for verity verification"
 555     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 556         return
 557     fi
 558     if ! command -v mksquashfs >/dev/null 2>&1; then
 559         dfatal "mksquashfs not found"
 560         exit 1
 561     fi
 562     if ! command -v veritysetup >/dev/null 2>&1; then
 563         dfatal "veritysetup not found"
 564         exit 1
 565     fi
 566     # Local modifications of some global variables is intentional in this
 567     # subshell (SC2030)
 568     # shellcheck disable=SC2030
 569     (
 570         BASICTOOLS=(
 571             bash
 572             cat
 573             grep
 574             mount
 575             sleep
 576         )
 577         oldinitdir="$initdir"
 578         rm -rfv "$TESTDIR/minimal"
 579         export initdir="$TESTDIR/minimal"
 580         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 581         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 582         setup_basic_dirs
 583         install_basic_tools
 584         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 585         # issue, thus falsely triggering SC2030 in this case
 586         # See: koalaman/shellcheck#1409
 587         if [[ -v ASAN_RT_PATH ]]; then
 588             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 589             # into the verity images to get rid of the annoying errors about
 590             # missing $LD_PRELOAD libraries.
 591             inst_libs "$ASAN_RT_PATH"
 592             inst_library "$ASAN_RT_PATH"
 593             # Create a dummy LSan suppression file otherwise gcc's ASan
 594             # complains as it doesn't exist in the minimal image
 595             # (i.e. when running TEST-29 or TEST-50 under sanitizers)
 596             touch "$initdir/systemd-lsan.supp"
 597         fi
 598         cp "$os_release" "$initdir/usr/lib/os-release"
 599         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 600         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 601         touch "$initdir/opt/some_file"
 602         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 603         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 604         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 605 [Service]
 606 ExecStartPre=cat /usr/lib/os-release
 607 ExecStart=sleep 120
 608 EOF
 609         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 610
 611         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 612         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 613             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 614
 615         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 616         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 617         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 618
 619         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 620         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 621             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 622
 623         # Rolling distros like Arch do not set VERSION_ID
 624         local version_id=""
 625         if grep -q "^VERSION_ID=" "$os_release"; then
 626             version_id="$(grep "^VERSION_ID=" "$os_release")"
 627         fi
 628
 629         export initdir="$TESTDIR/app0"
 630         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 631         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 632         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 633         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 634 [Service]
 635 Type=oneshot
 636 RemainAfterExit=yes
 637 ExecStart=/opt/script0.sh
 638 TemporaryFileSystem=/var/lib
 639 StateDirectory=app0
 640 RuntimeDirectory=app0
 641 EOF
 642         cat >"$initdir/opt/script0.sh" <<EOF
 643 #!/bin/bash
 644 set -e
 645 test -e /usr/lib/os-release
 646 echo bar > \${STATE_DIRECTORY}/foo
 647 cat /usr/lib/extension-release.d/extension-release.app0
 648 EOF
 649         chmod +x "$initdir/opt/script0.sh"
 650         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 651         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 652
 653         export initdir="$TESTDIR/app1"
 654         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 655         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 656         ( echo "${version_id}"
 657           echo "SYSEXT_SCOPE=portable"
 658           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 659         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 660         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 661 [Service]
 662 Type=oneshot
 663 RemainAfterExit=yes
 664 ExecStart=/opt/script1.sh
 665 StateDirectory=app1
 666 RuntimeDirectory=app1
 667 EOF
 668         cat >"$initdir/opt/script1.sh" <<EOF
 669 #!/bin/bash
 670 set -e
 671 test -e /usr/lib/os-release
 672 echo baz > \${STATE_DIRECTORY}/foo
 673 cat /usr/lib/extension-release.d/extension-release.app2
 674 EOF
 675         chmod +x "$initdir/opt/script1.sh"
 676         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 677         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 678     )
 679 }
 680
 681 setup_basic_environment() {
 682     # create the basic filesystem layout
 683     setup_basic_dirs
 684
 685     install_systemd
 686     install_missing_libraries
 687     install_config_files
 688     install_zoneinfo
 689     create_rc_local
 690     install_basic_tools
 691     install_libnss
 692     install_pam
 693     install_dbus
 694     install_fonts
 695     install_keymaps
 696     install_terminfo
 697     install_execs
 698     install_fs_tools
 699     install_modules
 700     install_plymouth
 701     install_haveged
 702     install_debug_tools
 703     install_ld_so_conf
 704     install_testuser
 705     has_user_dbus_socket && install_user_dbus
 706     setup_selinux
 707     strip_binaries
 708     instmods veth
 709     install_depmod_files
 710     generate_module_dependencies
 711     if get_bool "$IS_BUILT_WITH_ASAN"; then
 712         create_asan_wrapper
 713     fi
 714     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 715         install_verity_minimal
 716     fi
 717 }
 718
 719 setup_selinux() {
 720     dinfo "Setup SELinux"
 721     # don't forget KERNEL_APPEND='... selinux=1 ...'
 722     if ! get_bool "$SETUP_SELINUX"; then
 723         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 724         return 0
 725     fi
 726
 727     local conf_dir=/etc/selinux
 728     local fixfiles_tools=(bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles)
 729
 730     # Make sure the following statement can't expand to "/" to prevent
 731     # a potential where-are-my-backups situation
 732     rm -rf "${initdir:?}/$conf_dir"
 733     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 734         dfatal "Failed to copy $conf_dir"
 735         exit 1
 736     fi
 737
 738     touch "$initdir/.autorelabel"
 739     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 740     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 741
 742     image_install "${fixfiles_tools[@]}"
 743     image_install fixfiles
 744     image_install sestatus
 745 }
 746
 747 install_valgrind() {
 748     if ! type -p valgrind; then
 749         dfatal "Failed to install valgrind"
 750         exit 1
 751     fi
 752
 753     local valgrind_bins valgrind_libs valgrind_dbg_and_supp
 754
 755     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
 756     image_install "${valgrind_bins[@]}"
 757
 758     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
 759     image_install "${valgrind_libs[@]}"
 760
 761     readarray -t valgrind_dbg_and_supp < <(
 762         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 763         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 764     )
 765     image_install "${valgrind_dbg_and_supp[@]}"
 766 }
 767
 768 create_valgrind_wrapper() {
 769     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 770     ddebug "Create $valgrind_wrapper"
 771     cat >"$valgrind_wrapper" <<EOF
 772 #!/usr/bin/env bash
 773
 774 mount -t proc proc /proc
 775 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 776 EOF
 777     chmod 0755 "$valgrind_wrapper"
 778 }
 779
 780 create_asan_wrapper() {
 781     local asan_wrapper="$initdir/$ROOTLIBDIR/systemd-under-asan"
 782     dinfo "Create ASan wrapper as '$asan_wrapper'"
 783
 784     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 785
 786     # clang: install llvm-symbolizer to generate useful reports
 787     # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 788     [[ "$ASAN_COMPILER" == "clang" ]] && image_install "llvm-symbolizer"
 789
 790     cat >"$asan_wrapper" <<EOF
 791 #!/usr/bin/env bash
 792
 793 set -x
 794
 795 echo "ASan RT: $ASAN_RT_PATH"
 796 if [[ ! -e "$ASAN_RT_PATH" ]]; then
 797     echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
 798     exit 1
 799 fi
 800
 801 # Suppress certain leaks reported by LSan (either in external tools or bogus
 802 # ones)
 803 # Docs: # https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer#suppressions
 804 #
 805 # - fsck is called by systemd-homed and is reporting a leak we're not interested
 806 #   in
 807 # - libLLVM is a "side effect" caused by the previous fsck leak
 808 cat >/systemd-lsan.supp <<INNER_EOF
 809 leak:/bin/fsck$
 810 leak:/sbin/fsck$
 811 leak:/lib/libLLVM
 812 INNER_EOF
 813
 814 DEFAULT_LSAN_OPTIONS=${LSAN_OPTIONS:-}:suppressions=/systemd-lsan.supp
 815 DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
 816 DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
 817 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS LSAN_OPTIONS=\$DEFAULT_LSAN_OPTIONS"
 818
 819 # As right now bash is the PID 1, we can't expect PATH to have a sane value.
 820 # Let's make one to prevent unexpected "<bin> not found" issues in the future
 821 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 822
 823 mount -t proc proc /proc
 824 mount -t sysfs sysfs /sys
 825 mount -o remount,rw /
 826
 827 # A lot of services (most notably dbus) won't start without preloading libasan
 828 # See https://github.com/systemd/systemd/issues/5004
 829 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=$ASAN_RT_PATH"
 830
 831 if [[ "$ASAN_COMPILER" == "clang" ]]; then
 832   # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
 833   # unnecessary for gcc & libasan, however, for clang this is crucial, as its
 834   # runtime ASan DSO is in a non-standard (library) path.
 835   echo "${ASAN_RT_PATH%/*}" >/etc/ld.so.conf.d/asan-path-override.conf
 836   ldconfig
 837 fi
 838 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 839 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 840 echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
 841
 842 # ASAN and syscall filters aren't compatible with each other.
 843 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 844
 845 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 846 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 847 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 848 mkdir -p "\$JOURNALD_CONF_DIR"
 849 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 850
 851 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 852 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 853 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 854 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 855
 856 # 90s isn't enough for some services to finish when literally everything is run
 857 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 858 # Let's limit which environments such services should be executed in.
 859 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 860 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 861
 862 # Let's override another hard-coded timeout that kicks in too early
 863 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 864 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 865
 866 # D-Bus has troubles during system shutdown causing it to fail. Although it's
 867 # harmless, it causes unnecessary noise in the logs, so let's disable LSan's
 868 # at_exit check just for the dbus.service
 869 mkdir -p /etc/systemd/system/dbus.service.d
 870 printf "[Service]\nEnvironment=ASAN_OPTIONS=leak_check_at_exit=false\n" >/etc/systemd/system/dbus.service.d/disable-lsan.conf
 871
 872 # Some utilities run via IMPORT/RUN/PROGRAM udev directives fail because
 873 # they're uninstrumented (like dmsetup). Let's add a simple rule which sets
 874 # LD_PRELOAD to the ASan RT library to fix this.
 875 mkdir -p /etc/udev/rules.d
 876 cat >/etc/udev/rules.d/00-set-LD_PRELOAD.rules <<INNER_EOF
 877 SUBSYSTEM=="block", ENV{LD_PRELOAD}="$ASAN_RT_PATH"
 878 INNER_EOF
 879 chmod 0644 /etc/udev/rules.d/00-set-LD_PRELOAD.rules
 880
 881 # The 'mount' utility doesn't behave well under libasan, causing unexpected
 882 # fails during boot and subsequent test results check:
 883 # bash-5.0# mount -o remount,rw -v /
 884 # mount: /dev/sda1 mounted on /.
 885 # bash-5.0# echo \$?
 886 # 1
 887 # Let's workaround this by clearing the previously set LD_PRELOAD env variable,
 888 # so the libasan library is not loaded for this particular service
 889 unset_ld_preload() {
 890     local _dropin_dir="/etc/systemd/system/\$1.service.d"
 891     mkdir -p "\$_dropin_dir"
 892     printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$_dropin_dir/unset_ld_preload.conf"
 893 }
 894
 895 unset_ld_preload systemd-remount-fs
 896 unset_ld_preload testsuite-
 897
 898 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 899 exec "$ROOTLIBDIR/systemd" "\$@"
 900 EOF
 901
 902     chmod 0755 "$asan_wrapper"
 903 }
 904
 905 create_strace_wrapper() {
 906     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
 907     ddebug "Create $strace_wrapper"
 908     cat >"$strace_wrapper" <<EOF
 909 #!/usr/bin/env bash
 910
 911 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
 912 EOF
 913     chmod 0755 "$strace_wrapper"
 914 }
 915
 916 install_fs_tools() {
 917     dinfo "Install fsck"
 918     image_install /sbin/fsck*
 919     image_install -o /bin/fsck*
 920
 921     # fskc.reiserfs calls reiserfsck. so, install it
 922     image_install -o reiserfsck
 923
 924     # we use mkfs in system-repart tests
 925     image_install /sbin/mkfs.ext4
 926     image_install /sbin/mkfs.vfat
 927 }
 928
 929 install_modules() {
 930     dinfo "Install modules"
 931
 932     instmods loop
 933     instmods vfat
 934     instmods nls_ascii =nls
 935     instmods dummy
 936
 937     if get_bool "$LOOKS_LIKE_SUSE"; then
 938         instmods ext4
 939     fi
 940 }
 941
 942 install_dmevent() {
 943     instmods dm_crypt =crypto
 944     inst_binary dmeventd
 945     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
 946     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 947         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 948         # and since buster/bionic 95-dm-notify.rules
 949         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 950         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 951     else
 952         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 953     fi
 954     if get_bool "$LOOKS_LIKE_SUSE"; then
 955         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
 956     fi
 957 }
 958
 959 install_multipath() {
 960     instmods "=md" multipath
 961     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
 962     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
 963     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 964         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
 965     else
 966         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
 967     fi
 968     mkdir -p "${initdir:?}/etc/multipath"
 969
 970     local file
 971     while read -r file; do
 972         # Install libraries required by the given library
 973         inst_libs "$file"
 974         # Install the library itself and create necessary symlinks
 975         inst_library "$file"
 976     done < <(find /lib*/multipath -type f)
 977
 978     if get_bool "$LOOKS_LIKE_ARCH"; then
 979         # On Arch the multipath libraries are not linked against libgcc_s.so.1,
 980         # but it's still required at runtime
 981         inst_library "/lib64/libgcc_s.so.1"
 982     fi
 983 }
 984
 985 install_lvm() {
 986     image_install lvm
 987     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
 988     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
 989     image_install -o "/lib/tmpfiles.d/lvm2.conf"
 990     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 991         inst_rules 56-lvm.rules 69-lvm-metad.rules
 992     else
 993         # Support the new udev autoactivation introduced in lvm 2.03.14
 994         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
 995         # Static autoactivation (via lvm2-activation-generator) was dropped
 996         # in lvm 2.03.15
 997         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
 998         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
 999             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
1000         else
1001             image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1002             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1003             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1004         fi
1005     fi
1006     mkdir -p "${initdir:?}/etc/lvm"
1007 }
1008
1009 install_btrfs() {
1010     instmods btrfs
1011     # Not all utilities provided by btrfs-progs are listed here; extend the list
1012     # if necessary
1013     image_install btrfs btrfstune mkfs.btrfs
1014     inst_rules 64-btrfs-dm.rules
1015 }
1016
1017 install_iscsi() {
1018     # Install both client and server side stuff by default
1019     local inst="${1:-}"
1020     local file
1021
1022     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1023     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1024     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1025         image_install iscsi-iname iscsiadm iscsid iscsistart
1026         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1027         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1028         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1029         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1030         mkdir -p "${initdir:?}/etc/iscsi"
1031         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1032         inst_simple "/etc/iscsi/initiatorname.iscsi"
1033     fi
1034
1035     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1036     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1037     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1038         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1039         image_install -o /etc/sysconfig/tgtd
1040         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1041         mkdir -p "${initdir:?}/etc/tgt"
1042         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1043         # Install perl modules required by tgt-admin
1044         #
1045         # Forgive me father for I have sinned. The monstrosity below appends
1046         # a perl snippet to the `tgt-admin` perl script on the fly, which
1047         # dumps a list of files (perl modules) required by `tgt-admin` at
1048         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1049         # passed to `inst_simple` which installs the necessary files into the image
1050         #
1051         # shellcheck disable=SC2016
1052         while read -r file; do
1053             inst_simple "$file"
1054         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1055     fi
1056 }
1057
1058 install_mdadm() {
1059     local unit
1060     local mdadm_units=(
1061         system/mdadm-grow-continue@.service
1062         system/mdadm-last-resort@.service
1063         system/mdadm-last-resort@.timer
1064         system/mdmon@.service
1065         system/mdmonitor-oneshot.service
1066         system/mdmonitor-oneshot.timer
1067         system/mdmonitor.service
1068         system-shutdown/mdadm.shutdown
1069     )
1070
1071     image_install mdadm mdmon
1072     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1073     # Fedora/CentOS/RHEL ships this rule file
1074     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1075
1076     for unit in "${mdadm_units[@]}"; do
1077         image_install "${ROOTLIBDIR:?}/$unit"
1078     done
1079 }
1080
1081 install_compiled_systemd() {
1082     dinfo "Install compiled systemd"
1083
1084     local ninja_bin
1085     ninja_bin="$(type -P ninja || type -P ninja-build)"
1086     if [[ -z "$ninja_bin" ]]; then
1087         dfatal "ninja was not found"
1088         exit 1
1089     fi
1090     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1091
1092     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1093     # find them in the same directory as the runtime data counts
1094     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1095         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1096         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1097         # Set effective & default ACLs for the build dir so unprivileged
1098         # processes can write gcda files with coverage stats
1099         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1100     fi
1101 }
1102
1103 install_debian_systemd() {
1104     dinfo "Install debian systemd"
1105
1106     local files
1107
1108     while read -r deb; do
1109         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1110         ddebug "Install debian files from package $deb"
1111         for file in $files; do
1112             [ -e "$file" ] || continue
1113             [ -d "$file" ] && continue
1114             inst "$file"
1115         done
1116     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1117 }
1118
1119 install_suse_systemd() {
1120     local testsdir=/usr/lib/systemd/tests
1121     local pkgs
1122
1123     dinfo "Install SUSE systemd"
1124
1125     pkgs=(
1126         systemd
1127         systemd-container
1128         systemd-coredump
1129         systemd-experimental
1130         systemd-journal-remote
1131         systemd-portable
1132         udev
1133     )
1134
1135     for p in "${pkgs[@]}"; do
1136         rpm -q "$p" &>/dev/null || continue
1137
1138         ddebug "Install files from package $p"
1139         while read -r f; do
1140             [ -e "$f" ] || continue
1141             [ -d "$f" ] && continue
1142             inst "$f"
1143         done < <(rpm -ql "$p")
1144     done
1145
1146     # we only need testsdata dir as well as the unit tests (for
1147     # TEST-02-UNITTESTS) in the image.
1148     dinfo "Install unit tests and testdata directory"
1149
1150     mkdir -p "$initdir/$testsdir"
1151     cp "$testsdir"/test-* "$initdir/$testsdir/"
1152     cp -a "$testsdir/testdata" "$initdir/$testsdir/"
1153
1154     # On openSUSE, these dirs are not created at package install for now on.
1155     mkdir -p "$initdir/var/log/journal/remote"
1156 }
1157
1158 install_distro_systemd() {
1159     dinfo "Install distro systemd"
1160
1161     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1162         install_debian_systemd
1163     elif get_bool "$LOOKS_LIKE_SUSE"; then
1164         install_suse_systemd
1165     else
1166         dfatal "NO_BUILD not supported for this distro"
1167         exit 1
1168     fi
1169 }
1170
1171 install_systemd() {
1172     dinfo "Install systemd"
1173     if get_bool "$NO_BUILD"; then
1174         install_distro_systemd
1175     else
1176         install_compiled_systemd
1177     fi
1178
1179     # remove unneeded documentation
1180     rm -fr "${initdir:?}"/usr/share/{man,doc}
1181
1182     # enable debug logging in PID1
1183     echo LogLevel=debug >>"$initdir/etc/systemd/system.conf"
1184     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1185         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf"
1186     fi
1187     # store coredumps in journal
1188     echo Storage=journal >>"$initdir/etc/systemd/coredump.conf"
1189     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1190     mkdir "$initdir/etc/systemd/system/user@.service.d/"
1191     echo -e "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/override.conf"
1192
1193     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1194     # images, since it prevents gcov to write the coverage reports (*.gcda
1195     # files)
1196     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1197         mkdir -p "$initdir/etc/systemd/system/service.d/"
1198         echo -e "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1199         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image
1200         # to make the coverage work with units utilizing DynamicUser=yes. Do
1201         # this only for services from TEST-20, as setting this system-wide
1202         # has many undesirable side-effects
1203         mkdir -p "$initdir/etc/systemd/system/test20-.service.d/"
1204         echo -e "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/test20-.service.d/99-gcov-rwpaths-override.conf"
1205     fi
1206
1207     # If we're built with -Dportabled=false, tests with systemd-analyze
1208     # --profile will fail. Since we need just the profile (text) files, let's
1209     # copy them into the image if they don't exist there.
1210     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1211     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1212         dinfo "Couldn't find portable profiles in the test image"
1213         dinfo "Copying them directly from the source tree"
1214         mkdir -p "$portable_dir"
1215         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1216     fi
1217 }
1218
1219 get_ldpath() {
1220     local rpath
1221     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1222
1223     if [ -z "$rpath" ] ; then
1224         echo "$BUILD_DIR"
1225     else
1226         echo "$rpath"
1227     fi
1228 }
1229
1230 install_missing_libraries() {
1231     dinfo "Install missing libraries"
1232     # install possible missing libraries
1233     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
1234         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1235     done
1236
1237     local lib path
1238     # A number of dependencies is now optional via dlopen, so the install
1239     # script will not pick them up, since it looks at linkage.
1240     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw; do
1241         ddebug "Searching for $lib via pkg-config"
1242         if pkg-config --exists "$lib"; then
1243                 path="$(pkg-config --variable=libdir "$lib")"
1244                 if [ -z "${path}" ]; then
1245                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1246                     continue
1247                 fi
1248
1249                 if ! [[ ${lib} =~ ^lib ]]; then
1250                         lib="lib${lib}"
1251                 fi
1252                 # Some pkg-config files are broken and give out the wrong paths
1253                 # (eg: libcryptsetup), so just ignore them
1254                 inst_libs "${path}/${lib}.so" || true
1255                 inst_library "${path}/${lib}.so" || true
1256         else
1257             ddebug "$lib.pc not found, skipping"
1258             continue
1259         fi
1260     done
1261 }
1262
1263 cleanup_loopdev() {
1264     if [ -n "${LOOPDEV:=}" ]; then
1265         ddebug "losetup -d $LOOPDEV"
1266         losetup -d "${LOOPDEV}"
1267         unset LOOPDEV
1268     fi
1269 }
1270
1271 trap cleanup_loopdev EXIT INT QUIT PIPE
1272
1273 create_empty_image() {
1274     if [ -z "${IMAGE_NAME:=}" ]; then
1275         echo "create_empty_image: \$IMAGE_NAME not set"
1276         exit 1
1277     fi
1278
1279     local size=500
1280     if ! get_bool "$NO_BUILD"; then
1281         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1282             size=$((size+=200))
1283         fi
1284         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1285             size=$((size+=200))
1286         fi
1287         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1288             size=$((size+=250))
1289         fi
1290     fi
1291     if ! get_bool "$STRIP_BINARIES"; then
1292         size=$((4 * size))
1293     fi
1294
1295     echo "Setting up ${IMAGE_PUBLIC:?} (${size} MB)"
1296     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1297
1298     # Create the blank file to use as a root filesystem
1299     truncate -s "${size}M" "$IMAGE_PUBLIC"
1300
1301     LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
1302     [ -b "$LOOPDEV" ] || return 1
1303     sfdisk "$LOOPDEV" <<EOF
1304 ,$((size - 50))M,L,*
1305 ,
1306 EOF
1307
1308     udevadm settle
1309
1310     local label=(-L systemd_boot)
1311     # mkfs.reiserfs doesn't know -L. so, use --label instead
1312     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1313     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1314         dfatal "Failed to mkfs -t ${FSTYPE}"
1315         exit 1
1316     fi
1317 }
1318
1319 mount_initdir() {
1320     if [ -z "${LOOPDEV:=}" ]; then
1321         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1322         LOOPDEV="$(losetup --show -P -f "$image")"
1323         [ -b "$LOOPDEV" ] || return 1
1324
1325         udevadm settle
1326     fi
1327
1328     if ! mountpoint -q "${initdir:?}"; then
1329         mkdir -p "$initdir"
1330         mount "${LOOPDEV}p1" "$initdir"
1331         TEST_SETUP_CLEANUP_ROOTDIR=1
1332     fi
1333 }
1334
1335 cleanup_initdir() {
1336     # only umount if create_empty_image_rootdir() was called to mount it
1337     get_bool "$TEST_SETUP_CLEANUP_ROOTDIR" && _umount_dir "${initdir:?}"
1338 }
1339
1340 umount_loopback() {
1341     # unmount the loopback device from all places. Otherwise we risk file
1342     # system corruption.
1343     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1344         ddebug "Unmounting all uses of $device"
1345         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1346     done
1347 }
1348
1349 create_empty_image_rootdir() {
1350     create_empty_image
1351     mount_initdir
1352 }
1353
1354 check_asan_reports() {
1355     local ret=0
1356     local root="${1:?}"
1357
1358     if get_bool "$IS_BUILT_WITH_ASAN"; then
1359         ls -l "$root"
1360         if [[ -e "$root/systemd.asan.log.1" ]]; then
1361             cat "$root/systemd.asan.log.1"
1362             ret=$((ret+1))
1363         fi
1364
1365         journald_report="$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)"
1366         if [[ -n "$journald_report" ]]; then
1367             printf "%s\n" "$journald_report"
1368             cat "$root/systemd-journald.out" || :
1369             ret=$((ret+1))
1370         fi
1371
1372         pids="$(
1373             "$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
1374                  BEGIN {
1375                      %services_to_ignore = (
1376                          "dbus-daemon" => undef,
1377                          "dbus-broker-launch" => undef,
1378                      );
1379                  }
1380                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
1381         )"
1382         if [[ -n "$pids" ]]; then
1383             ret=$((ret+1))
1384             for pid in $pids; do
1385                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1386             done
1387         fi
1388     fi
1389
1390     return $ret
1391 }
1392
1393 check_coverage_reports() {
1394     local root="${1:?}"
1395
1396     if get_bool "$NO_BUILD"; then
1397         return 0
1398     fi
1399     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1400         return 0
1401     fi
1402
1403     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1404         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1405     else
1406         dest="${TESTDIR:?}/coverage-info"
1407     fi
1408
1409     # Create a coverage report that will later be uploaded. Remove info about
1410     # system libraries/headers, as we don't really care about them.
1411     if [[ -f "$dest" ]]; then
1412         # If the destination report file already exists, don't overwrite it, but
1413         # dump the new report in a temporary file and then merge it with the already
1414         # present one - this usually happens when running both "parts" of a test
1415         # in one run (the qemu and the nspawn part).
1416         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1417         lcov --remove "${dest}.new" -o "${dest}.new" '/usr/include/*' '/usr/lib/*'
1418         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1419         rm -f "${dest}.new"
1420     else
1421         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}"
1422         lcov --remove "${dest}" -o "${dest}" '/usr/include/*' '/usr/lib/*'
1423     fi
1424
1425     # If the test logs contain lines like:
1426     #
1427     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1428     #
1429     # it means we're possibly missing some coverage since gcov can't write the stats,
1430     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1431     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1432     # `setfacl` stuff in install_compiled_systemd().
1433     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1434        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1435         derror "Detected possibly missing coverage, check the journal"
1436         return 1
1437     fi
1438
1439     return 0
1440 }
1441
1442 save_journal() {
1443     # Default to always saving journal
1444     local save="yes"
1445
1446     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1447         save="no"
1448     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1449         save="no"
1450     fi
1451
1452     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1453         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1454     else
1455         dest="${TESTDIR:?}/system.journal"
1456     fi
1457
1458     for j in "${1:?}"/*; do
1459         if get_bool "$save"; then
1460             if [ "$SYSTEMD_JOURNAL_REMOTE" = "" ]; then
1461                 cp -a "$j" "$dest"
1462             else
1463                 "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1464             fi
1465         fi
1466
1467         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1468             echo "---- $j ----"
1469             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1470         fi
1471
1472         rm -r "$j"
1473     done
1474
1475     if ! get_bool "$save"; then
1476         return 0
1477     fi
1478
1479     if [ -n "${SUDO_USER}" ]; then
1480         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1481     fi
1482
1483     # we want to print this sometime later, so save this in a variable
1484     JOURNAL_LIST="$(ls -l "$dest"*)"
1485 }
1486
1487 check_result_common() {
1488     local workspace="${1:?}"
1489     local ret
1490
1491     if [ -s "$workspace/failed" ]; then
1492         # Non-empty …/failed has highest priority
1493         cp -a "$workspace/failed" "${TESTDIR:?}/"
1494         if [ -n "${SUDO_USER}" ]; then
1495             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1496         fi
1497         ret=1
1498     elif get_bool "$TIMED_OUT"; then
1499         echo "(timeout)" >"${TESTDIR:?}/failed"
1500         ret=2
1501     elif [ -e "$workspace/testok" ]; then
1502         # …/testok always counts (but with lower priority than …/failed)
1503         ret=0
1504     elif [ -e "$workspace/skipped" ]; then
1505         # …/skipped always counts (a message is expected)
1506         echo "${TESTNAME:?} was skipped:"
1507         cat "$workspace/skipped"
1508         ret=0
1509     else
1510         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1511         ret=3
1512     fi
1513
1514     check_asan_reports "$workspace" || ret=4
1515
1516     check_coverage_reports "$workspace" || ret=5
1517
1518     save_journal "$workspace/var/log/journal" $ret
1519
1520     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1521         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1522     fi
1523
1524     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1525         echo -n "${TESTNAME:?}: "
1526         cat "$TESTDIR/failed"
1527     fi
1528     echo "${JOURNAL_LIST:-"No journals were saved"}"
1529
1530     return ${ret:?}
1531 }
1532
1533 check_result_nspawn() {
1534     local workspace="${1:?}"
1535     local ret
1536
1537     check_result_common "${workspace}"
1538     ret=$?
1539
1540     # Run additional test-specific checks if defined by check_result_nspawn_hook()
1541     if declare -F check_result_nspawn_hook >/dev/null; then
1542         if ! check_result_nspawn_hook; then
1543             derror "check_result_nspawn_hook() returned with EC > 0"
1544             ret=4
1545         fi
1546     fi
1547
1548     _umount_dir "${initdir:?}"
1549
1550     return $ret
1551 }
1552
1553 # can be overridden in specific test
1554 check_result_qemu() {
1555     local ret
1556     mount_initdir
1557
1558     check_result_common "${initdir:?}"
1559     ret=$?
1560
1561     _umount_dir "${initdir:?}"
1562
1563     # Run additional test-specific checks if defined by check_result_qemu_hook()
1564     if declare -F check_result_qemu_hook >/dev/null; then
1565         if ! check_result_qemu_hook; then
1566             derror "check_result_qemu_hook() returned with EC > 0"
1567             ret=4
1568         fi
1569     fi
1570
1571     return $ret
1572 }
1573
1574 check_result_nspawn_unittests() {
1575     local workspace="${1:?}"
1576     local ret=1
1577
1578     [[ -e "$workspace/testok" ]] && ret=0
1579
1580     if [[ -s "$workspace/failed" ]]; then
1581         ret=$((ret + 1))
1582         echo "=== Failed test log ==="
1583         cat "$workspace/failed"
1584     else
1585         if [[ -s "$workspace/skipped" ]]; then
1586             echo "=== Skipped test log =="
1587             cat "$workspace/skipped"
1588             # We might have only skipped tests - that should not fail the job
1589             ret=0
1590         fi
1591         if [[ -s "$workspace/testok" ]]; then
1592             echo "=== Passed tests ==="
1593             cat "$workspace/testok"
1594         fi
1595     fi
1596
1597     get_bool "${TIMED_OUT:=}" && ret=1
1598     check_coverage_reports "$workspace" || ret=5
1599
1600     save_journal "$workspace/var/log/journal" $ret
1601
1602     _umount_dir "${initdir:?}"
1603
1604     return $ret
1605 }
1606
1607 check_result_qemu_unittests() {
1608     local ret=1
1609
1610     mount_initdir
1611     [[ -e "${initdir:?}/testok" ]] && ret=0
1612
1613     if [[ -s "$initdir/failed" ]]; then
1614         ret=$((ret + 1))
1615         echo "=== Failed test log ==="
1616         cat "$initdir/failed"
1617     else
1618         if [[ -s "$initdir/skipped" ]]; then
1619             echo "=== Skipped test log =="
1620             cat "$initdir/skipped"
1621             # We might have only skipped tests - that should not fail the job
1622             ret=0
1623         fi
1624         if [[ -s "$initdir/testok" ]]; then
1625             echo "=== Passed tests ==="
1626             cat "$initdir/testok"
1627         fi
1628     fi
1629
1630     get_bool "${TIMED_OUT:=}" && ret=1
1631     check_coverage_reports "$initdir" || ret=5
1632
1633     save_journal "$initdir/var/log/journal" $ret
1634
1635     _umount_dir "$initdir"
1636
1637     return $ret
1638 }
1639
1640 strip_binaries() {
1641     dinfo "Strip binaries"
1642     if ! get_bool "$STRIP_BINARIES"; then
1643         dinfo "STRIP_BINARIES == no, keeping binaries unstripped"
1644         return 0
1645     fi
1646     while read -r bin; do
1647         strip --strip-unneeded "$bin" |& grep -vi 'file format not recognized' | ddebug || :
1648     done < <(find "${initdir:?}" -executable -not -path '*/lib/modules/*.ko' -type f)
1649 }
1650
1651 create_rc_local() {
1652     dinfo "Create rc.local"
1653     mkdir -p "${initdir:?}/etc/rc.d"
1654     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1655 #!/usr/bin/env bash
1656 exit 0
1657 EOF
1658     chmod 0755 "$initdir/etc/rc.d/rc.local"
1659 }
1660
1661 install_execs() {
1662     ddebug "Install executables from the service files"
1663
1664     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1665     local systemunitdir userunitdir exe
1666     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1667     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1668     while read -r exe; do
1669         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1670         # also, plymouth is pulled in by rescue.service, but even there the exit code
1671         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1672         dinfo "Attempting to install $exe (based on unit file reference)"
1673         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1674     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1675 }
1676
1677 generate_module_dependencies() {
1678     dinfo "Generate modules dependencies"
1679     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1680         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1681             dfatal "\"depmod -a $KERNEL_VER\" failed."
1682             exit 1
1683     fi
1684 }
1685
1686 install_depmod_files() {
1687     dinfo "Install depmod files"
1688     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1689     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1690 }
1691
1692 install_plymouth() {
1693     dinfo "Install plymouth"
1694     # install plymouth, if found... else remove plymouth service files
1695     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1696     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1697     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1698     #         image_install plymouth plymouthd
1699     # else
1700         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1701     # fi
1702 }
1703
1704 install_haveged() {
1705     # If haveged is installed, it's probably included in initrd and needs to be
1706     # installed in the image too.
1707     if [ -x /usr/sbin/haveged ]; then
1708         dinfo "Install haveged files"
1709         inst /usr/sbin/haveged
1710         for u in /usr/lib/systemd/system/haveged*; do
1711             inst "$u"
1712         done
1713     fi
1714 }
1715
1716 install_ld_so_conf() {
1717     dinfo "Install /etc/ld.so.conf*"
1718     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1719     ldconfig -r "$initdir"
1720 }
1721
1722 install_testuser() {
1723     dinfo "Set up a test user"
1724     # create unprivileged user for user manager tests
1725     mkdir -p "${initdir:?}/etc/sysusers.d"
1726     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1727 u testuser    4711     "Test User" /home/testuser
1728 EOF
1729
1730     mkdir -p "$initdir/home/testuser"
1731     chmod 0700 "$initdir/home/testuser"
1732     chown 4711:4711 "$initdir/home/testuser"
1733 }
1734
1735 install_config_files() {
1736     dinfo "Install config files"
1737     inst /etc/sysconfig/init || :
1738     inst /etc/passwd
1739     inst /etc/shadow
1740     inst_any /etc/login.defs /usr/etc/login.defs
1741     inst /etc/group
1742     inst /etc/shells
1743     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
1744     inst /etc/pam.conf || :
1745     inst_any /etc/os-release /usr/lib/os-release
1746     inst /etc/localtime
1747     # we want an empty environment
1748     : >"${initdir:?}/etc/environment"
1749     : >"$initdir/etc/machine-id"
1750     : >"$initdir/etc/resolv.conf"
1751
1752     # set the hostname
1753     echo 'H' >"$initdir/etc/hostname"
1754
1755     # let's set up just one image with the traditional verbose output
1756     if [ "${IMAGE_NAME:?}" != "basic" ]; then
1757         mkdir -p "$initdir/etc/systemd/system.conf.d"
1758         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
1759     fi
1760 }
1761
1762 install_basic_tools() {
1763     dinfo "Install basic tools"
1764     image_install "${BASICTOOLS[@]}"
1765     image_install -o sushell
1766     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
1767     image_install -o ldconfig.real
1768 }
1769
1770 install_debug_tools() {
1771     dinfo "Install debug tools"
1772     image_install -o "${DEBUGTOOLS[@]}"
1773
1774     if get_bool "$INTERACTIVE_DEBUG"; then
1775         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
1776         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
1777         mkdir -p "$getty_override"
1778         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
1779
1780         cat >"$initdir/etc/motd" <<EOF
1781 To adjust the terminal size use:
1782     export COLUMNS=xx
1783     export LINES=yy
1784 or
1785     stty cols xx rows yy
1786 EOF
1787     fi
1788 }
1789
1790 install_libnss() {
1791     dinfo "Install libnss"
1792     # install libnss_files for login
1793     local NSS_LIBS
1794     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
1795     image_install "${NSS_LIBS[@]}"
1796 }
1797
1798 install_dbus() {
1799     dinfo "Install dbus"
1800     inst "${ROOTLIBDIR:?}/system/dbus.socket"
1801
1802     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1803     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
1804         inst "$ROOTLIBDIR/system/dbus-broker.service"
1805         inst_symlink /etc/systemd/system/dbus.service
1806         inst /usr/bin/dbus-broker
1807         inst /usr/bin/dbus-broker-launch
1808     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
1809         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1810         inst "$ROOTLIBDIR/system/dbus-daemon.service"
1811         # Alias symlink
1812         inst_symlink /etc/systemd/system/dbus.service
1813     else
1814         inst "$ROOTLIBDIR/system/dbus.service"
1815     fi
1816
1817     while read -r file; do
1818         inst "$file"
1819     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
1820
1821     # setup policy for Type=dbus test
1822     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
1823     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
1824 <?xml version="1.0"?>
1825 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1826         "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1827 <busconfig>
1828     <policy user="root">
1829         <allow own="systemd.test.ExecStopPost"/>
1830     </policy>
1831 </busconfig>
1832 EOF
1833 }
1834
1835 install_user_dbus() {
1836     dinfo "Install user dbus"
1837     local userunitdir
1838     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
1839         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
1840         userunitdir=/usr/lib/systemd/user
1841     fi
1842
1843     inst "$userunitdir/dbus.socket"
1844     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
1845
1846     # Append the After= dependency on dbus in case it isn't already set up
1847     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
1848     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
1849 [Unit]
1850 After=dbus.service
1851 EOF
1852
1853     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1854     if [ -f "$userunitdir/dbus-broker.service" ]; then
1855         inst "$userunitdir/dbus-broker.service"
1856         inst_symlink /etc/systemd/user/dbus.service
1857     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
1858         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1859         inst "$userunitdir/dbus-daemon.service"
1860         # Alias symlink
1861         inst_symlink /etc/systemd/user/dbus.service
1862     else
1863         inst "$userunitdir/dbus.service"
1864     fi
1865 }
1866
1867 install_pam() {
1868     dinfo "Install PAM"
1869     local paths=()
1870
1871     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
1872         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
1873     else
1874         paths+=(/lib*/security)
1875     fi
1876
1877     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
1878         [ -d "$d" ] && paths+=("$d")
1879     done
1880
1881     while read -r file; do
1882         inst "$file"
1883     done < <(find "${paths[@]}" -xtype f)
1884
1885     # pam_unix depends on unix_chkpwd.
1886     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
1887     image_install -o unix_chkpwd
1888
1889     # set empty root password for easy debugging
1890     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
1891
1892     # And make sure pam_unix will accept it by making sure that
1893     # the PAM module has the nullok option.
1894     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
1895         [ -d "$initdir/$d" ] || continue
1896         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
1897     done
1898 }
1899
1900 # shellcheck disable=SC2120
1901 install_keymaps() {
1902     dinfo "Install keymaps"
1903     # The first three paths may be deprecated.
1904     # It seems now the last two paths are used by many distributions.
1905     for i in \
1906         /usr/lib/kbd/keymaps/include/* \
1907         /usr/lib/kbd/keymaps/i386/include/* \
1908         /usr/lib/kbd/keymaps/i386/qwerty/us.* \
1909         /usr/lib/kbd/keymaps/legacy/include/* \
1910         /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
1911             [[ -f "$i" ]] || continue
1912             inst "$i"
1913     done
1914
1915     # When it takes any argument, then install more keymaps.
1916     if [[ $# -gt 1 ]]; then
1917         for i in \
1918         /usr/lib/kbd/keymaps/i386/*/* \
1919         /usr/lib/kbd/keymaps/legacy/i386/*/*; do
1920             [[ -f "$i" ]] || continue
1921             inst "$i"
1922         done
1923     fi
1924 }
1925
1926 install_zoneinfo() {
1927     dinfo "Install time zones"
1928     inst_any /usr/share/zoneinfo/Asia/Seoul
1929     inst_any /usr/share/zoneinfo/Asia/Vladivostok
1930     inst_any /usr/share/zoneinfo/Australia/Sydney
1931     inst_any /usr/share/zoneinfo/Europe/Berlin
1932     inst_any /usr/share/zoneinfo/Europe/Dublin
1933     inst_any /usr/share/zoneinfo/Europe/Kiev
1934     inst_any /usr/share/zoneinfo/Pacific/Auckland
1935     inst_any /usr/share/zoneinfo/Pacific/Honolulu
1936     inst_any /usr/share/zoneinfo/CET
1937     inst_any /usr/share/zoneinfo/EET
1938     inst_any /usr/share/zoneinfo/UTC
1939 }
1940
1941 install_fonts() {
1942     dinfo "Install system fonts"
1943     for i in \
1944         /usr/lib/kbd/consolefonts/eurlatgr* \
1945         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
1946             [[ -f "$i" ]] || continue
1947             inst "$i"
1948     done
1949 }
1950
1951 install_terminfo() {
1952     dinfo "Install terminfo files"
1953     local terminfodir
1954     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
1955         [ -f "${terminfodir}/l/linux" ] && break
1956     done
1957     image_install -o "${terminfodir}/l/linux"
1958 }
1959
1960 has_user_dbus_socket() {
1961     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
1962         return 0
1963     else
1964         echo "Per-user instances are not supported. Skipping..."
1965         return 1
1966     fi
1967 }
1968
1969 setup_nspawn_root_hook() { :;}
1970
1971 setup_nspawn_root() {
1972     if [ -z "${initdir}" ]; then
1973         dfatal "\$initdir not defined"
1974         exit 1
1975     fi
1976
1977     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
1978
1979     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
1980         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
1981         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
1982     fi
1983
1984     setup_nspawn_root_hook
1985 }
1986
1987 setup_basic_dirs() {
1988     mkdir -p "${initdir:?}/run"
1989     mkdir -p "$initdir/etc/systemd/system"
1990     mkdir -p "$initdir/var/log/journal"
1991
1992
1993     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
1994         if [ -L "/$d" ]; then
1995             inst_symlink "/$d"
1996         else
1997             inst_dir "/$d"
1998         fi
1999     done
2000
2001     ln -sfn /run "$initdir/var/run"
2002     ln -sfn /run/lock "$initdir/var/lock"
2003 }
2004
2005 mask_supporting_services() {
2006     # mask some services that we do not want to run in these tests
2007     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2008     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2009     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2010     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2011     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2012 }
2013
2014 inst_libs() {
2015     local bin="${1:?}"
2016     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2017     local file line
2018
2019     while read -r line; do
2020         [[ "$line" = 'not a dynamic executable' ]] && break
2021         # Ignore errors about our own stuff missing. This is most likely caused
2022         # by ldd attempting to use the unprefixed RPATH.
2023         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2024
2025         if [[ "$line" =~ $so_regex ]]; then
2026             file="${BASH_REMATCH[1]}"
2027             [[ -e "${initdir:?}/$file" ]] && continue
2028             inst_library "$file"
2029             continue
2030         fi
2031
2032         if [[ "$line" =~ not\ found ]]; then
2033             dfatal "Missing a shared library required by $bin."
2034             dfatal "Run \"ldd $bin\" to find out what it is."
2035             dfatal "$line"
2036             dfatal "Cannot create a test image."
2037             exit 1
2038         fi
2039     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2040 }
2041
2042 import_testdir() {
2043     # make sure we don't get a stale LOOPDEV value from old times
2044     local _LOOPDEV="${LOOPDEV:=}"
2045     # We don't want shellcheck to follow & check the $STATEFILE
2046     # shellcheck source=/dev/null
2047     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2048     LOOPDEV="$_LOOPDEV"
2049     if [[ ! -d "$TESTDIR" ]]; then
2050         if [[ -z "$TESTDIR" ]]; then
2051             TESTDIR="$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)"
2052         else
2053             mkdir -p "$TESTDIR"
2054         fi
2055
2056         cat >"$STATEFILE" <<EOF
2057 TESTDIR="$TESTDIR"
2058 EOF
2059         export TESTDIR
2060     fi
2061
2062     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2063     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2064 }
2065
2066 import_initdir() {
2067     initdir="${TESTDIR:?}/root"
2068     mkdir -p "$initdir"
2069     export initdir
2070 }
2071
2072 get_cgroup_hierarchy() {
2073     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2074         cgroup2fs)
2075             echo "unified"
2076             ;;
2077         tmpfs)
2078             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2079                 echo "hybrid"
2080             else
2081                 echo "legacy"
2082             fi
2083             ;;
2084         *)
2085             dfatal "Failed to determine host's cgroup hierarchy"
2086             exit 1
2087     esac
2088 }
2089
2090 ## @brief Converts numeric logging level to the first letter of level name.
2091 #
2092 # @param lvl Numeric logging level in range from 1 to 6.
2093 # @retval 1 if @a lvl is out of range.
2094 # @retval 0 if @a lvl is correct.
2095 # @result Echoes first letter of level name.
2096 _lvl2char() {
2097     case "$1" in
2098         1) echo F;;
2099         2) echo E;;
2100         3) echo W;;
2101         4) echo I;;
2102         5) echo D;;
2103         6) echo T;;
2104         *) return 1;;
2105     esac
2106 }
2107
2108 ## @brief Internal helper function for _do_dlog()
2109 #
2110 # @param lvl Numeric logging level.
2111 # @param msg Message.
2112 # @retval 0 It's always returned, even if logging failed.
2113 #
2114 # @note This function is not supposed to be called manually. Please use
2115 # dtrace(), ddebug(), or others instead which wrap this one.
2116 #
2117 # This function calls _do_dlog() either with parameter msg, or if
2118 # none is given, it will read standard input and will use every line as
2119 # a message.
2120 #
2121 # This enables:
2122 # dwarn "This is a warning"
2123 # echo "This is a warning" | dwarn
2124 LOG_LEVEL="${LOG_LEVEL:-4}"
2125
2126 dlog() {
2127     local lvl lvlc
2128
2129     [ -z "$LOG_LEVEL" ] && return 0
2130     lvl="${1:?}"; shift
2131     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2132     lvlc="$(_lvl2char "$lvl")" || return 0
2133
2134     if [ $# -ge 1 ]; then
2135         echo "$lvlc: $*"
2136     else
2137         while read -r line; do
2138             echo "$lvlc: " "$line"
2139         done
2140     fi
2141 }
2142
2143 ## @brief Logs message at TRACE level (6)
2144 #
2145 # @param msg Message.
2146 # @retval 0 It's always returned, even if logging failed.
2147 dtrace() {
2148     set +x
2149     dlog 6 "$@"
2150     if get_bool "${debug:=}"; then
2151         set -x
2152     fi
2153 }
2154
2155 ## @brief Logs message at DEBUG level (5)
2156 #
2157 # @param msg Message.
2158 # @retval 0 It's always returned, even if logging failed.
2159 ddebug() {
2160     dlog 5 "$@"
2161 }
2162
2163 ## @brief Logs message at INFO level (4)
2164 #
2165 # @param msg Message.
2166 # @retval 0 It's always returned, even if logging failed.
2167 dinfo() {
2168     set +x
2169     dlog 4 "$@"
2170     if get_bool "${debug:=}"; then
2171         set -x
2172     fi
2173 }
2174
2175 ## @brief Logs message at WARN level (3)
2176 #
2177 # @param msg Message.
2178 # @retval 0 It's always returned, even if logging failed.
2179 dwarn() {
2180     set +x
2181     dlog 3 "$@"
2182     if get_bool "${debug:=}"; then
2183         set -x
2184     fi
2185 }
2186
2187 ## @brief Logs message at ERROR level (2)
2188 #
2189 # @param msg Message.
2190 # @retval 0 It's always returned, even if logging failed.
2191 derror() {
2192     dlog 2 "$@"
2193 }
2194
2195 ## @brief Logs message at FATAL level (1)
2196 #
2197 # @param msg Message.
2198 # @retval 0 It's always returned, even if logging failed.
2199 dfatal() {
2200     set +x
2201     dlog 1 "$@"
2202     if get_bool "${debug:=}"; then
2203         set -x
2204     fi
2205 }
2206
2207
2208 # Generic substring function.  If $2 is in $1, return 0.
2209 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2210
2211 # normalize_path <path>
2212 # Prints the normalized path, where it removes any duplicated
2213 # and trailing slashes.
2214 # Example:
2215 # $ normalize_path ///test/test//
2216 # /test/test
2217 normalize_path() {
2218     shopt -q -s extglob
2219     set -- "${1//+(\/)//}"
2220     shopt -q -u extglob
2221     echo "${1%/}"
2222 }
2223
2224 # convert_abs_rel <from> <to>
2225 # Prints the relative path, when creating a symlink to <to> from <from>.
2226 # Example:
2227 # $ convert_abs_rel /usr/bin/test /bin/test-2
2228 # ../../bin/test-2
2229 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2230 convert_abs_rel() {
2231     local __current __absolute __abssize __cursize __newpath
2232     local -i __i __level
2233
2234     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2235
2236     # corner case #1 - self looping link
2237     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2238
2239     # corner case #2 - own dir link
2240     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2241
2242     IFS="/" read -ra __current <<< "$1"
2243     IFS="/" read -ra __absolute <<< "$2"
2244
2245     __abssize=${#__absolute[@]}
2246     __cursize=${#__current[@]}
2247
2248     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]
2249     do
2250         (( __level++ ))
2251         if (( __level > __abssize || __level > __cursize ))
2252         then
2253             break
2254         fi
2255     done
2256
2257     for ((__i = __level; __i < __cursize-1; __i++))
2258     do
2259         if ((__i > __level))
2260         then
2261             __newpath=$__newpath"/"
2262         fi
2263         __newpath=$__newpath".."
2264     done
2265
2266     for ((__i = __level; __i < __abssize; __i++))
2267     do
2268         if [[ -n $__newpath ]]
2269         then
2270             __newpath=$__newpath"/"
2271         fi
2272         __newpath=$__newpath${__absolute[__i]}
2273     done
2274
2275     echo "$__newpath"
2276 }
2277
2278
2279 # Install a directory, keeping symlinks as on the original system.
2280 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2281 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2282 # and a symlink ${initdir}/lib -> lib64.
2283 inst_dir() {
2284     local dir="${1:?}"
2285     local part="${dir%/*}"
2286     local file
2287
2288     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2289
2290     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2291         dir="$part $dir"
2292         part="${part%/*}"
2293     done
2294
2295     # iterate over parent directories
2296     for file in $dir; do
2297         [[ -e "${initdir}/$file" ]] && continue
2298         if [[ -L $file ]]; then
2299             inst_symlink "$file"
2300         else
2301             # create directory
2302             mkdir -m 0755 "${initdir}/$file" || return 1
2303             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2304             chmod u+w "${initdir}/$file"
2305         fi
2306     done
2307 }
2308
2309 # $1 = file to copy to ramdisk
2310 # $2 (optional) Name for the file on the ramdisk
2311 # Location of the image dir is assumed to be $initdir
2312 # We never overwrite the target if it exists.
2313 inst_simple() {
2314     [[ -f "${1:?}" ]] || return 1
2315     strstr "$1" "/" || return 1
2316
2317     local src="$1"
2318     local target="${2:-$1}"
2319     if ! [[ -d ${initdir:?}/$target ]]; then
2320         [[ -e ${initdir}/$target ]] && return 0
2321         [[ -L ${initdir}/$target ]] && return 0
2322         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2323     fi
2324     # install checksum files also
2325     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2326         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2327     fi
2328     ddebug "Installing $src"
2329     cp --sparse=always -pfL "$src" "${initdir}/$target"
2330 }
2331
2332 # find symlinks linked to given library file
2333 # $1 = library file
2334 # Function searches for symlinks by stripping version numbers appended to
2335 # library filename, checks if it points to the same target and finally
2336 # prints the list of symlinks to stdout.
2337 #
2338 # Example:
2339 # rev_lib_symlinks libfoo.so.8.1
2340 # output: libfoo.so.8 libfoo.so
2341 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2342 rev_lib_symlinks() {
2343     local fn="${1:?}"
2344     local links=""
2345     local orig
2346     orig="$(readlink -f "$1")"
2347
2348     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2349
2350     until [[ "${fn##*.}" == so ]]; do
2351         fn="${fn%.*}"
2352         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2353     done
2354
2355     echo "${links}"
2356 }
2357
2358 # Same as above, but specialized to handle dynamic libraries.
2359 # It handles making symlinks according to how the original library
2360 # is referenced.
2361 inst_library() {
2362     local src="${1:?}"
2363     local dest="${2:-$1}"
2364     local reallib symlink
2365
2366     strstr "$1" "/" || return 1
2367     [[ -e ${initdir:?}/$dest ]] && return 0
2368     if [[ -L $src ]]; then
2369         # install checksum files also
2370         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2371             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2372         fi
2373         reallib="$(readlink -f "$src")"
2374         inst_simple "$reallib" "$reallib"
2375         inst_dir "${dest%/*}"
2376         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2377         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2378     else
2379         inst_simple "$src" "$dest"
2380     fi
2381
2382     # Create additional symlinks.  See rev_symlinks description.
2383     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2384         if [[ ! -e "$initdir/$symlink" ]]; then
2385             ddebug "Creating extra symlink: $symlink"
2386             inst_symlink "$symlink"
2387         fi
2388     done
2389 }
2390
2391 # find a binary.  If we were not passed the full path directly,
2392 # search in the usual places to find the binary.
2393 find_binary() {
2394     local bin="${1:?}"
2395     if [[ -z ${bin##/*} ]]; then
2396         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2397             echo "$bin"
2398             return 0
2399         fi
2400     fi
2401
2402     type -P "$bin"
2403 }
2404
2405 # Same as above, but specialized to install binary executables.
2406 # Install binary executable, and all shared library dependencies, if any.
2407 inst_binary() {
2408     local bin="${1:?}"
2409     local path target
2410
2411     # In certain cases we might attempt to install a binary which is already
2412     # present in the test image, yet it's missing from the host system.
2413     # In such cases, let's check if the binary indeed exists in the image
2414     # before doing any other checks. If it does, immediately return with
2415     # success.
2416     if [[ $# -eq 1 ]]; then
2417         for path in "" bin sbin usr/bin usr/sbin; do
2418             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2419         done
2420     fi
2421
2422     bin="$(find_binary "$bin")" || return 1
2423     target="${2:-$bin}"
2424     [[ -e "${initdir:?}/$target" ]] && return 0
2425     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2426
2427     local file line
2428     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2429     # I love bash!
2430     while read -r line; do
2431         [[ "$line" = 'not a dynamic executable' ]] && break
2432
2433         # Ignore errors about our own stuff missing. This is most likely caused
2434         # by ldd attempting to use the unprefixed RPATH.
2435         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2436
2437         if [[ "$line" =~ $so_regex ]]; then
2438             file="${BASH_REMATCH[1]}"
2439             [[ -e "${initdir}/$file" ]] && continue
2440             inst_library "$file"
2441             continue
2442         fi
2443
2444         if [[ "$line" =~ not\ found ]]; then
2445             dfatal "Missing a shared library required by $bin."
2446             dfatal "Run \"ldd $bin\" to find out what it is."
2447             dfatal "$line"
2448             dfatal "Cannot create a test image."
2449             exit 1
2450         fi
2451     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2452     inst_simple "$bin" "$target"
2453 }
2454
2455 # same as above, except for shell scripts.
2456 # If your shell script does not start with shebang, it is not a shell script.
2457 inst_script() {
2458     local bin line shebang_regex
2459     bin="$(find_binary "${1:?}")" || return 1
2460     shift
2461
2462     read -r -n 80 line <"$bin"
2463     # If debug is set, clean unprintable chars to prevent messing up the term
2464     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2465     shebang_regex='(#! *)(/[^ ]+).*'
2466     [[ "$line" =~ $shebang_regex ]] || return 1
2467     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2468 }
2469
2470 # same as above, but specialized for symlinks
2471 inst_symlink() {
2472     local src="${1:?}"
2473     local target="${2:-$src}"
2474     local realsrc
2475
2476     strstr "$src" "/" || return 1
2477     [[ -L "$src" ]] || return 1
2478     [[ -L "${initdir:?}/$target" ]] && return 0
2479     realsrc="$(readlink -f "$src")"
2480     if ! [[ -e "$initdir/$realsrc" ]]; then
2481         if [[ -d "$realsrc" ]]; then
2482             inst_dir "$realsrc"
2483         else
2484             inst "$realsrc"
2485         fi
2486     fi
2487     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2488     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2489     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2490 }
2491
2492 # attempt to install any programs specified in a udev rule
2493 inst_rule_programs() {
2494     local rule="${1:?}"
2495     local prog bin
2496
2497     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2498         if [ -x "/lib/udev/$prog" ]; then
2499             bin="/lib/udev/$prog"
2500         else
2501             if ! bin="$(find_binary "$prog")"; then
2502                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2503                 continue
2504             fi
2505         fi
2506
2507         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2508         image_install "$bin"
2509     done
2510 }
2511
2512 # udev rules always get installed in the same place, so
2513 # create a function to install them to make life simpler.
2514 inst_rules() {
2515     local target=/etc/udev/rules.d
2516     local found rule
2517
2518     inst_dir "/lib/udev/rules.d"
2519     inst_dir "$target"
2520     for rule in "$@"; do
2521         if [ "${rule#/}" = "$rule" ]; then
2522             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2523                 if [[ -f "$r/$rule" ]]; then
2524                     found="$r/$rule"
2525                     inst_simple "$found"
2526                     inst_rule_programs "$found"
2527                 fi
2528             done
2529         fi
2530         for r in '' ./; do
2531             if [[ -f "${r}${rule}" ]]; then
2532                 found="${r}${rule}"
2533                 inst_simple "$found" "$target/${found##*/}"
2534                 inst_rule_programs "$found"
2535             fi
2536         done
2537         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2538         found=
2539     done
2540 }
2541
2542 # general purpose installation function
2543 # Same args as above.
2544 inst() {
2545     case $# in
2546         1) ;;
2547         2)
2548             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2549             [[ "$initdir" = "$2" ]] && set "$1"
2550             ;;
2551         3)
2552             [[ -z "$initdir" ]] && export initdir="$2"
2553             set "$1" "$3"
2554             ;;
2555         *)
2556             dfatal "inst only takes 1 or 2 or 3 arguments"
2557             exit 1
2558             ;;
2559     esac
2560
2561     local fun
2562     for fun in inst_symlink inst_script inst_binary inst_simple; do
2563         "$fun" "$@" && return 0
2564     done
2565
2566     dwarn "Failed to install '$1'"
2567     return 1
2568 }
2569
2570 # install any of listed files
2571 #
2572 # If first argument is '-d' and second some destination path, first accessible
2573 # source is installed into this path, otherwise it will installed in the same
2574 # path as source.  If none of listed files was installed, function return 1.
2575 # On first successful installation it returns with 0 status.
2576 #
2577 # Example:
2578 #
2579 # inst_any -d /bin/foo /bin/bar /bin/baz
2580 #
2581 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2582 # initramfs.
2583 inst_any() {
2584     local dest file
2585
2586     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2587
2588     for file in "$@"; do
2589         if [[ -e "$file" ]]; then
2590             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2591             inst "$file" && return 0
2592         fi
2593     done
2594
2595     return 1
2596 }
2597
2598 # image_install [-o ] <file> [<file> ... ]
2599 # Install <file> to the test image
2600 # -o optionally install the <file> and don't fail, if it is not there
2601 image_install() {
2602     local optional=no
2603     local prog="${1:?}"
2604
2605     if [[ "$prog" = '-o' ]]; then
2606         optional=yes
2607         shift
2608     fi
2609
2610     for prog in "$@"; do
2611         if ! inst "$prog" ; then
2612             if get_bool "$optional"; then
2613                 dinfo "Skipping program $prog as it cannot be found and is" \
2614                     "flagged to be optional"
2615             else
2616                 dfatal "Failed to install $prog"
2617                 exit 1
2618             fi
2619         fi
2620     done
2621 }
2622
2623 # Install a single kernel module along with any firmware it may require.
2624 # $1 = full path to kernel module to install
2625 install_kmod_with_fw() {
2626     local module="${1:?}"
2627     # no need to go further if the module is already installed
2628     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
2629     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
2630
2631     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
2632
2633     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
2634
2635     local modname="${module##*/}"
2636     local fwdir found fw
2637     modname="${modname%.ko*}"
2638
2639     while read -r fw; do
2640         found=
2641         for fwdir in /lib/firmware/updates /lib/firmware; do
2642             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
2643                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
2644                 found=yes
2645             fi
2646         done
2647         if ! get_bool "$found"; then
2648             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
2649                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
2650                     "\"${modname}.ko\""
2651             else
2652                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
2653                     "\"${modname}.ko\""
2654             fi
2655         fi
2656     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
2657     return 0
2658 }
2659
2660 # Do something with all the dependencies of a kernel module.
2661 # Note that kernel modules depend on themselves using the technique we use
2662 # $1 = function to call for each dependency we find
2663 #      It will be passed the full path to the found kernel module
2664 # $2 = module to get dependencies for
2665 # rest of args = arguments to modprobe
2666 for_each_kmod_dep() {
2667     local func="${1:?}"
2668     local kmod="${2:?}"
2669     local found=0
2670     local cmd modpath
2671     shift 2
2672
2673     while read -r cmd modpath _; do
2674         [[ "$cmd" = insmod ]] || continue
2675         "$func" "$modpath" || return $?
2676         found=1
2677     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
2678
2679     ! get_bool "$found" && return 1
2680     return 0
2681 }
2682
2683 # instmods [-c] <kernel module> [<kernel module> ... ]
2684 # instmods [-c] <kernel subsystem>
2685 # install kernel modules along with all their dependencies.
2686 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
2687 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
2688 instmods() {
2689     local check=no
2690     if [[ $# -ge 0 && "$1" = '-c' ]]; then
2691         check=yes
2692         shift
2693     fi
2694
2695     inst1mod() {
2696         local mod="${1:?}"
2697         local ret=0
2698         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
2699
2700         case "$mod" in
2701             =*)
2702                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
2703                     (
2704                         [[ "$mpargs" ]] && echo "$mpargs"
2705                         cat "${mod_dir}/modules.${mod#=}"
2706                     ) | instmods
2707                 else
2708                     (
2709                         [[ "$mpargs" ]] && echo "$mpargs"
2710                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
2711                     ) | instmods
2712                 fi
2713                 ;;
2714             --*)
2715                 mpargs+=" $mod"
2716                 ;;
2717             i2o_scsi)
2718                 # Do not load this diagnostic-only module
2719                 return
2720                 ;;
2721             *)
2722                 mod=${mod##*/}
2723                 # if we are already installed, skip this module and go on
2724                 # to the next one.
2725                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
2726
2727                 # We use '-d' option in modprobe only if modules prefix path
2728                 # differs from default '/'.  This allows us to use Dracut with
2729                 # old version of modprobe which doesn't have '-d' option.
2730                 local mod_dirname=${mod_dir%%/lib/modules/*}
2731                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
2732
2733                 # ok, load the module, all its dependencies, and any firmware
2734                 # it may require
2735                 for_each_kmod_dep install_kmod_with_fw "$mod" \
2736                     --set-version "$KERNEL_VER" \
2737                     ${mod_dirname:+"$mod_dirname"} \
2738                     ${mpargs:+"$mpargs"}
2739                 ((ret+=$?))
2740                 ;;
2741         esac
2742         return "$ret"
2743     }
2744
2745     local mod mpargs
2746
2747     if [[ $# -eq 0 ]]; then  # filenames from stdin
2748         while read -r mod; do
2749             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2750                 dfatal "Failed to install $mod"
2751                 return 1
2752             fi
2753         done
2754     fi
2755
2756     for mod in "$@"; do # filenames as arguments
2757         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2758             dfatal "Failed to install $mod"
2759             return 1
2760         fi
2761     done
2762
2763     return 0
2764 }
2765
2766 _umount_dir() {
2767     local mountpoint="${1:?}"
2768     if mountpoint -q "$mountpoint"; then
2769         ddebug "umount $mountpoint"
2770         umount "$mountpoint"
2771     fi
2772 }
2773
2774 # can be overridden in specific test
2775 test_setup_cleanup() {
2776     cleanup_initdir
2777 }
2778
2779 _test_cleanup() {
2780     # (post-test) cleanup should always ignore failure and cleanup as much as possible
2781     (
2782         set +e
2783         [[ -n "$initdir" ]] && _umount_dir "$initdir"
2784         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
2785         # If multiple setups/cleans are ran in parallel, this can cause a race
2786         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
2787             rm -vf "${IMAGESTATEDIR}/default.img"
2788         fi
2789         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
2790         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
2791     ) || :
2792 }
2793
2794 # can be overridden in specific test
2795 test_cleanup() {
2796     _test_cleanup
2797 }
2798
2799 test_cleanup_again() {
2800     [ -n "$TESTDIR" ] || return
2801     rm -rf "$TESTDIR/unprivileged-nspawn-root"
2802     [[ -n "$initdir" ]] && _umount_dir "$initdir"
2803 }
2804
2805 test_create_image() {
2806     create_empty_image_rootdir
2807
2808     # Create what will eventually be our root filesystem onto an overlay
2809     (
2810         LOG_LEVEL=5
2811         setup_basic_environment
2812     )
2813 }
2814
2815 test_setup() {
2816     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
2817             command -v meson >/dev/null && \
2818             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
2819         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
2820         exit 1
2821     fi
2822
2823     if [ -e "${IMAGE_PRIVATE:?}" ]; then
2824         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
2825         mount_initdir
2826     else
2827         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
2828             # default.img is the base that every test uses and optionally appends to
2829             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
2830                 # Create the backing public image, but then completely unmount
2831                 # it and drop the loopback device responsible for it, since we're
2832                 # going to symlink/copy the image and mount it again from
2833                 # elsewhere.
2834                 local image_old="${IMAGE_PUBLIC}"
2835                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
2836                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
2837                 fi
2838                 test_create_image
2839                 test_setup_cleanup
2840                 umount_loopback
2841                 cleanup_loopdev
2842                 IMAGE_PUBLIC="${image_old}"
2843             fi
2844             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
2845                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
2846             fi
2847         fi
2848
2849         local hook_defined
2850         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
2851
2852         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
2853         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
2854             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2855         else
2856             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2857         fi
2858
2859         mount_initdir
2860         # We want to test all services in TEST-01-BASIC, but mask them in
2861         # all other tests
2862         if [[ "${TESTID:?}" != "01" ]]; then
2863             dinfo "Masking supporting services"
2864             mask_supporting_services
2865         fi
2866
2867         if get_bool "$hook_defined"; then
2868             test_append_files "${initdir:?}"
2869         fi
2870     fi
2871
2872     setup_nspawn_root
2873 }
2874
2875 test_run() {
2876     local test_id="${1:?}"
2877     mount_initdir
2878
2879     if ! get_bool "${TEST_NO_QEMU:=}"; then
2880         if run_qemu "$test_id"; then
2881             check_result_qemu || { echo "qemu test failed"; return 1; }
2882         else
2883             dwarn "can't run qemu, skipping"
2884         fi
2885     fi
2886     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
2887         mount_initdir
2888         if run_nspawn "${initdir:?}" "$test_id"; then
2889             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
2890         else
2891             dwarn "can't run systemd-nspawn, skipping"
2892         fi
2893
2894         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
2895             dir="$TESTDIR/unprivileged-nspawn-root"
2896             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
2897                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
2898             else
2899                 dwarn "can't run systemd-nspawn, skipping"
2900             fi
2901         fi
2902     fi
2903     return 0
2904 }
2905
2906 do_test() {
2907     if [[ $UID != "0" ]]; then
2908         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
2909         exit 0
2910     fi
2911
2912     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
2913         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
2914         exit 0
2915     fi
2916
2917     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2918         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
2919         exit 0
2920     fi
2921
2922     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2923         TEST_NO_QEMU=1
2924     fi
2925
2926     # Detect lib paths
2927     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
2928         [[ -d $libdir ]] && libdirs+=" $libdir" && break
2929     done
2930
2931     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
2932         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
2933     done
2934
2935     mkdir -p "$STATEDIR"
2936
2937     import_testdir
2938     import_initdir
2939
2940     if [ -n "${SUDO_USER}" ]; then
2941         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
2942         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
2943     fi
2944
2945     testname="$(basename "$PWD")"
2946
2947     while (($# > 0)); do
2948         case $1 in
2949             --run)
2950                 echo "${testname} RUN: $TEST_DESCRIPTION"
2951                 test_run "$TESTID"
2952                 ret=$?
2953                 if [ $ret -eq 0 ]; then
2954                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
2955                 else
2956                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
2957                 fi
2958                 exit $ret
2959                 ;;
2960             --setup)
2961                 echo "${testname} SETUP: $TEST_DESCRIPTION"
2962                 test_setup
2963                 test_setup_cleanup
2964                 ;;
2965             --clean)
2966                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
2967                 test_cleanup
2968                 ;;
2969             --clean-again)
2970                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
2971                 test_cleanup_again
2972                 ;;
2973             --all)
2974                 ret=0
2975                 echo -n "${testname}: $TEST_DESCRIPTION "
2976                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
2977                 # and loop devices will leak
2978                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
2979                 if [ $ret -eq 0 ]; then
2980                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2981                 fi
2982                 if [ $ret -eq 0 ]; then
2983                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2984                 fi
2985                 test_cleanup
2986                 if [ $ret -eq 0 ]; then
2987                     rm "$TESTLOG"
2988                     echo "[OK]"
2989                 else
2990                     echo "[FAILED]"
2991                     echo "see $TESTLOG"
2992                 fi
2993                 exit $ret
2994                 ;;
2995             *)
2996                 break
2997                 ;;
2998         esac
2999         shift
3000     done
3001 }