ceph/src/spdk/test/common/autotest_common.sh

   1 #!/usr/bin/env bash
   2
   3 function xtrace_disable() {
   4         if [ "$XTRACE_DISABLED" != "yes" ]; then
   5                 PREV_BASH_OPTS="$-"
   6                 if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then
   7                         XTRACE_DISABLED="yes"
   8                 fi
   9                 set +x
  10         elif [ -z $XTRACE_NESTING_LEVEL ]; then
  11                 XTRACE_NESTING_LEVEL=1
  12         else
  13                 XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL))
  14         fi
  15 }
  16
  17 xtrace_disable
  18 set -e
  19 shopt -s expand_aliases
  20
  21 source "$rootdir/test/common/applications.sh"
  22 if [[ -e $rootdir/test/common/build_config.sh ]]; then
  23         source "$rootdir/test/common/build_config.sh"
  24 elif [[ -e $rootdir/mk/config.mk ]]; then
  25         build_config=$(< "$rootdir/mk/config.mk")
  26         source <(echo "${build_config//\?=/=}")
  27 else
  28         source "$rootdir/CONFIG"
  29 fi
  30
  31 # Dummy function to be called after restoring xtrace just so that it appears in the
  32 # xtrace log. This way we can consistently track when xtrace is enabled/disabled.
  33 function xtrace_enable() {
  34         # We have to do something inside a function in bash, and calling any command
  35         # (even `:`) will produce an xtrace entry, so we just define another function.
  36         function xtrace_dummy() { :; }
  37 }
  38
  39 # Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore.
  40 # xtrace_enable will appear as called directly from the user script, from the same line
  41 # that "called" xtrace_restore.
  42 alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then
  43         if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then
  44                 XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable;
  45         fi
  46 else
  47         XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL));
  48         if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then
  49                 unset XTRACE_NESTING_LEVEL
  50         fi
  51 fi'
  52
  53 : ${RUN_NIGHTLY:=0}
  54 export RUN_NIGHTLY
  55
  56 # Set defaults for missing test config options
  57 : ${SPDK_AUTOTEST_DEBUG_APPS:=0}
  58 export SPDK_AUTOTEST_DEBUG_APPS
  59 : ${SPDK_RUN_VALGRIND=0}
  60 export SPDK_RUN_VALGRIND
  61 : ${SPDK_RUN_FUNCTIONAL_TEST=0}
  62 export SPDK_RUN_FUNCTIONAL_TEST
  63 : ${SPDK_TEST_UNITTEST=0}
  64 export SPDK_TEST_UNITTEST
  65 : ${SPDK_TEST_AUTOBUILD=0}
  66 export SPDK_TEST_AUTOBUILD
  67 : ${SPDK_TEST_ISAL=0}
  68 export SPDK_TEST_ISAL
  69 : ${SPDK_TEST_ISCSI=0}
  70 export SPDK_TEST_ISCSI
  71 : ${SPDK_TEST_ISCSI_INITIATOR=0}
  72 export SPDK_TEST_ISCSI_INITIATOR
  73 : ${SPDK_TEST_NVME=0}
  74 export SPDK_TEST_NVME
  75 : ${SPDK_TEST_NVME_CLI=0}
  76 export SPDK_TEST_NVME_CLI
  77 : ${SPDK_TEST_NVME_CUSE=0}
  78 export SPDK_TEST_NVME_CUSE
  79 : ${SPDK_TEST_NVMF=0}
  80 export SPDK_TEST_NVMF
  81 : ${SPDK_TEST_NVMF_TRANSPORT="rdma"}
  82 export SPDK_TEST_NVMF_TRANSPORT
  83 : ${SPDK_TEST_RBD=0}
  84 export SPDK_TEST_RBD
  85 : ${SPDK_TEST_VHOST=0}
  86 export SPDK_TEST_VHOST
  87 : ${SPDK_TEST_BLOCKDEV=0}
  88 export SPDK_TEST_BLOCKDEV
  89 : ${SPDK_TEST_IOAT=0}
  90 export SPDK_TEST_IOAT
  91 : ${SPDK_TEST_BLOBFS=0}
  92 export SPDK_TEST_BLOBFS
  93 : ${SPDK_TEST_VHOST_INIT=0}
  94 export SPDK_TEST_VHOST_INIT
  95 : ${SPDK_TEST_PMDK=0}
  96 export SPDK_TEST_PMDK
  97 : ${SPDK_TEST_LVOL=0}
  98 export SPDK_TEST_LVOL
  99 : ${SPDK_TEST_JSON=0}
 100 export SPDK_TEST_JSON
 101 : ${SPDK_TEST_REDUCE=0}
 102 export SPDK_TEST_REDUCE
 103 : ${SPDK_TEST_VPP=0}
 104 export SPDK_TEST_VPP
 105 : ${SPDK_RUN_ASAN=0}
 106 export SPDK_RUN_ASAN
 107 : ${SPDK_RUN_UBSAN=0}
 108 export SPDK_RUN_UBSAN
 109 : ${SPDK_RUN_INSTALLED_DPDK=0}
 110 export SPDK_RUN_INSTALLED_DPDK
 111 : ${SPDK_RUN_NON_ROOT=0}
 112 export SPDK_RUN_NON_ROOT
 113 : ${SPDK_TEST_CRYPTO=0}
 114 export SPDK_TEST_CRYPTO
 115 : ${SPDK_TEST_FTL=0}
 116 export SPDK_TEST_FTL
 117 : ${SPDK_TEST_OCF=0}
 118 export SPDK_TEST_OCF
 119 : ${SPDK_TEST_FTL_EXTENDED=0}
 120 export SPDK_TEST_FTL_EXTENDED
 121 : ${SPDK_TEST_VMD=0}
 122 export SPDK_TEST_VMD
 123 : ${SPDK_TEST_OPAL=0}
 124 export SPDK_TEST_OPAL
 125 : ${SPDK_AUTOTEST_X=true}
 126 export SPDK_AUTOTEST_X
 127 : ${SPDK_TEST_RAID5=0}
 128 export SPDK_TEST_RAID5
 129 : ${SPDK_TEST_URING=0}
 130 export SPDK_TEST_URING
 131
 132 # Export PYTHONPATH with addition of RPC framework. New scripts can be created
 133 # specific use cases for tests.
 134 export PYTHONPATH=$PYTHONPATH:$rootdir/scripts
 135
 136 # Don't create Python .pyc files. When running with sudo these will be
 137 # created with root ownership and can cause problems when cleaning the repository.
 138 export PYTHONDONTWRITEBYTECODE=1
 139
 140 # Export flag to skip the known bug that exists in librados
 141 # Bug is reported on ceph bug tracker with number 24078
 142 export ASAN_OPTIONS=new_delete_type_mismatch=0
 143 export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1'
 144
 145 # Export LeakSanitizer option to use suppression file in order to prevent false positives
 146 # and known leaks in external executables or libraries from showing up.
 147 asan_suppression_file="/var/tmp/asan_suppression_file"
 148 sudo rm -rf "$asan_suppression_file"
 149 cat << EOL >> "$asan_suppression_file"
 150 # ASAN has some bugs around thread_local variables.  We have a destructor in place
 151 # to free the thread contexts, but ASAN complains about the leak before those
 152 # destructors have a chance to run.  So suppress this one specific leak using
 153 # LSAN_OPTIONS.
 154 leak:spdk_fs_alloc_thread_ctx
 155
 156 # Suppress known leaks in fio project
 157 leak:$CONFIG_FIO_SOURCE_DIR/parse.c
 158 leak:$CONFIG_FIO_SOURCE_DIR/iolog.c
 159 leak:$CONFIG_FIO_SOURCE_DIR/init.c
 160 leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c
 161 leak:fio_memalign
 162 leak:spdk_fio_io_u_init
 163
 164 # Suppress leaks in libiscsi
 165 leak:libiscsi.so
 166 EOL
 167
 168 # Suppress leaks in libfuse3
 169 echo "leak:libfuse3.so" >> "$asan_suppression_file"
 170
 171 export LSAN_OPTIONS=suppressions="$asan_suppression_file"
 172
 173 export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock"
 174
 175 if [ -z "$DEPENDENCY_DIR" ]; then
 176         export DEPENDENCY_DIR=/home/sys_sgsw
 177 else
 178         export DEPENDENCY_DIR
 179 fi
 180
 181 # Export location of where all the SPDK binaries are
 182 export SPDK_BIN_DIR="$rootdir/build/bin"
 183 export SPDK_EXAMPLE_DIR="$rootdir/build/examples"
 184
 185 # pass our valgrind desire on to unittest.sh
 186 if [ $SPDK_RUN_VALGRIND -eq 0 ]; then
 187         export valgrind=''
 188 fi
 189
 190 if [ "$(uname -s)" = "Linux" ]; then
 191         MAKE="make"
 192         MAKEFLAGS=${MAKEFLAGS:--j$(nproc)}
 193         DPDK_LINUX_DIR=/usr/share/dpdk/x86_64-default-linuxapp-gcc
 194         if [ -d $DPDK_LINUX_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then
 195                 WITH_DPDK_DIR=$DPDK_LINUX_DIR
 196         fi
 197         # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages.
 198         export HUGEMEM=8192
 199 elif [ "$(uname -s)" = "FreeBSD" ]; then
 200         MAKE="gmake"
 201         MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')}
 202         DPDK_FREEBSD_DIR=/usr/local/share/dpdk/x86_64-native-bsdapp-clang
 203         if [ -d $DPDK_FREEBSD_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then
 204                 WITH_DPDK_DIR=$DPDK_FREEBSD_DIR
 205         fi
 206         # FreeBSD runs a much more limited set of tests, so keep the default 2GB.
 207         export HUGEMEM=2048
 208 else
 209         echo "Unknown OS \"$(uname -s)\""
 210         exit 1
 211 fi
 212
 213 if [ -z "$output_dir" ]; then
 214         if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then
 215                 output_dir=.
 216         else
 217                 output_dir=$rootdir/../output
 218         fi
 219         export output_dir
 220 fi
 221
 222 TEST_MODE=
 223 for i in "$@"; do
 224         case "$i" in
 225                 --iso)
 226                         TEST_MODE=iso
 227                         ;;
 228                 --transport=*)
 229                         TEST_TRANSPORT="${i#*=}"
 230                         ;;
 231                 --sock=*)
 232                         TEST_SOCK="${i#*=}"
 233                         ;;
 234         esac
 235 done
 236
 237 # start rpc.py coprocess if it's not started yet
 238 if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then
 239         coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; }
 240         exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]}
 241         # all descriptors will automatically close together with this bash
 242         # process, this will make rpc.py stop reading and exit gracefully
 243 fi
 244
 245 if [ $SPDK_TEST_VPP -eq 1 ]; then
 246         VPP_PATH="/usr/local/src/vpp-19.04/build-root/install-vpp_debug-native/vpp/"
 247         export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${VPP_PATH}/lib/
 248         export PATH=${PATH}:${VPP_PATH}/bin/
 249 fi
 250
 251 function set_test_storage() {
 252         [[ -v testdir ]] || return 0
 253
 254         local requested_size=$1 # bytes
 255         local mount target_dir
 256
 257         local -A mounts fss sizes avails uses
 258         local source fs size avail mount use
 259
 260         local storage_fallback storage_candidates
 261         local storage_fallback_purge
 262
 263         storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????)
 264
 265         if ((${#storage_fallback_purge[@]} > 0)); then
 266                 printf '* Purging old temporary test storage (%s)\n' \
 267                         "${storage_fallback_purge[*]}" >&2
 268                 rm -rf "${storage_fallback_purge[@]}"
 269         fi
 270
 271         storage_fallback=$(mktemp -udt spdk.XXXXXX)
 272         storage_candidates=(
 273                 "$testdir"
 274                 "$storage_fallback/tests/${testdir##*/}"
 275                 "$storage_fallback"
 276         )
 277
 278         if [[ -n $ADD_TEST_STORAGE ]]; then
 279                 # List of dirs|mounts separated by whitespaces
 280                 storage_candidates+=($ADD_TEST_STORAGE)
 281         fi
 282
 283         if [[ -n $DEDICATED_TEST_STORAGE ]]; then
 284                 # Single, dedicated dir|mount
 285                 storage_candidates=("$DEDICATED_TEST_STORAGE")
 286         fi
 287
 288         mkdir -p "${storage_candidates[@]}"
 289
 290         # add some headroom - 64M
 291         requested_size=$((requested_size + (64 << 20)))
 292
 293         while read -r source fs size use avail _ mount; do
 294                 mounts["$mount"]=$source fss["$mount"]=$fs
 295                 avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024))
 296                 uses["$mount"]=$((use * 1024))
 297         done < <(df -T | grep -v Filesystem)
 298
 299         printf '* Looking for test storage...\n' >&2
 300
 301         local target_space new_size
 302         for target_dir in "${storage_candidates[@]}"; do
 303                 # FreeBSD's df is lacking the --output arg
 304                 # mount=$(df --output=target "$target_dir" | grep -v "Mounted on")
 305                 mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}')
 306
 307                 target_space=${avails["$mount"]}
 308                 if ((target_space == 0 || target_space < requested_size)); then
 309                         continue
 310                 fi
 311                 if ((target_space >= requested_size)); then
 312                         # For in-memory fs, and / make sure our requested size won't fill most of the space.
 313                         if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then
 314                                 new_size=$((uses["$mount"] + requested_size))
 315                                 if ((new_size * 100 / sizes["$mount"] > 95)); then
 316                                         continue
 317                                 fi
 318                         fi
 319                 fi
 320                 export SPDK_TEST_STORAGE=$target_dir
 321                 printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2
 322                 return 0
 323         done
 324         printf '* Test storage is not available\n'
 325         return 1
 326 }
 327
 328 function get_config_params() {
 329         xtrace_disable
 330         config_params='--enable-debug --enable-werror'
 331
 332         # for options with dependencies but no test flag, set them here
 333         if [ -f /usr/include/infiniband/verbs.h ]; then
 334                 config_params+=' --with-rdma'
 335         fi
 336
 337         if [ $(uname -s) == "FreeBSD" ]; then
 338                 intel="hw.model: Intel"
 339                 cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15)
 340         else
 341                 intel="GenuineIntel"
 342                 cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1)
 343         fi
 344         if [[ "$cpu_vendor" != *"$intel"* ]]; then
 345                 config_params+=" --without-idxd"
 346         else
 347                 config_params+=" --with-idxd"
 348         fi
 349
 350         if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then
 351                 config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR"
 352         fi
 353
 354         if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then
 355                 config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes'
 356         fi
 357
 358         if [ -d /usr/include/iscsi ]; then
 359                 libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}')
 360                 if [ $libiscsi_version -ge 20150621 ]; then
 361                         config_params+=' --with-iscsi-initiator'
 362                 fi
 363         fi
 364
 365         if [ $SPDK_TEST_UNITTEST -eq 0 ]; then
 366                 config_params+=' --disable-unit-tests'
 367         fi
 368
 369         if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then
 370                 config_params+=' --with-nvme-cuse'
 371         fi
 372
 373         # for options with both dependencies and a test flag, set them here
 374         if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then
 375                 config_params+=' --with-pmdk'
 376         fi
 377
 378         if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then
 379                 if [ $SPDK_TEST_ISAL -eq 1 ]; then
 380                         config_params+=' --with-reduce'
 381                 fi
 382         fi
 383
 384         if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then
 385                 config_params+=' --with-rbd'
 386         fi
 387
 388         if [ $SPDK_TEST_VPP -eq 1 ]; then
 389                 config_params+=" --with-vpp=${VPP_PATH}"
 390         fi
 391
 392         # for options with no required dependencies, just test flags, set them here
 393         if [ $SPDK_TEST_CRYPTO -eq 1 ]; then
 394                 config_params+=' --with-crypto'
 395         fi
 396
 397         if [ $SPDK_TEST_OCF -eq 1 ]; then
 398                 config_params+=" --with-ocf"
 399         fi
 400
 401         if [ $SPDK_RUN_UBSAN -eq 1 ]; then
 402                 config_params+=' --enable-ubsan'
 403         fi
 404
 405         if [ $SPDK_RUN_ASAN -eq 1 ]; then
 406                 config_params+=' --enable-asan'
 407         fi
 408
 409         if [ "$(uname -s)" = "Linux" ]; then
 410                 config_params+=' --enable-coverage'
 411         fi
 412
 413         if [ $SPDK_TEST_ISAL -eq 0 ]; then
 414                 config_params+=' --without-isal'
 415         fi
 416
 417         if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
 418                 if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then
 419                         config_params+=' --with-fuse'
 420                 fi
 421         fi
 422
 423         if [ $SPDK_TEST_RAID5 -eq 1 ]; then
 424                 config_params+=' --with-raid5'
 425         fi
 426
 427         # Check whether liburing library header exists
 428         if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then
 429                 config_params+=' --with-uring'
 430         fi
 431
 432         # By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule.
 433         # If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be
 434         # set which will override the default and use that DPDK installation instead.
 435         if [ -n "$WITH_DPDK_DIR" ]; then
 436                 config_params+=" --with-dpdk=$WITH_DPDK_DIR"
 437         fi
 438
 439         echo "$config_params"
 440         xtrace_restore
 441 }
 442
 443 function rpc_cmd() {
 444         xtrace_disable
 445         local rsp rc
 446
 447         echo "$@" >&$RPC_PIPE_INPUT
 448         while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do
 449                 if [[ $rsp == "**STATUS="* ]]; then
 450                         break
 451                 fi
 452                 echo "$rsp"
 453         done
 454
 455         rc=${rsp#*=}
 456         xtrace_restore
 457         [[ $rc == 0 ]]
 458 }
 459
 460 function rpc_cmd_simple_data_json() {
 461
 462         local elems="$1[@]" elem
 463         local -gA jq_out=()
 464         local jq val
 465
 466         local lvs=(
 467                 "uuid"
 468                 "name"
 469                 "base_bdev"
 470                 "total_data_clusters"
 471                 "free_clusters"
 472                 "block_size"
 473                 "cluster_size"
 474         )
 475
 476         local bdev=(
 477                 "name"
 478                 "aliases[0]"
 479                 "block_size"
 480                 "num_blocks"
 481                 "uuid"
 482                 "product_name"
 483         )
 484
 485         [[ -v $elems ]] || return 1
 486
 487         for elem in "${!elems}"; do
 488                 jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem"
 489         done
 490         jq+=',"\n"'
 491
 492         shift
 493         while read -r elem val; do
 494                 jq_out["$elem"]=$val
 495         done < <(rpc_cmd "$@" | jq -jr "$jq")
 496         ((${#jq_out[@]} > 0)) || return 1
 497 }
 498
 499 # invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix)
 500 function NOT() {
 501         if "$@"; then
 502                 return 1
 503         fi
 504 }
 505
 506 function timing() {
 507         direction="$1"
 508         testname="$2"
 509
 510         now=$(date +%s)
 511
 512         if [ "$direction" = "enter" ]; then
 513                 export timing_stack="${timing_stack};${now}"
 514                 export test_stack="${test_stack};${testname}"
 515         else
 516                 touch "$output_dir/timing.txt"
 517                 child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}')
 518
 519                 start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@')
 520                 timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@')
 521
 522                 elapsed=$((now - start_time - child_time))
 523                 echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt
 524
 525                 test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@')
 526         fi
 527 }
 528
 529 function timing_enter() {
 530         xtrace_disable
 531         timing "enter" "$1"
 532         xtrace_restore
 533 }
 534
 535 function timing_exit() {
 536         xtrace_disable
 537         timing "exit" "$1"
 538         xtrace_restore
 539 }
 540
 541 function timing_finish() {
 542         flamegraph='/usr/local/FlameGraph/flamegraph.pl'
 543         if [ -x "$flamegraph" ]; then
 544                 "$flamegraph" \
 545                         --title 'Build Timing' \
 546                         --nametype 'Step:' \
 547                         --countname seconds \
 548                         $output_dir/timing.txt \
 549                         > $output_dir/timing.svg
 550         fi
 551 }
 552
 553 function create_test_list() {
 554         xtrace_disable
 555         # First search all scripts in main SPDK directory.
 556         completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*)
 557         # Follow up with search in test directory recursively.
 558         completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test)
 559         printf "%s" "$completion" | grep -v "#" \
 560                 | sed 's/^.*run_test/run_test/' | awk '{print $2}' \
 561                 | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true
 562         xtrace_restore
 563 }
 564
 565 function gdb_attach() {
 566         gdb -q --batch \
 567                 -ex 'handle SIGHUP nostop pass' \
 568                 -ex 'handle SIGQUIT nostop pass' \
 569                 -ex 'handle SIGPIPE nostop pass' \
 570                 -ex 'handle SIGALRM nostop pass' \
 571                 -ex 'handle SIGTERM nostop pass' \
 572                 -ex 'handle SIGUSR1 nostop pass' \
 573                 -ex 'handle SIGUSR2 nostop pass' \
 574                 -ex 'handle SIGCHLD nostop pass' \
 575                 -ex 'set print thread-events off' \
 576                 -ex 'cont' \
 577                 -ex 'thread apply all bt' \
 578                 -ex 'quit' \
 579                 --tty=/dev/stdout \
 580                 -p $1
 581 }
 582
 583 function process_core() {
 584         ret=0
 585         while IFS= read -r -d '' core; do
 586                 exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/")
 587                 if [[ ! -f "$exe" ]]; then
 588                         exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+")
 589                 fi
 590                 echo "exe for $core is $exe"
 591                 if [[ -n "$exe" ]]; then
 592                         if hash gdb &> /dev/null; then
 593                                 gdb -batch -ex "thread apply all bt full" $exe $core
 594                         fi
 595                         cp $exe $output_dir
 596                 fi
 597                 mv $core $output_dir
 598                 chmod a+r $output_dir/$core
 599                 ret=1
 600         done < <(find . -type f \( -name 'core\.?[0-9]*' -o -name '*.core' \) -print0)
 601         return $ret
 602 }
 603
 604 function process_shm() {
 605         type=$1
 606         id=$2
 607         if [ "$type" = "--pid" ]; then
 608                 id="pid${id}"
 609         elif [ "$type" = "--id" ]; then
 610                 id="${id}"
 611         else
 612                 echo "Please specify to search for pid or shared memory id."
 613                 return 1
 614         fi
 615
 616         shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n")
 617
 618         if [[ -z $shm_files ]]; then
 619                 echo "SHM File for specified PID or shared memory id: ${id} not found!"
 620                 return 1
 621         fi
 622         for n in $shm_files; do
 623                 tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n}
 624         done
 625         return 0
 626 }
 627
 628 function waitforlisten() {
 629         # $1 = process pid
 630         if [ -z "$1" ]; then
 631                 exit 1
 632         fi
 633
 634         local rpc_addr="${2:-$DEFAULT_RPC_ADDR}"
 635
 636         echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..."
 637         # turn off trace for this loop
 638         xtrace_disable
 639         local ret=0
 640         local i
 641         for ((i = 40; i != 0; i--)); do
 642                 # if the process is no longer running, then exit the script
 643                 #  since it means the application crashed
 644                 if ! kill -s 0 $1; then
 645                         echo "ERROR: process (pid: $1) is no longer running"
 646                         ret=1
 647                         break
 648                 fi
 649
 650                 if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then
 651                         break
 652                 fi
 653
 654                 sleep 0.5
 655         done
 656
 657         xtrace_restore
 658         if ((i == 0)); then
 659                 echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'"
 660                 ret=1
 661         fi
 662         return $ret
 663 }
 664
 665 function waitfornbd() {
 666         local nbd_name=$1
 667         local i
 668
 669         for ((i = 1; i <= 20; i++)); do
 670                 if grep -q -w $nbd_name /proc/partitions; then
 671                         break
 672                 else
 673                         sleep 0.1
 674                 fi
 675         done
 676
 677         # The nbd device is now recognized as a block device, but there can be
 678         #  a small delay before we can start I/O to that block device.  So loop
 679         #  here trying to read the first block of the nbd block device to a temp
 680         #  file.  Note that dd returns success when reading an empty file, so we
 681         #  need to check the size of the output file instead.
 682         for ((i = 1; i <= 20; i++)); do
 683                 dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct
 684                 size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest")
 685                 rm -f "$SPDK_TEST_STORAGE/nbdtest"
 686                 if [ "$size" != "0" ]; then
 687                         return 0
 688                 else
 689                         sleep 0.1
 690                 fi
 691         done
 692
 693         return 1
 694 }
 695
 696 function waitforbdev() {
 697         local bdev_name=$1
 698         local i
 699
 700         for ((i = 1; i <= 20; i++)); do
 701                 if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then
 702                         return 0
 703                 fi
 704
 705                 if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then
 706                         return 0
 707                 fi
 708
 709                 sleep 0.1
 710         done
 711
 712         return 1
 713 }
 714
 715 function make_filesystem() {
 716         local fstype=$1
 717         local dev_name=$2
 718         local i=0
 719         local force
 720
 721         if [ $fstype = ext4 ]; then
 722                 force=-F
 723         else
 724                 force=-f
 725         fi
 726
 727         while ! mkfs.${fstype} $force ${dev_name}; do
 728                 if [ $i -ge 15 ]; then
 729                         return 1
 730                 fi
 731                 i=$((i + 1))
 732                 sleep 1
 733         done
 734
 735         return 0
 736 }
 737
 738 function killprocess() {
 739         # $1 = process pid
 740         if [ -z "$1" ]; then
 741                 exit 1
 742         fi
 743
 744         if kill -0 $1; then
 745                 if [ $(uname) = Linux ]; then
 746                         process_name=$(ps --no-headers -o comm= $1)
 747                 else
 748                         process_name=$(ps -c -o command $1 | tail -1)
 749                 fi
 750                 if [ "$process_name" = "sudo" ]; then
 751                         # kill the child process, which is the actual app
 752                         # (assume $1 has just one child)
 753                         local child
 754                         child="$(pgrep -P $1)"
 755                         echo "killing process with pid $child"
 756                         kill $child
 757                 else
 758                         echo "killing process with pid $1"
 759                         kill $1
 760                 fi
 761
 762                 # wait for the process regardless if its the dummy sudo one
 763                 # or the actual app - it should terminate anyway
 764                 wait $1
 765         else
 766                 # the process is not there anymore
 767                 echo "Process with pid $1 is not found"
 768                 exit 1
 769         fi
 770 }
 771
 772 function iscsicleanup() {
 773         echo "Cleaning up iSCSI connection"
 774         iscsiadm -m node --logout || true
 775         iscsiadm -m node -o delete || true
 776         rm -rf /var/lib/iscsi/nodes/*
 777 }
 778
 779 function stop_iscsi_service() {
 780         if cat /etc/*-release | grep Ubuntu; then
 781                 service open-iscsi stop
 782         else
 783                 service iscsid stop
 784         fi
 785 }
 786
 787 function start_iscsi_service() {
 788         if cat /etc/*-release | grep Ubuntu; then
 789                 service open-iscsi start
 790         else
 791                 service iscsid start
 792         fi
 793 }
 794
 795 function rbd_setup() {
 796         # $1 = monitor ip address
 797         # $2 = name of the namespace
 798         if [ -z "$1" ]; then
 799                 echo "No monitor IP address provided for ceph"
 800                 exit 1
 801         fi
 802         if [ -n "$2" ]; then
 803                 if ip netns list | grep "$2"; then
 804                         NS_CMD="ip netns exec $2"
 805                 else
 806                         echo "No namespace $2 exists"
 807                         exit 1
 808                 fi
 809         fi
 810
 811         if hash ceph; then
 812                 export PG_NUM=128
 813                 export RBD_POOL=rbd
 814                 export RBD_NAME=foo
 815                 $NS_CMD $rootdir/scripts/ceph/stop.sh || true
 816                 $NS_CMD $rootdir/scripts/ceph/start.sh $1
 817
 818                 $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true
 819                 $NS_CMD rbd create $RBD_NAME --size 1000
 820         fi
 821 }
 822
 823 function rbd_cleanup() {
 824         if hash ceph; then
 825                 $rootdir/scripts/ceph/stop.sh || true
 826                 rm -f /var/tmp/ceph_raw.img
 827         fi
 828 }
 829
 830 function nvme_cli_build() {
 831         if [[ -z "${DEPENDENCY_DIR}" ]]; then
 832                 echo DEPENDENCY_DIR not defined!
 833                 exit 1
 834         fi
 835
 836         spdk_nvme_cli="${DEPENDENCY_DIR}/nvme-cli"
 837
 838         if [[ ! -d $spdk_nvme_cli ]]; then
 839                 echo "nvme-cli repository not found at $spdk_nvme_cli; skipping tests."
 840                 exit 1
 841         fi
 842
 843         if ! grep -q "DEF_VER=v1.6" $spdk_nvme_cli/NVME-VERSION-GEN; then
 844                 echo "SPDK supports only \"spdk/nvme-cli\" project on \"spdk-1.6\" branch."
 845                 exit 1
 846         fi
 847
 848         # Build against the version of SPDK under test
 849         pushd $spdk_nvme_cli
 850
 851         # Remove and recreate git index in case it became corrupted
 852         if ! git clean -dfx; then
 853                 rm -f .git/index
 854                 git clean -dfx
 855                 git reset --hard
 856         fi
 857
 858         rm -f "$spdk_nvme_cli/spdk"
 859         ln -sf "$rootdir" "$spdk_nvme_cli/spdk"
 860
 861         make -j$(nproc) LDFLAGS="$(make -s -C $spdk_nvme_cli/spdk ldflags)"
 862         popd
 863 }
 864
 865 function _start_stub() {
 866         # Disable ASLR for multi-process testing.  SPDK does support using DPDK multi-process,
 867         # but ASLR can still be unreliable in some cases.
 868         # We will reenable it again after multi-process testing is complete in kill_stub().
 869         # Save current setting so it can be restored upon calling kill_stub().
 870         _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space)
 871         echo 0 > /proc/sys/kernel/randomize_va_space
 872         $rootdir/test/app/stub/stub $1 &
 873         stubpid=$!
 874         echo Waiting for stub to ready for secondary processes...
 875         while ! [ -e /var/run/spdk_stub0 ]; do
 876                 # If stub dies while we wait, bail
 877                 [[ -e /proc/$stubpid ]] || return 1
 878                 sleep 1s
 879         done
 880         echo done.
 881 }
 882
 883 function start_stub() {
 884         if ! _start_stub "$@"; then
 885                 echo "stub failed" >&2
 886                 return 1
 887         fi
 888 }
 889
 890 function kill_stub() {
 891         if [[ -e /proc/$stubpid ]]; then
 892                 kill $1 $stubpid
 893                 wait $stubpid
 894         fi 2> /dev/null || :
 895         rm -f /var/run/spdk_stub0
 896         # Re-enable ASLR now that we are done with multi-process testing
 897         # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment
 898         #  randomizing and is the default on all recent Linux kernels
 899         echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space
 900 }
 901
 902 function run_test() {
 903         if [ $# -le 1 ]; then
 904                 echo "Not enough parameters"
 905                 echo "usage: run_test test_name test_script [script_params]"
 906                 exit 1
 907         fi
 908
 909         xtrace_disable
 910         local test_name="$1"
 911         shift
 912
 913         if [ -n "$test_domain" ]; then
 914                 export test_domain="${test_domain}.${test_name}"
 915         else
 916                 export test_domain="$test_name"
 917         fi
 918
 919         timing_enter $test_name
 920         echo "************************************"
 921         echo "START TEST $test_name"
 922         echo "************************************"
 923         xtrace_restore
 924         time "$@"
 925         xtrace_disable
 926         echo "************************************"
 927         echo "END TEST $test_name"
 928         echo "************************************"
 929         timing_exit $test_name
 930
 931         export test_domain=${test_domain%"$test_name"}
 932         if [ -n "$test_domain" ]; then
 933                 export test_domain=${test_domain%?}
 934         fi
 935
 936         if [ -z "$test_domain" ]; then
 937                 echo "top_level $test_name" >> $output_dir/test_completions.txt
 938         else
 939                 echo "$test_domain $test_name" >> $output_dir/test_completions.txt
 940         fi
 941         xtrace_restore
 942 }
 943
 944 function skip_run_test_with_warning() {
 945         echo "WARNING: $1"
 946         echo "Test run may fail if run with autorun.sh"
 947         echo "Please check your $rootdir/test/common/skipped_tests.txt"
 948 }
 949
 950 function print_backtrace() {
 951         # if errexit is not enabled, don't print a backtrace
 952         [[ "$-" =~ e ]] || return 0
 953
 954         local args=("${BASH_ARGV[@]}")
 955
 956         xtrace_disable
 957         echo "========== Backtrace start: =========="
 958         echo ""
 959         for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do
 960                 local func="${FUNCNAME[$i]}"
 961                 local line_nr="${BASH_LINENO[$((i - 1))]}"
 962                 local src="${BASH_SOURCE[$i]}"
 963                 local bt="" cmdline=()
 964
 965                 if [[ -f $src ]]; then
 966                         bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \
 967                                 | sed "s/^/   /g" | sed "s/^   $line_nr /=> $line_nr /g")
 968                 fi
 969
 970                 # If extdebug set the BASH_ARGC[i], try to fetch all the args
 971                 if ((BASH_ARGC[i] > 0)); then
 972                         # Use argc as index to reverse the stack
 973                         local argc=${BASH_ARGC[i]} arg
 974                         for arg in "${args[@]::BASH_ARGC[i]}"; do
 975                                 cmdline[argc--]="[\"$arg\"]"
 976                         done
 977                         args=("${args[@]:BASH_ARGC[i]}")
 978                 fi
 979
 980                 echo "in $src:$line_nr -> $func($(
 981                         IFS=","
 982                         printf '%s\n' "${cmdline[*]:-[]}"
 983                 ))"
 984                 echo "     ..."
 985                 echo "${bt:-backtrace unavailable}"
 986                 echo "     ..."
 987         done
 988         echo ""
 989         echo "========== Backtrace end =========="
 990         xtrace_restore
 991         return 0
 992 }
 993
 994 function waitforserial() {
 995         local i=0
 996         local nvme_device_counter=1
 997         if [[ -n "$2" ]]; then
 998                 nvme_device_counter=$2
 999         fi
1000
1001         while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do
1002                 [ $i -lt 15 ] || break
1003                 i=$((i + 1))
1004                 echo "Waiting for devices"
1005                 sleep 1
1006         done
1007
1008         if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then
1009                 return 1
1010         fi
1011
1012         return 0
1013 }
1014
1015 function waitforserial_disconnect() {
1016         local i=0
1017         while lsblk -o NAME,SERIAL | grep -q -w $1; do
1018                 [ $i -lt 15 ] || break
1019                 i=$((i + 1))
1020                 echo "Waiting for disconnect devices"
1021                 sleep 1
1022         done
1023
1024         if lsblk -l -o NAME | grep -q -w $1; then
1025                 return 1
1026         fi
1027
1028         return 0
1029 }
1030
1031 function waitforblk() {
1032         local i=0
1033         while ! lsblk -l -o NAME | grep -q -w $1; do
1034                 [ $i -lt 15 ] || break
1035                 i=$((i + 1))
1036                 sleep 1
1037         done
1038
1039         if ! lsblk -l -o NAME | grep -q -w $1; then
1040                 return 1
1041         fi
1042
1043         return 0
1044 }
1045
1046 function waitforblk_disconnect() {
1047         local i=0
1048         while lsblk -l -o NAME | grep -q -w $1; do
1049                 [ $i -lt 15 ] || break
1050                 i=$((i + 1))
1051                 sleep 1
1052         done
1053
1054         if lsblk -l -o NAME | grep -q -w $1; then
1055                 return 1
1056         fi
1057
1058         return 0
1059 }
1060
1061 function waitforfile() {
1062         local i=0
1063         while [ ! -e $1 ]; do
1064                 [ $i -lt 200 ] || break
1065                 i=$((i + 1))
1066                 sleep 0.1
1067         done
1068
1069         if [ ! -e $1 ]; then
1070                 return 1
1071         fi
1072
1073         return 0
1074 }
1075
1076 function fio_config_gen() {
1077         local config_file=$1
1078         local workload=$2
1079         local bdev_type=$3
1080         local fio_dir=$CONFIG_FIO_SOURCE_DIR
1081
1082         if [ -e "$config_file" ]; then
1083                 echo "Configuration File Already Exists!: $config_file"
1084                 return 1
1085         fi
1086
1087         if [ -z "$workload" ]; then
1088                 workload=randrw
1089         fi
1090
1091         touch $1
1092
1093         cat > $1 << EOL
1094 [global]
1095 thread=1
1096 group_reporting=1
1097 direct=1
1098 norandommap=1
1099 percentile_list=50:99:99.9:99.99:99.999
1100 time_based=1
1101 ramp_time=0
1102 EOL
1103
1104         if [ "$workload" == "verify" ]; then
1105                 cat <<- EOL >> $config_file
1106                         verify=sha1
1107                         verify_backlog=1024
1108                         rw=randwrite
1109                 EOL
1110
1111                 # To avoid potential data race issue due to the AIO device
1112                 # flush mechanism, add the flag to serialize the writes.
1113                 # This is to fix the intermittent IO failure issue of #935
1114                 if [ "$bdev_type" == "AIO" ]; then
1115                         if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then
1116                                 echo "serialize_overlap=1" >> $config_file
1117                         fi
1118                 fi
1119         elif [ "$workload" == "trim" ]; then
1120                 echo "rw=trimwrite" >> $config_file
1121         else
1122                 echo "rw=$workload" >> $config_file
1123         fi
1124 }
1125
1126 function fio_bdev() {
1127         # Setup fio binary cmd line
1128         local fio_dir=$CONFIG_FIO_SOURCE_DIR
1129         local bdev_plugin="$rootdir/build/fio/spdk_bdev"
1130
1131         # Preload AddressSanitizer library to fio if fio_plugin was compiled with it
1132         local asan_lib
1133         asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}')
1134
1135         LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@"
1136 }
1137
1138 function fio_nvme() {
1139         # Setup fio binary cmd line
1140         local fio_dir=$CONFIG_FIO_SOURCE_DIR
1141         local nvme_plugin="$rootdir/build/fio/spdk_nvme"
1142
1143         # Preload AddressSanitizer library to fio if fio_plugin was compiled with it
1144         asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}')
1145
1146         LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@"
1147 }
1148
1149 function get_lvs_free_mb() {
1150         local lvs_uuid=$1
1151         local lvs_info
1152         local fc
1153         local cs
1154         lvs_info=$($rpc_py bdev_lvol_get_lvstores)
1155         fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info")
1156         cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info")
1157
1158         # Change to MB's
1159         free_mb=$((fc * cs / 1024 / 1024))
1160         echo "$free_mb"
1161 }
1162
1163 function get_bdev_size() {
1164         local bdev_name=$1
1165         local bdev_info
1166         local bs
1167         local nb
1168         bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name)
1169         bs=$(jq ".[] .block_size" <<< "$bdev_info")
1170         nb=$(jq ".[] .num_blocks" <<< "$bdev_info")
1171
1172         # Change to MB's
1173         bdev_size=$((bs * nb / 1024 / 1024))
1174         echo "$bdev_size"
1175 }
1176
1177 function autotest_cleanup() {
1178         $rootdir/scripts/setup.sh reset
1179         $rootdir/scripts/setup.sh cleanup
1180         if [ $(uname -s) = "Linux" ]; then
1181                 if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then
1182                         [[ -e /sys/module/igb_uio ]] && rmmod igb_uio
1183                 else
1184                         modprobe -r uio_pci_generic
1185                 fi
1186         fi
1187         rm -rf "$asan_suppression_file"
1188 }
1189
1190 function freebsd_update_contigmem_mod() {
1191         if [ $(uname) = FreeBSD ]; then
1192                 kldunload contigmem.ko || true
1193                 if [ -n "$WITH_DPDK_DIR" ]; then
1194                         echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule"
1195                         cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/modules/
1196                         cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/kernel/
1197                         cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/modules/
1198                         cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/kernel/
1199                 else
1200                         cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/
1201                         cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/
1202                         cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/
1203                         cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/
1204                 fi
1205         fi
1206 }
1207
1208 function get_nvme_name_from_bdf() {
1209         blkname=()
1210
1211         nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true
1212         if [ -z "$nvme_devs" ]; then
1213                 return
1214         fi
1215         for dev in $nvme_devs; do
1216                 link_name=$(readlink /sys/block/$dev/device/device) || true
1217                 if [ -z "$link_name" ]; then
1218                         link_name=$(readlink /sys/block/$dev/device)
1219                 fi
1220                 bdf=$(basename "$link_name")
1221                 if [ "$bdf" = "$1" ]; then
1222                         blkname+=($dev)
1223                 fi
1224         done
1225
1226         printf '%s\n' "${blkname[@]}"
1227 }
1228
1229 function get_nvme_ctrlr_from_bdf() {
1230         bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme")
1231         if [[ -z "$bdf_sysfs_path" ]]; then
1232                 return
1233         fi
1234
1235         printf '%s\n' "$(basename $bdf_sysfs_path)"
1236 }
1237
1238 # Get BDF addresses of all NVMe drives currently attached to
1239 # uio-pci-generic or vfio-pci
1240 function get_nvme_bdfs() {
1241         xtrace_disable
1242         bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh --json))
1243         if [[ -z $bdfs ]]; then
1244                 echo "No devices to test on!"
1245                 exit 1
1246         fi
1247         echo "$bdfs"
1248         xtrace_restore
1249 }
1250
1251 # Same as function above, but just get the first disks BDF address
1252 function get_first_nvme_bdf() {
1253         head -1 <<< "$(get_nvme_bdfs)"
1254 }
1255
1256 function nvme_namespace_revert() {
1257         $rootdir/scripts/setup.sh
1258         sleep 1
1259         bdfs=$(get_nvme_bdfs)
1260
1261         $rootdir/scripts/setup.sh reset
1262         sleep 1
1263
1264         for bdf in $bdfs; do
1265                 nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf})
1266                 if [[ -z "$nvme_ctrlr" ]]; then
1267                         continue
1268                 fi
1269
1270                 # Check Optional Admin Command Support for Namespace Management
1271                 oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2)
1272                 oacs_ns_manage=$((oacs & 0x8))
1273
1274                 if [[ "$oacs_ns_manage" -ne 0 ]]; then
1275                         # This assumes every NVMe controller contains single namespace,
1276                         # encompassing Total NVM Capacity and formatted as 512 block size.
1277                         # 512 block size is needed for test/vhost/vhost_boot.sh to
1278                         # succesfully run.
1279
1280                         unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2)
1281                         if [[ "$unvmcap" -eq 0 ]]; then
1282                                 # All available space already used
1283                                 continue
1284                         fi
1285                         tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2)
1286                         blksize=512
1287
1288                         size=$((tnvmcap / blksize))
1289
1290                         nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true
1291                         nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true
1292                         nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize}
1293                         nvme attach-ns ${nvme_ctrlr} -n 1 -c 0
1294                         nvme reset ${nvme_ctrlr}
1295                         waitforblk "${nvme_ctrlr}n1"
1296                 fi
1297         done
1298 }
1299
1300 # Get BDFs based on device ID, such as 0x0a54
1301 function get_nvme_bdfs_by_id() {
1302         local bdfs=()
1303
1304         for bdf in $(get_nvme_bdfs); do
1305                 device=$(cat /sys/bus/pci/devices/$bdf/device) || true
1306                 if [[ "$device" == "$1" ]]; then
1307                         bdfs+=($bdf)
1308                 fi
1309         done
1310
1311         printf '%s\n' "${bdfs[@]}"
1312 }
1313
1314 function opal_revert_cleanup() {
1315         # The OPAL CI tests is only used for P4510 devices.
1316         mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54)
1317         if [[ -z ${bdfs[0]} ]]; then
1318                 return 0
1319         fi
1320
1321         $SPDK_BIN_DIR/spdk_tgt &
1322         spdk_tgt_pid=$!
1323         waitforlisten $spdk_tgt_pid
1324
1325         for bdf in "${bdfs[@]}"; do
1326                 $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme0" -t "pcie" -a ${bdf}
1327                 # Ignore if this fails.
1328                 $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b nvme0 -p test || true
1329         done
1330
1331         killprocess $spdk_tgt_pid
1332 }
1333
1334 # Define temp storage for all the tests. Look for 2GB at minimum
1335 set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}"
1336
1337 set -o errtrace
1338 shopt -s extdebug
1339 trap "trap - ERR; print_backtrace >&2" ERR
1340
1341 PS4=' \t        \$ '
1342 if $SPDK_AUTOTEST_X; then
1343         # explicitly enable xtraces, overriding any tracking information.
1344         unset XTRACE_DISABLED
1345         unset XTRACE_NESTING_LEVEL
1346         set -x
1347         xtrace_enable
1348 else
1349         xtrace_restore
1350 fi