ceph/src/spdk/test/json_config/json_config.sh

   1 #!/usr/bin/env bash
   2
   3 rootdir=$(readlink -f $(dirname $0)/../..)
   4 source "$rootdir/test/common/autotest_common.sh"
   5 source "$rootdir/test/nvmf/common.sh"
   6
   7 if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then
   8         source "$rootdir/test/iscsi_tgt/common.sh"
   9 fi
  10
  11 if [[ $SPDK_TEST_VHOST -ne 1 && $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
  12         SPDK_TEST_VHOST=1
  13         echo "WARNING: Virtio initiator JSON_config test requires vhost target."
  14         echo "         Setting SPDK_TEST_VHOST=1 for duration of current script."
  15 fi
  16
  17 if ((SPDK_TEST_BLOCKDEV + \
  18         SPDK_TEST_ISCSI + \
  19         SPDK_TEST_NVMF + \
  20         SPDK_TEST_VHOST + \
  21         SPDK_TEST_VHOST_INIT + \
  22         SPDK_TEST_PMDK + \
  23         SPDK_TEST_RBD == 0)); then
  24         echo "WARNING: No tests are enabled so not running JSON configuration tests"
  25         exit 0
  26 fi
  27
  28 declare -A app_pid=([target]="" [initiator]="")
  29 declare -A app_socket=([target]='/var/tmp/spdk_tgt.sock' [initiator]='/var/tmp/spdk_initiator.sock')
  30 declare -A app_params=([target]='-m 0x1 -s 1024' [initiator]='-m 0x2 -g -u -s 1024')
  31 declare -A configs_path=([target]="$rootdir/spdk_tgt_config.json" [initiator]="$rootdir/spdk_initiator_config.json")
  32
  33 function tgt_rpc() {
  34         $rootdir/scripts/rpc.py -s "${app_socket[target]}" "$@"
  35 }
  36
  37 function initiator_rpc() {
  38         $rootdir/scripts/rpc.py -s "${app_socket[initiator]}" "$@"
  39 }
  40
  41 RE_UUID="[[:alnum:]-]+"
  42 last_event_id=0
  43
  44 function tgt_check_notification_types() {
  45         timing_enter "${FUNCNAME[0]}"
  46
  47         local ret=0
  48         local enabled_types=("bdev_register" "bdev_unregister")
  49
  50         local get_types=($(tgt_rpc notify_get_types | jq -r '.[]'))
  51         if [[ ${enabled_types[*]} != "${get_types[*]}" ]]; then
  52                 echo "ERROR: expected types: ${enabled_types[*]}, but got: ${get_types[*]}"
  53                 ret=1
  54         fi
  55
  56         timing_exit "${FUNCNAME[0]}"
  57         return $ret
  58 }
  59
  60 function tgt_check_notifications() {
  61         local event_line event ev_type ev_ctx
  62         local rc=""
  63
  64         while read -r event_line; do
  65                 # remove ID
  66                 event="${event_line%:*}"
  67
  68                 ev_type=${event%:*}
  69                 ev_ctx=${event#*:}
  70
  71                 ex_ev_type=${1%%:*}
  72                 ex_ev_ctx=${1#*:}
  73
  74                 last_event_id=${event_line##*:}
  75
  76                 # set rc=false in case of failure so all errors can be printed
  77                 if (($# == 0)); then
  78                         echo "ERROR: got extra event: $event_line"
  79                         rc=false
  80                         continue
  81                 elif ! echo "$ev_type" | grep -E -q "^${ex_ev_type}\$" || ! echo "$ev_ctx" | grep -E -q "^${ex_ev_ctx}\$"; then
  82                         echo "ERROR: expected event '$1' but got '$event' (whole event line: $event_line)"
  83                         rc=false
  84                 fi
  85
  86                 shift
  87         done < <(tgt_rpc notify_get_notifications -i ${last_event_id} | jq -r '.[] | "\(.type):\(.ctx):\(.id)"')
  88
  89         $rc
  90
  91         if (($# != 0)); then
  92                 echo "ERROR: missing events:"
  93                 echo "$@"
  94                 return 1
  95         fi
  96 }
  97
  98 # $1 - target / initiator
  99 # $2..$n app parameters
 100 function json_config_test_start_app() {
 101         local app=$1
 102         shift
 103
 104         [[ -n "${#app_socket[$app]}" ]] # Check app type
 105         [[ -z "${app_pid[$app]}" ]]     # Assert if app is not running
 106
 107         local app_extra_params=""
 108         if [[ $SPDK_TEST_VHOST -eq 1 || $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
 109                 # If PWD is nfs/sshfs we can't create UNIX sockets there. Always use safe location instead.
 110                 app_extra_params='-S /var/tmp'
 111         fi
 112
 113         $SPDK_BIN_DIR/spdk_tgt ${app_params[$app]} ${app_extra_params} -r ${app_socket[$app]} "$@" &
 114         app_pid[$app]=$!
 115
 116         echo "Waiting for $app to run..."
 117         waitforlisten ${app_pid[$app]} ${app_socket[$app]}
 118         echo ""
 119 }
 120
 121 # $1 - target / initiator
 122 function json_config_test_shutdown_app() {
 123         local app=$1
 124
 125         # Check app type && assert app was started
 126         [[ -n "${#app_socket[$app]}" ]]
 127         [[ -n "${app_pid[$app]}" ]]
 128
 129         # spdk_kill_instance RPC will trigger ASAN
 130         kill -SIGINT ${app_pid[$app]}
 131
 132         for ((i = 0; i < 30; i++)); do
 133                 if ! kill -0 ${app_pid[$app]} 2> /dev/null; then
 134                         app_pid[$app]=
 135                         break
 136                 fi
 137                 sleep 0.5
 138         done
 139
 140         if [[ -n "${app_pid[$app]}" ]]; then
 141                 echo "SPDK $app shutdown timeout"
 142                 return 1
 143         fi
 144
 145         echo "SPDK $app shutdown done"
 146 }
 147
 148 function create_bdev_subsystem_config() {
 149         timing_enter "${FUNCNAME[0]}"
 150
 151         local expected_notifications=()
 152
 153         if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then
 154                 local lvol_store_base_bdev=Nvme0n1
 155                 if ! tgt_rpc get_bdevs --name ${lvol_store_base_bdev} > /dev/null; then
 156                         if [[ $(uname -s) = Linux ]]; then
 157                                 lvol_store_base_bdev=aio_disk
 158                                 echo "WARNING: No NVMe drive found. Using '$lvol_store_base_bdev' instead."
 159                         else
 160                                 echo "ERROR: No NVMe drive found and bdev_aio is not supported on $(uname -s)."
 161                                 timing_exit "${FUNCNAME[0]}"
 162                                 return 1
 163                         fi
 164                 fi
 165
 166                 tgt_rpc bdev_split_create $lvol_store_base_bdev 2
 167                 tgt_rpc bdev_split_create Malloc0 3
 168                 tgt_rpc bdev_malloc_create 8 4096 --name Malloc3
 169                 tgt_rpc bdev_passthru_create -b Malloc3 -p PTBdevFromMalloc3
 170
 171                 tgt_rpc bdev_null_create Null0 32 512
 172
 173                 tgt_rpc bdev_malloc_create 32 512 --name Malloc0
 174                 tgt_rpc bdev_malloc_create 16 4096 --name Malloc1
 175
 176                 expected_notifications+=(
 177                         bdev_register:${lvol_store_base_bdev}
 178                         bdev_register:${lvol_store_base_bdev}p0
 179                         bdev_register:${lvol_store_base_bdev}p1
 180                         bdev_register:Malloc3
 181                         bdev_register:PTBdevFromMalloc3
 182                         bdev_register:Null0
 183                         bdev_register:Malloc0p0
 184                         bdev_register:Malloc0p1
 185                         bdev_register:Malloc0p2
 186                         bdev_register:Malloc0
 187                         bdev_register:Malloc1
 188                 )
 189
 190                 if [[ $(uname -s) = Linux ]]; then
 191                         # This AIO bdev must be large enough to be used as LVOL store
 192                         dd if=/dev/zero of="$SPDK_TEST_STORAGE/sample_aio" bs=1024 count=102400
 193                         tgt_rpc bdev_aio_create "$SPDK_TEST_STORAGE/sample_aio" aio_disk 1024
 194                         expected_notifications+=(bdev_register:aio_disk)
 195                 fi
 196
 197                 # For LVOLs use split to check for proper order of initialization.
 198                 # If LVOLs cofniguration will be reordered (eg moved before splits or AIO/NVMe)
 199                 # it should fail loading JSON config from file.
 200                 tgt_rpc bdev_lvol_create_lvstore -c 1048576 ${lvol_store_base_bdev}p0 lvs_test
 201                 tgt_rpc bdev_lvol_create -l lvs_test lvol0 32
 202                 tgt_rpc bdev_lvol_create -l lvs_test -t lvol1 32
 203                 tgt_rpc bdev_lvol_snapshot lvs_test/lvol0 snapshot0
 204                 tgt_rpc bdev_lvol_clone lvs_test/snapshot0 clone0
 205
 206                 expected_notifications+=(
 207                         "bdev_register:$RE_UUID"
 208                         "bdev_register:$RE_UUID"
 209                         "bdev_register:$RE_UUID"
 210                         "bdev_register:$RE_UUID"
 211                 )
 212         fi
 213
 214         if [[ $SPDK_TEST_CRYPTO -eq 1 ]]; then
 215                 tgt_rpc bdev_malloc_create 8 1024 --name MallocForCryptoBdev
 216                 if [[ $(lspci -d:37c8 | wc -l) -eq 0 ]]; then
 217                         local crypto_dirver=crypto_aesni_mb
 218                 else
 219                         local crypto_dirver=crypto_qat
 220                 fi
 221
 222                 tgt_rpc bdev_crypto_create MallocForCryptoBdev CryptoMallocBdev $crypto_dirver 0123456789123456
 223                 expected_notifications+=(
 224                         bdev_register:MallocForCryptoBdev
 225                         bdev_register:CryptoMallocBdev
 226                 )
 227         fi
 228
 229         if [[ $SPDK_TEST_PMDK -eq 1 ]]; then
 230                 pmem_pool_file=$(mktemp /tmp/pool_file1.XXXXX)
 231                 rm -f $pmem_pool_file
 232                 tgt_rpc create_pmem_pool $pmem_pool_file 128 4096
 233                 tgt_rpc bdev_pmem_create -n pmem1 $pmem_pool_file
 234                 expected_notifications+=(bdev_register:pmem1)
 235         fi
 236
 237         if [[ $SPDK_TEST_RBD -eq 1 ]]; then
 238                 rbd_setup 127.0.0.1
 239                 tgt_rpc bdev_rbd_create $RBD_POOL $RBD_NAME 4096
 240                 expected_notifications+=(bdev_register:Ceph0)
 241         fi
 242
 243         tgt_check_notifications "${expected_notifications[@]}"
 244
 245         timing_exit "${FUNCNAME[0]}"
 246 }
 247
 248 function cleanup_bdev_subsystem_config() {
 249         timing_enter "${FUNCNAME[0]}"
 250
 251         if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then
 252                 tgt_rpc bdev_lvol_delete lvs_test/clone0
 253                 tgt_rpc bdev_lvol_delete lvs_test/lvol0
 254                 tgt_rpc bdev_lvol_delete lvs_test/snapshot0
 255                 tgt_rpc bdev_lvol_delete_lvstore -l lvs_test
 256         fi
 257
 258         if [[ $(uname -s) = Linux ]]; then
 259                 rm -f "$SPDK_TEST_STORAGE/sample_aio"
 260         fi
 261
 262         if [[ $SPDK_TEST_PMDK -eq 1 && -n "$pmem_pool_file" && -f "$pmem_pool_file" ]]; then
 263                 tgt_rpc bdev_pmem_delete pmem1
 264                 tgt_rpc bdev_pmem_delete_pool $pmem_pool_file
 265                 rm -f $pmem_pool_file
 266         fi
 267
 268         if [[ $SPDK_TEST_RBD -eq 1 ]]; then
 269                 rbd_cleanup
 270         fi
 271
 272         timing_exit "${FUNCNAME[0]}"
 273 }
 274
 275 function create_vhost_subsystem_config() {
 276         timing_enter "${FUNCNAME[0]}"
 277
 278         tgt_rpc bdev_malloc_create 64 1024 --name MallocForVhost0
 279         tgt_rpc bdev_split_create MallocForVhost0 8
 280
 281         tgt_rpc vhost_create_scsi_controller VhostScsiCtrlr0
 282         tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 0 MallocForVhost0p3
 283         tgt_rpc vhost_scsi_controller_add_target VhostScsiCtrlr0 -1 MallocForVhost0p4
 284         tgt_rpc vhost_controller_set_coalescing VhostScsiCtrlr0 1 100
 285
 286         tgt_rpc vhost_create_blk_controller VhostBlkCtrlr0 MallocForVhost0p5
 287
 288         # FIXME: enable after vhost-nvme is properly implemented against the latest rte_vhost (DPDK 19.05+)
 289         #       tgt_rpc vhost_create_nvme_controller   VhostNvmeCtrlr0 16
 290         #       tgt_rpc vhost_nvme_controller_add_ns                 VhostNvmeCtrlr0 MallocForVhost0p6
 291
 292         timing_exit "${FUNCNAME[0]}"
 293 }
 294
 295 function create_iscsi_subsystem_config() {
 296         timing_enter "${FUNCNAME[0]}"
 297         tgt_rpc bdev_malloc_create 64 1024 --name MallocForIscsi0
 298         tgt_rpc iscsi_create_portal_group $PORTAL_TAG 127.0.0.1:$ISCSI_PORT
 299         tgt_rpc iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK
 300         tgt_rpc iscsi_create_target_node Target3 Target3_alias 'MallocForIscsi0:0' $PORTAL_TAG:$INITIATOR_TAG 64 -d
 301         timing_exit "${FUNCNAME[0]}"
 302 }
 303
 304 function create_nvmf_subsystem_config() {
 305         timing_enter "${FUNCNAME[0]}"
 306
 307         RDMA_IP_LIST=$(get_available_rdma_ips)
 308         NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1)
 309         if [[ -z $NVMF_FIRST_TARGET_IP ]]; then
 310                 echo "Error: no NIC for nvmf test"
 311                 return 1
 312         fi
 313
 314         tgt_rpc bdev_malloc_create 8 512 --name MallocForNvmf0
 315         tgt_rpc bdev_malloc_create 4 1024 --name MallocForNvmf1
 316
 317         tgt_rpc nvmf_create_transport -t RDMA -u 8192 -c 0
 318         tgt_rpc nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001
 319         tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf0
 320         tgt_rpc nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 MallocForNvmf1
 321         tgt_rpc nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT"
 322
 323         timing_exit "${FUNCNAME[0]}"
 324 }
 325
 326 function create_virtio_initiator_config() {
 327         timing_enter "${FUNCNAME[0]}"
 328         initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostScsiCtrlr0 -d scsi VirtioScsiCtrlr0
 329         initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostBlkCtrlr0 -d blk VirtioBlk0
 330         # TODO: initiator_rpc bdev_virtio_attach_controller -t user -a /var/tmp/VhostNvmeCtrlr0 -d nvme VirtioNvme0
 331         timing_exit "${FUNCNAME[0]}"
 332 }
 333
 334 function json_config_test_init() {
 335         timing_enter "${FUNCNAME[0]}"
 336         timing_enter json_config_setup_target
 337
 338         json_config_test_start_app target --wait-for-rpc
 339
 340         #TODO: global subsystem params
 341
 342         # Load nvme configuration. The load_config will issue framework_start_init automatically
 343         (
 344                 echo '{"subsystems": ['
 345                 $rootdir/scripts/gen_nvme.sh --json | jq -r "del(.config[] | select(.params.name!=\"Nvme0\"))"
 346                 echo ']}'
 347         ) | tgt_rpc load_config
 348
 349         tgt_check_notification_types
 350
 351         if [[ $SPDK_TEST_BLOCKDEV -eq 1 ]]; then
 352                 create_bdev_subsystem_config
 353         fi
 354
 355         if [[ $SPDK_TEST_VHOST -eq 1 ]]; then
 356                 create_vhost_subsystem_config
 357         fi
 358
 359         if [[ $SPDK_TEST_ISCSI -eq 1 ]]; then
 360                 create_iscsi_subsystem_config
 361         fi
 362
 363         if [[ $SPDK_TEST_NVMF -eq 1 ]]; then
 364                 create_nvmf_subsystem_config
 365         fi
 366         timing_exit json_config_setup_target
 367
 368         if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
 369                 json_config_test_start_app initiator
 370                 create_virtio_initiator_config
 371         fi
 372
 373         tgt_rpc bdev_malloc_create 8 512 --name MallocBdevForConfigChangeCheck
 374
 375         timing_exit "${FUNCNAME[0]}"
 376 }
 377
 378 function json_config_test_fini() {
 379         timing_enter "${FUNCNAME[0]}"
 380         local ret=0
 381
 382         if [[ -n "${app_pid[initiator]}" ]]; then
 383                 killprocess ${app_pid[initiator]}
 384         fi
 385
 386         if [[ -n "${app_pid[target]}" ]]; then
 387
 388                 # Remove any artifacts we created (files, lvol etc)
 389                 cleanup_bdev_subsystem_config
 390
 391                 # SPDK_TEST_NVMF: Should we clear something?
 392                 killprocess ${app_pid[target]}
 393         fi
 394
 395         rm -f "${configs_path[@]}"
 396         timing_exit "${FUNCNAME[0]}"
 397         return $ret
 398 }
 399
 400 function json_config_clear() {
 401         [[ -n "${#app_socket[$1]}" ]] # Check app type
 402         $rootdir/test/json_config/clear_config.py -s ${app_socket[$1]} clear_config
 403
 404         # Check if config is clean.
 405         # Global params can't be cleared so need to filter them out.
 406         local config_filter="$rootdir/test/json_config/config_filter.py"
 407
 408         # RPC's used to cleanup configuration (e.g. to delete split and nvme bdevs)
 409         # complete immediately and they don't wait for the unregister callback.
 410         # It causes that configuration may not be fully cleaned at this moment and
 411         # we should to wait a while. (See github issue #789)
 412         count=100
 413         while [ $count -gt 0 ]; do
 414                 $rootdir/scripts/rpc.py -s "${app_socket[$1]}" save_config | $config_filter -method delete_global_parameters | $config_filter -method check_empty && break
 415                 count=$((count - 1))
 416                 sleep 0.1
 417         done
 418
 419         if [ $count -eq 0 ]; then
 420                 return 1
 421         fi
 422 }
 423
 424 on_error_exit() {
 425         set -x
 426         set +e
 427         print_backtrace
 428         trap - ERR
 429         echo "Error on $1 - $2"
 430         json_config_test_fini
 431         exit 1
 432 }
 433
 434 trap 'on_error_exit "${FUNCNAME}" "${LINENO}"' ERR
 435 echo "INFO: JSON configuration test init"
 436 json_config_test_init
 437
 438 tgt_rpc save_config > ${configs_path[target]}
 439
 440 echo "INFO: shutting down applications..."
 441 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
 442         initiator_rpc save_config > ${configs_path[initiator]}
 443         json_config_clear initiator
 444         json_config_test_shutdown_app initiator
 445 fi
 446
 447 json_config_clear target
 448 json_config_test_shutdown_app target
 449
 450 echo "INFO: relaunching applications..."
 451 json_config_test_start_app target --json ${configs_path[target]}
 452 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
 453         json_config_test_start_app initiator --json ${configs_path[initiator]}
 454 fi
 455
 456 echo "INFO: Checking if target configuration is the same..."
 457 $rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}"
 458 if [[ $SPDK_TEST_VHOST_INIT -eq 1 ]]; then
 459         echo "INFO: Checking if virtio initiator configuration is the same..."
 460         $rootdir/test/json_config/json_diff.sh <(initiator_rpc save_config) "${configs_path[initiator]}"
 461 fi
 462
 463 echo "INFO: changing configuration and checking if this can be detected..."
 464 # Self test to check if configuration diff can be detected.
 465 tgt_rpc bdev_malloc_delete MallocBdevForConfigChangeCheck
 466 if $rootdir/test/json_config/json_diff.sh <(tgt_rpc save_config) "${configs_path[target]}" > /dev/null; then
 467         echo "ERROR: intentional configuration difference not detected!"
 468         false
 469 else
 470         echo "INFO: configuration change detected."
 471 fi
 472
 473 json_config_test_fini
 474
 475 echo "INFO: Success"