ceph/src/spdk/test/vhost/fiotest/autotest.sh

   1 #!/usr/bin/env bash
   2 set -e
   3 BASE_DIR=$(readlink -f $(dirname $0))
   4 [[ -z "$TEST_DIR" ]] && TEST_DIR="$(cd $BASE_DIR/../../../../ && pwd)"
   5
   6 dry_run=false
   7 no_shutdown=false
   8 fio_bin="fio"
   9 fio_jobs="$BASE_DIR/fio_jobs/"
  10 test_type=spdk_vhost
  11 reuse_vms=false
  12 force_build=false
  13 vms=()
  14 used_vms=""
  15 disk_split=""
  16 x=""
  17
  18 max_sectors_kb=4
  19
  20 function usage()
  21 {
  22         [[ ! -z $2 ]] && ( echo "$2"; echo ""; )
  23         echo "Shortcut script for doing automated test"
  24         echo "Usage: $(basename $1) [OPTIONS]"
  25         echo
  26         echo "-h, --help                print help and exit"
  27         echo "    --test-type=TYPE      Perform specified test:"
  28         echo "                          virtio - test host virtio-scsi-pci using file as disk image"
  29         echo "                          kernel_vhost - use kernel driver vhost-scsi"
  30         echo "                          spdk_vhost - use spdk vhost"
  31         echo "-x                        set -x for script debug"
  32         echo "    --fio-bin=FIO         Use specific fio binary (will be uploaded to VM)"
  33         echo "    --qemu-src=QEMU_DIR   Location of the QEMU sources"
  34         echo "    --dpdk-src=DPDK_DIR   Location of the DPDK sources"
  35         echo "    --fio-jobs=           Fio configs to use for tests. Can point to a directory or"
  36         echo "                          can point to a directory with regex mask, example: ./dir/*.job"
  37         echo "                          All VMs will run the same fio job when FIO executes."
  38         echo "                          (no unique jobs for specific VMs)"
  39         echo "    --work-dir=WORK_DIR   Where to find build file. Must exist. [default: $TEST_DIR]"
  40         echo "    --dry-run             Don't perform any tests, run only and wait for enter to terminate"
  41         echo "    --no-shutdown         Don't shutdown at the end but leave envirionment working"
  42         echo "    --force-build         Force SPDK rebuild with the specified DPDK path."
  43         echo "    --vm=NUM[,OS][,DISKS] VM configuration. This parameter might be used more than once:"
  44         echo "                          NUM - VM number (mandatory)"
  45         echo "                          OS - VM os disk path (optional)"
  46         echo "                          DISKS - VM os test disks/devices path (virtio - optional, kernel_vhost - mandatory)"
  47         echo "    --disk-split          By default all test types execute fio jobs on all disks which are available on guest"
  48         echo "                          system. Use this option if only some of the disks should be used for testing."
  49         echo "                          Example: --disk-split=4,1-3 will result in VM 1 using it's first disk (ex. /dev/sda)"
  50         echo "                          and VM 2 using it's disks 1-3 (ex. /dev/sdb, /dev/sdc, /dev/sdd)"
  51         echo "    --max-sectors=NUM     Set max_sectors_kb for test disk to NUM (default: $max_sectors_kb)"
  52         exit 0
  53 }
  54
  55 #default raw file is NVMe drive
  56
  57 while getopts 'xh-:' optchar; do
  58         case "$optchar" in
  59                 -)
  60                 case "$OPTARG" in
  61                         help) usage $0 ;;
  62                         work-dir=*) TEST_DIR="${OPTARG#*=}" ;;
  63                         fio-bin=*) fio_bin="--fio-bin=${OPTARG#*=}" ;;
  64                         qemu-src=*) QEMU_SRC_DIR="${OPTARG#*=}" ;;
  65                         dpdk-src=*) DPDK_SRC_DIR="${OPTARG#*=}" ;;
  66                         fio-jobs=*) fio_jobs="${OPTARG#*=}" ;;
  67                         dry-run) dry_run=true ;;
  68                         no-shutdown) no_shutdown=true ;;
  69                         test-type=*) test_type="${OPTARG#*=}" ;;
  70                         force-build) force_build=true ;;
  71                         vm=*) vms+=("${OPTARG#*=}") ;;
  72                         disk-split=*) disk_split="${OPTARG#*=}" ;;
  73                         max-sectors=*) max_sectors_kb="${OPTARG#*=}" ;;
  74                         *) usage $0 "Invalid argument '$OPTARG'" ;;
  75                 esac
  76                 ;;
  77         h) usage $0 ;;
  78         x) set -x
  79                 x="-x" ;;
  80         *) usage $0 "Invalid argument '$OPTARG'"
  81         esac
  82 done
  83 shift $(( OPTIND - 1 ))
  84
  85 if [[ -d "$fio_jobs" ]]; then
  86         fio_jobs="$fio_jobs/*.job"
  87 fi
  88
  89 . $BASE_DIR/common.sh
  90
  91 trap 'error_exit "${FUNCNAME}" "${LINENO}"' ERR
  92
  93 echo "==============="
  94 echo "INFO: checking qemu"
  95
  96 if [[ ! -x $INSTALL_DIR/bin/qemu-system-x86_64 ]]; then
  97         echo "INFO: can't find $INSTALL_DIR/bin/qemu-system-x86_64 - building and installing"
  98
  99         if [[ ! -d $QEMU_SRC_DIR ]]; then
 100                 echo "ERROR: Cannot find qemu source in $QEMU_SRC_DIR"
 101                 exit 1
 102         else
 103                 echo "INFO: qemu source exists $QEMU_SRC_DIR - building"
 104                 qemu_build_and_install
 105         fi
 106 fi
 107
 108 echo "==============="
 109 echo ""
 110 echo "INFO: checking spdk"
 111 echo ""
 112
 113 if [[ ! -x $SPDK_BUILD_DIR/app/vhost/vhost ]] || $force_build ; then
 114         echo "INFO: $SPDK_BUILD_DIR/app/vhost/vhost - building and installing"
 115         spdk_build_and_install
 116 fi
 117
 118 vm_kill_all
 119
 120 if [[ $test_type == "spdk_vhost" ]]; then
 121         echo "==============="
 122         echo ""
 123         echo "INFO: running SPDK"
 124         echo ""
 125         $BASE_DIR/run_vhost.sh $x --work-dir=$TEST_DIR
 126         echo
 127 fi
 128
 129 echo "==============="
 130 echo ""
 131 echo "Setting up VM"
 132 echo ""
 133
 134 rpc_py="python $SPDK_BUILD_DIR/scripts/rpc.py "
 135 rpc_py+="-s 127.0.0.1 "
 136
 137 for vm_conf in ${vms[@]}; do
 138         IFS=',' read -ra conf <<< "$vm_conf"
 139         setup_cmd="$BASE_DIR/vm_setup.sh $x --work-dir=$TEST_DIR --test-type=$test_type"
 140         if [[ x"${conf[0]}" == x"" ]] || ! assert_number ${conf[0]}; then
 141                 echo "ERROR: invalid VM configuration syntax $vm_conf"
 142                 exit 1;
 143         fi
 144
 145         # Sanity check if VM is not defined twice
 146         for vm_num in $used_vms; do
 147                 if [[ $vm_num -eq ${conf[0]} ]]; then
 148                         echo "ERROR: VM$vm_num defined more than twice ( $(printf "'%s' " "${vms[@]}"))!"
 149                         exit 1
 150                 fi
 151         done
 152
 153         setup_cmd+=" -f ${conf[0]}"
 154         used_vms+=" ${conf[0]}"
 155         [[ x"${conf[1]}" != x"" ]] && setup_cmd+=" --os=${conf[1]}"
 156         [[ x"${conf[2]}" != x"" ]] && setup_cmd+=" --disk=${conf[2]}"
 157
 158         if [[ $test_type == "spdk_vhost" ]]; then
 159                 echo "INFO: Adding device via RPC ..."
 160                 echo ""
 161
 162                 eval $(grep "^vhost_reactor_mask=" $BASE_DIR/autotest.config)
 163                 while IFS=':' read -ra disks; do
 164                         for disk in "${disks[@]}"; do
 165                                 $rpc_py construct_vhost_scsi_controller naa.$disk.${conf[0]} \
 166                                 --cpumask $vhost_reactor_mask
 167                                 $rpc_py add_vhost_scsi_lun naa.$disk.${conf[0]} 0 $disk
 168                         done
 169                 done <<< "${conf[2]}"
 170                 unset IFS;
 171                 $rpc_py get_vhost_scsi_controllers
 172         fi
 173         $setup_cmd
 174 done
 175
 176 # Run everything
 177 $BASE_DIR/vm_run.sh $x --work-dir=$TEST_DIR $used_vms
 178 vm_wait_for_boot 600 $used_vms
 179
 180 echo "==============="
 181 echo ""
 182 echo "INFO: Testing..."
 183
 184 echo "INFO: Running fio jobs ..."
 185 run_fio="python $BASE_DIR/run_fio.py "
 186 run_fio+="$fio_bin "
 187 run_fio+="--job-file="
 188 for job in $fio_jobs; do
 189         run_fio+="$job,"
 190 done
 191 run_fio="${run_fio::-1}"
 192 run_fio+=" "
 193 run_fio+="--out=$TEST_DIR "
 194
 195 if [[ ! $disk_split == '' ]]; then
 196         run_fio+="--split-disks=$disk_split "
 197 fi
 198
 199 # Check if all VM have disk in tha same location
 200 DISK=""
 201
 202 for vm_num in $used_vms; do
 203         vm_dir=$VM_BASE_DIR/$vm_num
 204         host_name="VM-$vm_num-$(cat $BASE_DIR/autotest.config|grep qemu_mask|awk -F'=' '{print $2}'|sed "$(($vm_num+1))q;d")"
 205         echo "INFO: Setting up hostname: $host_name"
 206         vm_ssh $vm_num "hostname $host_name"
 207         vm_start_fio_server $fio_bin $readonly $vm_num
 208         vm_check_scsi_location $vm_num
 209
 210         SCSI_DISK="${SCSI_DISK::-1}"
 211         for DISK in $SCSI_DISK; do
 212                 echo "INFO: VM$vm_num Setting max_sectors_kb=$max_sectors_kb on disk $DISK"
 213                 echo ""
 214                 vm_ssh $vm_num "echo $max_sectors_kb > /sys/block/$DISK/queue/max_sectors_kb"
 215         done
 216
 217         vm_reset_scsi_devices $vm_num $SCSI_DISK
 218
 219         run_fio+="127.0.0.1:$(cat $vm_dir/fio_socket):"
 220         for disk in $SCSI_DISK; do
 221                 run_fio+="/dev/$disk:"
 222         done
 223         run_fio="${run_fio::-1}"
 224         run_fio+=","
 225 done
 226
 227 run_fio="${run_fio%,}"
 228 run_fio+=" "
 229 run_fio="${run_fio::-1}"
 230
 231 echo -e "$run_fio"
 232
 233 if $dry_run; then
 234         read -p "Enter to kill evething" xx
 235         sleep 3
 236         at_app_exit
 237         exit 0
 238 fi
 239
 240 $run_fio
 241
 242 for vm_num in $used_vms; do
 243         vm_reset_scsi_devices $vm_num $SCSI_DISK
 244 done
 245
 246 if ! $no_shutdown; then
 247         echo "==============="
 248         echo "INFO: Testing done -> shutting down"
 249         at_app_exit
 250         echo "==============="
 251 else
 252         echo "==============="
 253         echo
 254         echo "INFO: Leaving environment working!"
 255         echo ""
 256         echo "==============="
 257 fi