1 source $rootdir/test
/nvmf
/common.sh
2 source $testdir/autotest.config
6 incoming_vm_ctrlr
=naa.VhostScsi0.
$incoming_vm
7 target_vm_ctrlr
=naa.VhostScsi0.
$target_vm
8 share_dir
=$TEST_DIR/share
9 spdk_repo_share_dir
=$TEST_DIR/share_spdk
10 job_file
=$testdir/migration-tc3.job
12 if [ -z "$MGMT_TARGET_IP" ]; then
13 error
"No IP address of target is given"
16 if [ -z "$MGMT_INITIATOR_IP" ]; then
17 error
"No IP address of initiator is given"
20 if [ -z "$RDMA_TARGET_IP" ]; then
21 error
"No IP address of targets RDMA capable NIC is given"
24 if [ -z "$RDMA_INITIATOR_IP" ]; then
25 error
"No IP address of initiators RDMA capable NIC is given"
30 local ssh_cmd
="ssh -i $SPDK_VHOST_SSH_KEY_FILE \
31 -o UserKnownHostsFile=/dev/null \
32 -o StrictHostKeyChecking=no -o ControlMaster=auto \
39 function wait_for_remote
()
43 while [[ ! -f $share_dir/DONE
]]; do
45 if (( timeout--
== 0 )); then
46 error
"timeout while waiting for FIO!"
54 function check_rdma_connection
()
56 local nic_name
=$
(ip
-4 -o addr show to
$RDMA_TARGET_IP up | cut
-d' ' -f2)
57 if [[ -z $nic_name ]]; then
58 error
"There is no NIC with IP address $RDMA_TARGET_IP configured"
61 if ! ls /sys
/class
/infiniband
/*/device
/net
/$nic_name &> /dev
/null
; then
62 error
"$nic_name with IP $RDMA_TARGET_IP is not a RDMA capable NIC"
67 function host1_cleanup_nvmf
()
69 notice
"Shutting down nvmf_tgt on local server"
70 if [[ ! -z "$1" ]]; then
71 pkill
--signal $1 -F $nvmf_dir/nvmf_tgt.pid
73 pkill
-F $nvmf_dir/nvmf_tgt.pid
75 rm -f $nvmf_dir/nvmf_tgt.pid
78 function host1_cleanup_vhost
()
80 trap 'host1_cleanup_nvmf SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
81 notice
"Shutting down VM $incoming_vm"
84 notice
"Removing bdev & controller from vhost on local server"
85 $rpc_0 delete_nvme_controller Nvme0
86 $rpc_0 remove_vhost_controller
$incoming_vm_ctrlr
88 notice
"Shutting down vhost app"
94 function host1_start_nvmf
()
96 nvmf_dir
="$TEST_DIR/nvmf_tgt"
97 rpc_nvmf
="$rootdir/scripts/rpc.py -s $nvmf_dir/nvmf_rpc.sock"
99 notice
"Starting nvmf_tgt instance on local server"
103 trap 'host1_cleanup_nvmf SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
104 $rootdir/app
/nvmf_tgt
/nvmf_tgt
-s 512 -m 0xF -r $nvmf_dir/nvmf_rpc.sock
--wait-for-rpc &
106 echo $nvmf_tgt_pid > $nvmf_dir/nvmf_tgt.pid
107 waitforlisten
"$nvmf_tgt_pid" "$nvmf_dir/nvmf_rpc.sock"
108 $rpc_nvmf start_subsystem_init
109 $rpc_nvmf nvmf_create_transport
-t RDMA
-u 8192
110 $rootdir/scripts
/gen_nvme.sh
--json |
$rpc_nvmf load_subsystem_config
112 $rpc_nvmf nvmf_subsystem_create nqn
.2018-02.io.spdk
:cnode1
-a -s SPDK01
113 $rpc_nvmf nvmf_subsystem_add_ns nqn
.2018-02.io.spdk
:cnode1 Nvme0n1
114 $rpc_nvmf nvmf_subsystem_add_listener nqn
.2018-02.io.spdk
:cnode1
-t rdma
-a $RDMA_TARGET_IP -s 4420
117 function host1_start_vhost
()
119 rpc_0
="$rootdir/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock"
121 notice
"Starting vhost0 instance on local server"
122 trap 'host1_cleanup_vhost; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
123 vhost_run
--vhost-num=0 --no-pci
124 $rpc_0 construct_nvme_bdev
-b Nvme0
-t rdma
-f ipv4
-a $RDMA_TARGET_IP -s 4420 -n "nqn.2018-02.io.spdk:cnode1"
125 $rpc_0 construct_vhost_scsi_controller
$incoming_vm_ctrlr
126 $rpc_0 add_vhost_scsi_lun
$incoming_vm_ctrlr 0 Nvme0n1
128 vm_setup
--os="$share_dir/migration.qcow2" --force=$incoming_vm --disk-type=spdk_vhost_scsi
--disks=VhostScsi0 \
129 --migrate-to=$target_vm --memory=512 --queue_num=1
131 # TODO: Fix loop calculating cpu_num in common.sh
132 # We need -smp 1 and -queue_num 1 for this test to work, and this loop
133 # in some cases calculates wrong cpu_num.
134 sed -i "s#smp 2#smp 1#g" $VM_BASE_DIR/$incoming_vm/run.sh
136 vm_wait_for_boot
300 $incoming_vm
139 function cleanup_share
()
142 notice
"Cleaning up share directory on remote and local server"
143 ssh_remote
$MGMT_INITIATOR_IP "umount $VM_BASE_DIR"
144 ssh_remote
$MGMT_INITIATOR_IP "umount $share_dir; rm -f $share_dir/* rm -rf $spdk_repo_share_dir"
145 rm -f $share_dir/migration.qcow2
146 rm -f $share_dir/spdk.
tar.gz
150 function host_1_create_share
()
152 notice
"Creating share directory on local server to re-use on remote"
154 mkdir
-p $VM_BASE_DIR # This dir would've been created later but we need it now
155 rm -rf $share_dir/spdk.
tar.gz
$share_dir/spdk || true
156 cp $os_image $share_dir/migration.qcow2
157 tar --exclude="*.o"--exclude="*.d" --exclude="*.git" -C $rootdir -zcf $share_dir/spdk.
tar.gz .
160 function host_2_create_share
()
162 # Copy & compile the sources for later use on remote server.
163 ssh_remote
$MGMT_INITIATOR_IP "uname -a"
164 ssh_remote
$MGMT_INITIATOR_IP "mkdir -p $share_dir"
165 ssh_remote
$MGMT_INITIATOR_IP "mkdir -p $spdk_repo_share_dir"
166 ssh_remote
$MGMT_INITIATOR_IP "mkdir -p $VM_BASE_DIR"
167 ssh_remote
$MGMT_INITIATOR_IP "sshfs -o\
168 ssh_command=\"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto\
169 -i $SPDK_VHOST_SSH_KEY_FILE\" root@$MGMT_TARGET_IP:$VM_BASE_DIR $VM_BASE_DIR"
170 ssh_remote
$MGMT_INITIATOR_IP "sshfs -o\
171 ssh_command=\"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto\
172 -i $SPDK_VHOST_SSH_KEY_FILE\" root@$MGMT_TARGET_IP:$share_dir $share_dir"
173 ssh_remote
$MGMT_INITIATOR_IP "mkdir -p $spdk_repo_share_dir/spdk"
174 ssh_remote
$MGMT_INITIATOR_IP "tar -zxf $share_dir/spdk.tar.gz -C $spdk_repo_share_dir/spdk --strip-components=1"
175 ssh_remote
$MGMT_INITIATOR_IP "cd $spdk_repo_share_dir/spdk; make clean; ./configure --with-rdma --enable-debug; make -j40"
178 function host_2_start_vhost
()
180 ssh_remote
$MGMT_INITIATOR_IP "nohup $spdk_repo_share_dir/spdk/test/vhost/migration/migration.sh\
181 --test-cases=3b --os=$share_dir/migration.qcow2\
182 --rdma-tgt-ip=$RDMA_TARGET_IP &>$share_dir/output.log &"
183 notice
"Waiting for remote to be done with vhost & VM setup..."
187 function setup_share
()
189 trap 'cleanup_share; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
194 function migration_tc3
()
196 check_rdma_connection
203 notice
"Starting fio on local VM"
204 vm_check_scsi_location
$incoming_vm
206 run_fio
$fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
209 if ! is_fio_running
$incoming_vm; then
210 vh_ssh
$incoming_vm "cat /root/$(basename ${job_file}).out"
211 error
"Fio not running on local VM before starting migration!"
214 vm_migrate
$incoming_vm $RDMA_INITIATOR_IP
217 # Verify migration on remote host and clean up vhost
218 ssh_remote
$MGMT_INITIATOR_IP "pkill -CONT -F $TEST_DIR/tc3b.pid"
219 notice
"Waiting for remote to finish FIO on VM and clean up..."
222 # Clean up local stuff