test/run_qemu_migrate_tests.pl

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use JSON;
   7 use Test::More;
   8 use Test::MockModule;
   9
  10 use PVE::JSONSchema;
  11 use PVE::Tools qw(file_set_contents file_get_contents run_command);
  12
  13 my $QM_LIB_PATH = '..';
  14 my $MIGRATE_LIB_PATH = '..';
  15 my $RUN_DIR_PATH = './MigrationTest/run/';
  16
  17 # test configuration shared by all tests
  18
  19 my $replication_config = {
  20     'ids' => {
  21         '105-0' => {
  22             'guest' => '105',
  23             'id' => '105-0',
  24             'jobnum' => '0',
  25             'source' => 'pve0',
  26             'target' => 'pve2',
  27             'type' => 'local'
  28         },
  29     },
  30     'order' => {
  31         '105-0' => 1,
  32     }
  33 };
  34
  35 my $storage_config = {
  36     ids => {
  37         local => {
  38             content => {
  39                 images => 1,
  40             },
  41             path => "/var/lib/vz",
  42             type => "dir",
  43             shared => 0,
  44         },
  45         "local-lvm" => {
  46             content => {
  47                 images => 1,
  48             },
  49             nodes => {
  50                 pve0 => 1,
  51                 pve1 => 1,
  52             },
  53             type => "lvmthin",
  54             thinpool => "data",
  55             vgname => "pve",
  56         },
  57         "local-zfs" => {
  58             content => {
  59                 images => 1,
  60                 rootdir => 1,
  61             },
  62             pool => "rpool/data",
  63             sparse => 1,
  64             type => "zfspool",
  65         },
  66         "rbd-store" => {
  67             monhost => "127.0.0.42,127.0.0.21,::1",
  68             content => {
  69                 images => 1,
  70             },
  71             type => "rbd",
  72             pool => "cpool",
  73             username => "admin",
  74             shared => 1,
  75         },
  76         "local-dir" => {
  77             content => {
  78                 images => 1,
  79             },
  80             path => "/some/dir/",
  81             type => "dir",
  82         },
  83         "other-dir" => {
  84             content => {
  85                 images => 1,
  86             },
  87             path => "/some/other/dir/",
  88             type => "dir",
  89         },
  90     },
  91 };
  92
  93 my $vm_configs = {
  94      105 => {
  95         'bootdisk' => 'scsi0',
  96         'cores' => 1,
  97         'ide0' => 'local-zfs:vm-105-disk-1,size=103M',
  98         'ide2' => 'none,media=cdrom',
  99         'memory' => 512,
 100         'name' => 'Copy-of-VM-newapache',
 101         'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 102         'numa' => 0,
 103         'ostype' => 'l26',
 104         'parent' => 'ohsnap',
 105         'pending' => {},
 106         'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 107         'scsihw' => 'virtio-scsi-pci',
 108         'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 109         'snapshots' => {
 110             'ohsnap' => {
 111                 'bootdisk' => 'scsi0',
 112                 'cores' => 1,
 113                 'ide2' => 'none,media=cdrom',
 114                 'memory' => 512,
 115                 'name' => 'Copy-of-VM-newapache',
 116                 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 117                 'numa' => 0,
 118                 'ostype' => 'l26',
 119                 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 120                 'scsihw' => 'virtio-scsi-pci',
 121                 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 122                 'snaptime' => 1580976924,
 123                 'sockets' => 1,
 124                 'startup' => 'order=2',
 125                 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b'
 126             },
 127         },
 128         'sockets' => 1,
 129         'startup' => 'order=2',
 130         'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b',
 131     },
 132     149 => {
 133         'agent' => '0',
 134         'bootdisk' => 'scsi0',
 135         'cores' => 1,
 136         'hotplug' => 'disk,network,usb,memory,cpu',
 137         'ide2' => 'none,media=cdrom',
 138         'memory' => 4096,
 139         'name' => 'asdf',
 140         'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1',
 141         'numa' => 1,
 142         'ostype' => 'l26',
 143         'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G',
 144         'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G',
 145         'scsihw' => 'virtio-scsi-pci',
 146         'snapshots' => {},
 147         'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460',
 148         'sockets' => 1,
 149         'startup' => 'order=2',
 150         'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0',
 151     },
 152     341 => {
 153         'arch' => 'aarch64',
 154         'bootdisk' => 'scsi0',
 155         'cores' => 1,
 156         'efidisk0' => 'local-lvm:vm-341-disk-0',
 157         'ide2' => 'none,media=cdrom',
 158         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 159         'memory' => 4096,
 160         'name' => 'VM1033',
 161         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 162         'numa' => 0,
 163         'ostype' => 'l26',
 164         'scsi0' => 'rbd-store:vm-341-disk-0,size=1G',
 165         'scsihw' => 'virtio-scsi-pci',
 166         'snapshots' => {},
 167         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 168         'sockets' => 2,
 169         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 170     },
 171     1033 => {
 172         'bootdisk' => 'scsi0',
 173         'cores' => 1,
 174         'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M',
 175         'ide2' => 'none,media=cdrom',
 176         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 177         'memory' => 4096,
 178         'name' => 'VM1033',
 179         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 180         'numa' => 0,
 181         'ostype' => 'l26',
 182         'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G',
 183         'scsihw' => 'virtio-scsi-pci',
 184         'snapshots' => {},
 185         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 186         'sockets' => 2,
 187         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 188     },
 189     4567 => {
 190         'bootdisk' => 'scsi0',
 191         'cores' => 1,
 192         'ide2' => 'none,media=cdrom',
 193         'memory' => 512,
 194         'name' => 'snapme',
 195         'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 196         'numa' => 0,
 197         'ostype' => 'l26',
 198         'parent' => 'snap1',
 199         'pending' => {},
 200         'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 201         'scsihw' => 'virtio-scsi-pci',
 202         'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 203         'snapshots' => {
 204             'snap1' => {
 205                 'bootdisk' => 'scsi0',
 206                 'cores' => 1,
 207                 'ide2' => 'none,media=cdrom',
 208                 'memory' => 512,
 209                 'name' => 'snapme',
 210                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 211                 'numa' => 0,
 212                 'ostype' => 'l26',
 213                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 214                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 215                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 216                 'scsihw' => 'virtio-scsi-pci',
 217                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 218                 'snaptime' => 1595928799,
 219                 'sockets' => 1,
 220                 'startup' => 'order=2',
 221                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 222                 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw',
 223             },
 224             'snap2' => {
 225                 'bootdisk' => 'scsi0',
 226                 'cores' => 1,
 227                 'ide2' => 'none,media=cdrom',
 228                 'memory' => 512,
 229                 'name' => 'snapme',
 230                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 231                 'numa' => 0,
 232                 'ostype' => 'l26',
 233                 'parent' => 'snap1',
 234                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 235                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 236                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 237                 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 238                 'scsihw' => 'virtio-scsi-pci',
 239                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 240                 'snaptime' => 1595928871,
 241                 'sockets' => 1,
 242                 'startup' => 'order=2',
 243                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 244                 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw',
 245             },
 246         },
 247         'sockets' => 1,
 248         'startup' => 'order=2',
 249         'unused0' => 'local-zfs:vm-4567-disk-0',
 250         'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a',
 251         },
 252 };
 253
 254 my $source_vdisks = {
 255     'local-dir' => [
 256         {
 257             'ctime' => 1589439681,
 258             'format' => 'qcow2',
 259             'parent' => undef,
 260             'size' => 1073741824,
 261             'used' => 335872,
 262             'vmid' => '149',
 263             'volid' => 'local-dir:149/vm-149-disk-0.qcow2',
 264         },
 265         {
 266             'ctime' => 1595928898,
 267             'format' => 'qcow2',
 268             'parent' => undef,
 269             'size' => 4294967296,
 270             'used' => 1811664896,
 271             'vmid' => '4567',
 272             'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2',
 273         },
 274         {
 275             'ctime' => 1595928800,
 276             'format' => 'raw',
 277             'parent' => undef,
 278             'size' => 274666496,
 279             'used' => 274669568,
 280             'vmid' => '4567',
 281             'volid' => 'local-dir:4567/vm-4567-state-snap1.raw',
 282         },
 283         {
 284             'ctime' => 1595928872,
 285             'format' => 'raw',
 286             'parent' => undef,
 287             'size' => 273258496,
 288             'used' => 273260544,
 289             'vmid' => '4567',
 290             'volid' => 'local-dir:4567/vm-4567-state-snap2.raw',
 291         },
 292     ],
 293     'local-lvm' => [
 294         {
 295             'ctime' => '1589277334',
 296             'format' => 'raw',
 297             'size' => 4294967296,
 298             'vmid' => '149',
 299             'volid' => 'local-lvm:vm-149-disk-0',
 300         },
 301         {
 302             'ctime' => '1589277334',
 303             'format' => 'raw',
 304             'size' => 4194304,
 305             'vmid' => '341',
 306             'volid' => 'local-lvm:vm-341-disk-0',
 307         },
 308     ],
 309     'local-zfs' => [
 310         {
 311             'ctime' => '1589277334',
 312             'format' => 'raw',
 313             'size' => 4294967296,
 314             'vmid' => '105',
 315             'volid' => 'local-zfs:vm-105-disk-0',
 316         },
 317         {
 318             'ctime' => '1589277334',
 319             'format' => 'raw',
 320             'size' => 108003328,
 321             'vmid' => '105',
 322             'volid' => 'local-zfs:vm-105-disk-1',
 323         },
 324         {
 325             'format' => 'raw',
 326             'name' => 'vm-4567-disk-0',
 327             'parent' => undef,
 328             'size' => 1073741824,
 329             'vmid' => '4567',
 330             'volid' => 'local-zfs:vm-4567-disk-0',
 331         },
 332     ],
 333     'rbd-store' => [
 334         {
 335             'ctime' => '1589277334',
 336             'format' => 'raw',
 337             'size' => 1073741824,
 338             'vmid' => '1033',
 339             'volid' => 'rbd-store:vm-1033-disk-1',
 340         },
 341         {
 342             'ctime' => '1589277334',
 343             'format' => 'raw',
 344             'size' => 1073741824,
 345             'vmid' => '1033',
 346             'volid' => 'rbd-store:vm-1033-cloudinit',
 347         },
 348     ],
 349 };
 350
 351 my $default_expected_calls_online = {
 352     move_config_to_node => 1,
 353     ssh_qm_start => 1,
 354     vm_stop => 1,
 355 };
 356
 357 my $default_expected_calls_offline = {
 358     move_config_to_node => 1,
 359 };
 360
 361 my $replicated_expected_calls_online = {
 362     %{$default_expected_calls_online},
 363     transfer_replication_state => 1,
 364     switch_replication_job_target => 1,
 365 };
 366
 367 my $replicated_expected_calls_offline = {
 368     %{$default_expected_calls_offline},
 369     transfer_replication_state => 1,
 370     switch_replication_job_target => 1,
 371 };
 372
 373 # helpers
 374
 375 sub get_patched_config {
 376     my ($vmid, $patch) = @_;
 377
 378     my $new_config = { %{$vm_configs->{$vmid}} };
 379     patch_config($new_config, $patch) if defined($patch);
 380
 381     return $new_config;
 382 }
 383
 384 sub patch_config {
 385     my ($config, $patch) = @_;
 386
 387     foreach my $key (keys %{$patch}) {
 388         if ($key eq 'snapshots' && defined($patch->{$key})) {
 389             my $new_snapshot_configs = {};
 390             foreach my $snap (keys %{$patch->{snapshots}}) {
 391                 my $new_snapshot_config = { %{$config->{snapshots}->{$snap}} };
 392                 patch_config($new_snapshot_config, $patch->{snapshots}->{$snap});
 393                 $new_snapshot_configs->{$snap} = $new_snapshot_config;
 394             }
 395             $config->{snapshots} = $new_snapshot_configs;
 396         } elsif (defined($patch->{$key})) {
 397             $config->{$key} = $patch->{$key};
 398         } else { # use undef value for deletion
 399             delete $config->{$key};
 400         }
 401     }
 402 }
 403
 404 sub local_volids_for_vm {
 405     my ($vmid) = @_;
 406
 407     my $res = {};
 408     foreach my $storeid (keys %{$source_vdisks}) {
 409         next if $storage_config->{ids}->{$storeid}->{shared};
 410         $res = {
 411             %{$res},
 412             map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{$source_vdisks->{$storeid}}
 413         };
 414     }
 415     return $res;
 416 }
 417
 418 my $tests = [
 419 # each test consists of the following:
 420 # name           - unique name for the test which also serves as a dir name and
 421 #                  gets passed to make, so don't use whitespace or slash
 422 # target         - hostname of target node
 423 # vmid           - ID of the VM to migrate
 424 # opts           - options for the migrate() call
 425 # target_volids  - hash of volids on the target at the beginning
 426 # vm_status      - hash with running, runningmachine and optionally runningcpu
 427 # expected_calls - hash whose keys are calls which are required
 428 #                  to be made if the migration gets far enough
 429 # expect_die     - expect the migration call to fail, and an error message
 430 #                  matching the specified text in the log
 431 # expected       - hash consisting of:
 432 #                  source_volids    - hash of volids expected on the source
 433 #                  target_volids    - hash of volids expected on the target
 434 #                  vm_config        - vm configuration hash
 435 #                  vm_status        - hash with running, runningmachine and optionally runningcpu
 436     {
 437         # NOTE get_efivars_size is mocked and returns 128K
 438         name => '341_running_efidisk_targetstorage_dir',
 439         target => 'pve1',
 440         vmid => 341,
 441         vm_status => {
 442             running => 1,
 443             runningmachine => 'pc-q35-5.0+pve0',
 444         },
 445         opts => {
 446             online => 1,
 447             'with-local-disks' => 1,
 448             targetstorage => 'local-dir',
 449         },
 450         expected_calls => $default_expected_calls_online,
 451         expected => {
 452             source_volids => {},
 453             target_volids => {
 454                 'local-dir:341/vm-341-disk-10.raw' => 1,
 455             },
 456             vm_config => get_patched_config(341, {
 457                 efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K',
 458             }),
 459             vm_status => {
 460                 running => 1,
 461                 runningmachine => 'pc-q35-5.0+pve0',
 462             },
 463         },
 464     },
 465     {
 466         # NOTE get_efivars_size is mocked and returns 128K
 467         name => '341_running_efidisk',
 468         target => 'pve1',
 469         vmid => 341,
 470         vm_status => {
 471             running => 1,
 472             runningmachine => 'pc-q35-5.0+pve0',
 473         },
 474         opts => {
 475             online => 1,
 476             'with-local-disks' => 1,
 477         },
 478         expected_calls => $default_expected_calls_online,
 479         expected => {
 480             source_volids => {},
 481             target_volids => {
 482                 'local-lvm:vm-341-disk-10' => 1,
 483             },
 484             vm_config => get_patched_config(341, {
 485                 efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K',
 486             }),
 487             vm_status => {
 488                 running => 1,
 489                 runningmachine => 'pc-q35-5.0+pve0',
 490             },
 491         },
 492     },
 493     {
 494         name => '149_running_vdisk_alloc_and_pvesm_free_fail',
 495         target => 'pve1',
 496         vmid => 149,
 497         vm_status => {
 498             running => 1,
 499             runningmachine => 'pc-q35-5.0+pve0',
 500         },
 501         opts => {
 502             online => 1,
 503             'with-local-disks' => 1,
 504         },
 505         fail_config => {
 506             vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2',
 507             pvesm_free => 'local-lvm:vm-149-disk-10',
 508         },
 509         expected_calls => {},
 510         expect_die => "remote command failed with exit code",
 511         expected => {
 512             source_volids => local_volids_for_vm(149),
 513             target_volids => {
 514                 'local-lvm:vm-149-disk-10' => 1,
 515             },
 516             vm_config => $vm_configs->{149},
 517             vm_status => {
 518                 running => 1,
 519                 runningmachine => 'pc-q35-5.0+pve0',
 520             },
 521         },
 522     },
 523     {
 524         name => '149_running_vdisk_alloc_fail',
 525         target => 'pve1',
 526         vmid => 149,
 527         vm_status => {
 528             running => 1,
 529             runningmachine => 'pc-q35-5.0+pve0',
 530         },
 531         opts => {
 532             online => 1,
 533             'with-local-disks' => 1,
 534         },
 535         fail_config => {
 536             vdisk_alloc => 'local-lvm:vm-149-disk-10',
 537         },
 538         expected_calls => {},
 539         expect_die => "remote command failed with exit code",
 540         expected => {
 541             source_volids => local_volids_for_vm(149),
 542             target_volids => {},
 543             vm_config => $vm_configs->{149},
 544             vm_status => {
 545                 running => 1,
 546                 runningmachine => 'pc-q35-5.0+pve0',
 547             },
 548         },
 549     },
 550     {
 551         name => '149_vdisk_free_fail',
 552         target => 'pve1',
 553         vmid => 149,
 554         vm_status => {
 555             running => 0,
 556         },
 557         opts => {
 558             'with-local-disks' => 1,
 559         },
 560         fail_config => {
 561             'vdisk_free' => 'local-lvm:vm-149-disk-0',
 562         },
 563         expected_calls => $default_expected_calls_offline,
 564         expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error",
 565         expected => {
 566             source_volids => {
 567                 'local-lvm:vm-149-disk-0' => 1,
 568             },
 569             target_volids => local_volids_for_vm(149),
 570             vm_config => $vm_configs->{149},
 571             vm_status => {
 572                 running => 0,
 573             },
 574         },
 575     },
 576     {
 577         name => '105_replicated_run_replication_fail',
 578         target => 'pve2',
 579         vmid => 105,
 580         vm_status => {
 581             running => 0,
 582         },
 583         target_volids => local_volids_for_vm(105),
 584         fail_config => {
 585             run_replication => 1,
 586         },
 587         expected_calls => {},
 588         expect_die => 'run_replication error',
 589         expected => {
 590             source_volids => local_volids_for_vm(105),
 591             target_volids => local_volids_for_vm(105),
 592             vm_config => $vm_configs->{105},
 593             vm_status => {
 594                 running => 0,
 595             },
 596         },
 597     },
 598     {
 599         name => '1033_running_query_migrate_fail',
 600         target => 'pve2',
 601         vmid => 1033,
 602         vm_status => {
 603             running => 1,
 604             runningmachine => 'pc-q35-5.0+pve0',
 605         },
 606         opts => {
 607             online => 1,
 608         },
 609         fail_config => {
 610             'query-migrate' => 1,
 611         },
 612         expected_calls => {},
 613         expect_die => 'online migrate failure - aborting',
 614         expected => {
 615             source_volids => {},
 616             target_volids => {},
 617             vm_config => $vm_configs->{1033},
 618             vm_status => {
 619                 running => 1,
 620                 runningmachine => 'pc-q35-5.0+pve0',
 621             },
 622         },
 623     },
 624     {
 625         name => '4567_targetstorage_dirotherdir',
 626         target => 'pve1',
 627         vmid => 4567,
 628         vm_status => {
 629             running => 0,
 630         },
 631         opts => {
 632             targetstorage => 'local-dir:other-dir,local-zfs:local-zfs',
 633         },
 634         storage_migrate_map => {
 635             'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2',
 636             'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw',
 637             'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw',
 638         },
 639         expected_calls => $default_expected_calls_offline,
 640         expected => {
 641             source_volids => {},
 642             target_volids => {
 643                 'other-dir:4567/vm-4567-disk-0.qcow2' => 1,
 644                 'other-dir:4567/vm-4567-state-snap1.raw' => 1,
 645                 'other-dir:4567/vm-4567-state-snap2.raw' => 1,
 646                 'local-zfs:vm-4567-disk-0' => 1,
 647             },
 648             vm_config => get_patched_config(4567, {
 649                 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 650                 snapshots => {
 651                     snap1 => {
 652                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 653                         'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw',
 654                     },
 655                     snap2 => {
 656                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 657                         'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 658                         'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw',
 659                     },
 660                 },
 661             }),
 662             vm_status => {
 663                 running => 0,
 664             },
 665         },
 666     },
 667     {
 668         name => '4567_running',
 669         target => 'pve1',
 670         vmid => 4567,
 671         vm_status => {
 672             running => 1,
 673             runningmachine => 'pc-i440fx-5.0+pve0',
 674         },
 675         opts => {
 676             online => 1,
 677             'with-local-disks' => 1,
 678         },
 679         expected_calls => {},
 680         expect_die => 'online storage migration not possible if snapshot exists',
 681         expected => {
 682             source_volids => local_volids_for_vm(4567),
 683             target_volids => {},
 684             vm_config => $vm_configs->{4567},
 685             vm_status => {
 686                 running => 1,
 687                 runningmachine => 'pc-i440fx-5.0+pve0',
 688             },
 689         },
 690     },
 691     {
 692         name => '4567_offline',
 693         target => 'pve1',
 694         vmid => 4567,
 695         vm_status => {
 696             running => 0,
 697         },
 698         expected_calls => $default_expected_calls_offline,
 699         expected => {
 700             source_volids => {},
 701             target_volids => local_volids_for_vm(4567),
 702             vm_config => $vm_configs->{4567},
 703             vm_status => {
 704                 running => 0,
 705             },
 706         },
 707     },
 708     {
 709         # FIXME: Maybe add orphaned drives as unused?
 710         name => '149_running_orphaned_disk_targetstorage_zfs',
 711         target => 'pve1',
 712         vmid => 149,
 713         vm_status => {
 714             running => 1,
 715             runningmachine => 'pc-q35-5.0+pve0',
 716         },
 717         opts => {
 718             online => 1,
 719             'with-local-disks' => 1,
 720             targetstorage => 'local-zfs',
 721         },
 722         config_patch => {
 723             scsi1 => undef,
 724         },
 725         storage_migrate_map => {
 726             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 727         },
 728         expected_calls => $default_expected_calls_online,
 729         expected => {
 730             source_volids => {},
 731             target_volids => {
 732                 'local-zfs:vm-149-disk-10' => 1,
 733                 'local-zfs:vm-149-disk-0' => 1,
 734             },
 735             vm_config => get_patched_config(149, {
 736                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 737                 scsi1 => undef,
 738             }),
 739             vm_status => {
 740                 running => 1,
 741                 runningmachine => 'pc-q35-5.0+pve0',
 742             },
 743         },
 744     },
 745     {
 746         # FIXME: Maybe add orphaned drives as unused?
 747         name => '149_running_orphaned_disk',
 748         target => 'pve1',
 749         vmid => 149,
 750         vm_status => {
 751             running => 1,
 752             runningmachine => 'pc-q35-5.0+pve0',
 753         },
 754         opts => {
 755             online => 1,
 756             'with-local-disks' => 1,
 757         },
 758         config_patch => {
 759             scsi1 => undef,
 760         },
 761         storage_migrate_map => {
 762             'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2',
 763         },
 764         expected_calls => $default_expected_calls_online,
 765         expected => {
 766             source_volids => {},
 767             target_volids => {
 768                 'local-lvm:vm-149-disk-10' => 1,
 769                 'local-dir:149/vm-149-disk-0.qcow2' => 1,
 770             },
 771             vm_config => get_patched_config(149, {
 772                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 773                 scsi1 => undef,
 774             }),
 775             vm_status => {
 776                 running => 1,
 777                 runningmachine => 'pc-q35-5.0+pve0',
 778             },
 779         },
 780     },
 781     {
 782         # FIXME: This test is not (yet) a realistic situation, because
 783         # storage_migrate currently never changes the format (AFAICT)
 784         # But if such migrations become possible, we need to either update
 785         # the 'format' property or simply remove it for drives migrated
 786         # with storage_migrate (the property is optional, so it shouldn't be a problem)
 787         name => '149_targetstorage_map_lvmzfs_defaultlvm',
 788         target => 'pve1',
 789         vmid => 149,
 790         vm_status => {
 791             running => 0,
 792         },
 793         opts => {
 794             targetstorage => 'local-lvm:local-zfs,local-lvm',
 795         },
 796         storage_migrate_map => {
 797             'local-lvm:vm-149-disk-0' => 'vm-149-disk-0',
 798             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 799         },
 800         expected_calls => $default_expected_calls_offline,
 801         expected => {
 802             source_volids => {},
 803             target_volids => {
 804                 'local-zfs:vm-149-disk-0' => 1,
 805                 'local-lvm:vm-149-disk-0' => 1,
 806             },
 807             vm_config => get_patched_config(149, {
 808                 scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G',
 809                 scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G',
 810             }),
 811             vm_status => {
 812                 running => 0,
 813             },
 814         },
 815     },
 816     {
 817         # FIXME same as for the previous test
 818         name => '149_targetstorage_map_dirzfs_lvmdir',
 819         target => 'pve1',
 820         vmid => 149,
 821         vm_status => {
 822             running => 0,
 823         },
 824         opts => {
 825             online => 1,
 826             'with-local-disks' => 1,
 827             targetstorage => 'local-dir:local-zfs,local-lvm:local-dir',
 828         },
 829         storage_migrate_map => {
 830             'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw',
 831             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 832         },
 833         expected_calls => $default_expected_calls_offline,
 834         expected => {
 835             source_volids => {},
 836             target_volids => {
 837                 'local-dir:149/vm-149-disk-0.raw' => 1,
 838                 'local-zfs:vm-149-disk-0' => 1,
 839             },
 840             vm_config => get_patched_config(149, {
 841                 scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G',
 842                 scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G',
 843             }),
 844             vm_status => {
 845                 running => 0,
 846             },
 847         },
 848     },
 849     {
 850         name => '149_running_targetstorage_map_lvmzfs_defaultlvm',
 851         target => 'pve1',
 852         vmid => 149,
 853         vm_status => {
 854             running => 1,
 855             runningmachine => 'pc-q35-5.0+pve0',
 856         },
 857         opts => {
 858             online => 1,
 859             'with-local-disks' => 1,
 860             targetstorage => 'local-lvm:local-zfs,local-lvm',
 861         },
 862         expected_calls => $default_expected_calls_online,
 863         expected => {
 864             source_volids => {},
 865             target_volids => {
 866                 'local-zfs:vm-149-disk-10' => 1,
 867                 'local-lvm:vm-149-disk-11' => 1,
 868             },
 869             vm_config => get_patched_config(149, {
 870                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 871                 scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G',
 872             }),
 873             vm_status => {
 874                 running => 1,
 875                 runningmachine => 'pc-q35-5.0+pve0',
 876             },
 877         },
 878     },
 879     {
 880         name => '149_running_targetstorage_map_lvmzfs_dirdir',
 881         target => 'pve1',
 882         vmid => 149,
 883         vm_status => {
 884             running => 1,
 885             runningmachine => 'pc-q35-5.0+pve0',
 886         },
 887         opts => {
 888             online => 1,
 889             'with-local-disks' => 1,
 890             targetstorage => 'local-lvm:local-zfs,local-dir:local-dir',
 891         },
 892         expected_calls => $default_expected_calls_online,
 893         expected => {
 894             source_volids => {},
 895             target_volids => {
 896                 'local-zfs:vm-149-disk-10' => 1,
 897                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 898             },
 899             vm_config => get_patched_config(149, {
 900                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 901                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 902             }),
 903             vm_status => {
 904                 running => 1,
 905                 runningmachine => 'pc-q35-5.0+pve0',
 906             },
 907         },
 908     },
 909     {
 910         name => '149_running_targetstorage_zfs',
 911         target => 'pve1',
 912         vmid => 149,
 913         vm_status => {
 914             running => 1,
 915             runningmachine => 'pc-q35-5.0+pve0',
 916         },
 917         opts => {
 918             online => 1,
 919             'with-local-disks' => 1,
 920             targetstorage => 'local-zfs',
 921         },
 922         expected_calls => $default_expected_calls_online,
 923         expected => {
 924             source_volids => {},
 925             target_volids => {
 926                 'local-zfs:vm-149-disk-10' => 1,
 927                 'local-zfs:vm-149-disk-11' => 1,
 928             },
 929             vm_config => get_patched_config(149, {
 930                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 931                 scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G',
 932             }),
 933             vm_status => {
 934                 running => 1,
 935                 runningmachine => 'pc-q35-5.0+pve0',
 936             },
 937         },
 938     },
 939     {
 940         name => '149_running_wrong_size',
 941         target => 'pve1',
 942         vmid => 149,
 943         vm_status => {
 944             running => 1,
 945             runningmachine => 'pc-q35-5.0+pve0',
 946         },
 947         opts => {
 948             online => 1,
 949             'with-local-disks' => 1,
 950         },
 951         config_patch => {
 952             scsi0 => 'local-lvm:vm-149-disk-0,size=123T',
 953         },
 954         expected_calls => $default_expected_calls_online,
 955         expected => {
 956             source_volids => {},
 957             target_volids => {
 958                 'local-lvm:vm-149-disk-10' => 1,
 959                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 960             },
 961             vm_config => get_patched_config(149, {
 962                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 963                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 964             }),
 965             vm_status => {
 966                 running => 1,
 967                 runningmachine => 'pc-q35-5.0+pve0',
 968             },
 969         },
 970     },
 971     {
 972         name => '149_running_missing_size',
 973         target => 'pve1',
 974         vmid => 149,
 975         vm_status => {
 976             running => 1,
 977             runningmachine => 'pc-q35-5.0+pve0',
 978         },
 979         opts => {
 980             online => 1,
 981             'with-local-disks' => 1,
 982         },
 983         config_patch => {
 984             scsi0 => 'local-lvm:vm-149-disk-0',
 985         },
 986         expected_calls => $default_expected_calls_online,
 987         expected => {
 988             source_volids => {},
 989             target_volids => {
 990                 'local-lvm:vm-149-disk-10' => 1,
 991                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 992             },
 993             vm_config => get_patched_config(149, {
 994                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 995                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 996             }),
 997             vm_status => {
 998                 running => 1,
 999                 runningmachine => 'pc-q35-5.0+pve0',
1000             },
1001         },
1002     },
1003     {
1004         name => '105_local_device_shared',
1005         target => 'pve1',
1006         vmid => 105,
1007         vm_status => {
1008             running => 0,
1009         },
1010         config_patch => {
1011             ide2 => '/dev/sde,shared=1',
1012         },
1013         expected_calls => $default_expected_calls_offline,
1014         expected => {
1015             source_volids => {},
1016             target_volids => local_volids_for_vm(105),
1017             vm_config => get_patched_config(105, {
1018                 ide2 => '/dev/sde,shared=1',
1019             }),
1020             vm_status => {
1021                 running => 0,
1022             },
1023         },
1024     },
1025     {
1026         name => '105_local_device_in_snapshot',
1027         target => 'pve1',
1028         vmid => 105,
1029         vm_status => {
1030             running => 0,
1031         },
1032         config_patch => {
1033             snapshots => {
1034                 ohsnap => {
1035                     ide2 => '/dev/sde',
1036                 },
1037             },
1038         },
1039         expected_calls => {},
1040         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1041         expected => {
1042             source_volids => local_volids_for_vm(105),
1043             target_volids => {},
1044             vm_config => get_patched_config(105, {
1045                 snapshots => {
1046                     ohsnap => {
1047                         ide2 => '/dev/sde',
1048                     },
1049                 },
1050             }),
1051             vm_status => {
1052                 running => 0,
1053             },
1054         },
1055     },
1056     {
1057         name => '105_local_device',
1058         target => 'pve1',
1059         vmid => 105,
1060         vm_status => {
1061             running => 0,
1062         },
1063         config_patch => {
1064             ide2 => '/dev/sde',
1065         },
1066         expected_calls => {},
1067         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1068         expected => {
1069             source_volids => local_volids_for_vm(105),
1070             target_volids => {},
1071             vm_config => get_patched_config(105, {
1072                 ide2 => '/dev/sde',
1073             }),
1074             vm_status => {
1075                 running => 0,
1076             },
1077         },
1078     },
1079     {
1080         name => '105_cdrom_in_snapshot',
1081         target => 'pve1',
1082         vmid => 105,
1083         vm_status => {
1084             running => 0,
1085         },
1086         config_patch => {
1087             snapshots => {
1088                 ohsnap => {
1089                     ide2 => 'cdrom,media=cdrom',
1090                 },
1091             },
1092         },
1093         expected_calls => {},
1094         expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap",
1095         expected => {
1096             source_volids => local_volids_for_vm(105),
1097             target_volids => {},
1098             vm_config => get_patched_config(105, {
1099                 snapshots => {
1100                     ohsnap => {
1101                         ide2 => 'cdrom,media=cdrom',
1102                     },
1103                 },
1104             }),
1105             vm_status => {
1106                 running => 0,
1107             },
1108         },
1109     },
1110     {
1111         name => '105_cdrom',
1112         target => 'pve1',
1113         vmid => 105,
1114         vm_status => {
1115             running => 0,
1116         },
1117         config_patch => {
1118             ide2 => 'cdrom,media=cdrom',
1119         },
1120         expected_calls => {},
1121         expect_die => "can't migrate local cdrom drive",
1122         expected => {
1123             source_volids => local_volids_for_vm(105),
1124             target_volids => {},
1125             vm_config => get_patched_config(105, {
1126                 ide2 => 'cdrom,media=cdrom',
1127             }),
1128             vm_status => {
1129                 running => 0,
1130             },
1131         },
1132     },
1133     {
1134         name => '149_running_missing_option_withlocaldisks',
1135         target => 'pve1',
1136         vmid => 149,
1137         vm_status => {
1138             running => 1,
1139             runningmachine => 'pc-q35-5.0+pve0',
1140         },
1141         opts => {
1142             online => 1,
1143         },
1144         expected_calls => {},
1145         expect_die => "can't live migrate attached local disks without with-local-disks option",
1146         expected => {
1147             source_volids => local_volids_for_vm(149),
1148             target_volids => {},
1149             vm_config => $vm_configs->{149},
1150             vm_status => {
1151                 running => 1,
1152                 runningmachine => 'pc-q35-5.0+pve0',
1153             },
1154         },
1155     },
1156     {
1157         name => '149_running_missing_option_online',
1158         target => 'pve1',
1159         vmid => 149,
1160         vm_status => {
1161             running => 1,
1162             runningmachine => 'pc-q35-5.0+pve0',
1163         },
1164         opts => {
1165             'with-local-disks' => 1,
1166         },
1167         expected_calls => {},
1168         expect_die => "can't migrate running VM without --online",
1169         expected => {
1170             source_volids => local_volids_for_vm(149),
1171             target_volids => {},
1172             vm_config => $vm_configs->{149},
1173             vm_status => {
1174                 running => 1,
1175                 runningmachine => 'pc-q35-5.0+pve0',
1176             },
1177         },
1178     },
1179     {
1180         name => '1033_running_customcpu',
1181         target => 'pve1',
1182         vmid => 1033,
1183         vm_status => {
1184             running => 1,
1185             runningmachine => 'pc-q35-5.0+pve0',
1186             runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1187         },
1188         opts => {
1189             online => 1,
1190         },
1191         config_patch => {
1192             cpu => 'custom-mycpu',
1193         },
1194         expected_calls => $default_expected_calls_online,
1195         expected => {
1196             source_volids => {},
1197             target_volids => {},
1198             vm_config => get_patched_config(1033, {
1199                 cpu => 'custom-mycpu',
1200             }),
1201             vm_status => {
1202                 running => 1,
1203                 runningmachine => 'pc-q35-5.0+pve0',
1204                 runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1205             },
1206         },
1207     },
1208     {
1209         name => '105_replicated_to_non_replication_target',
1210         target => 'pve1',
1211         vmid => 105,
1212         vm_status => {
1213             running => 0,
1214         },
1215         target_volids => {},
1216         expected_calls => $replicated_expected_calls_offline,
1217         expected => {
1218             source_volids => {},
1219             target_volids => local_volids_for_vm(105),
1220             vm_config => $vm_configs->{105},
1221             vm_status => {
1222                 running => 0,
1223             },
1224         },
1225     },
1226     {
1227         name => '105_running_replicated',
1228         target => 'pve2',
1229         vmid => 105,
1230         vm_status => {
1231             running => 1,
1232             runningmachine => 'pc-i440fx-5.0+pve0',
1233         },
1234         opts => {
1235             online => 1,
1236             'with-local-disks' => 1,
1237         },
1238         target_volids => local_volids_for_vm(105),
1239         expected_calls => {},
1240         expect_die => "online storage migration not possible if snapshot exists",
1241         expected => {
1242             source_volids => local_volids_for_vm(105),
1243             target_volids => local_volids_for_vm(105),
1244             vm_config => $vm_configs->{105},
1245             vm_status => {
1246                 running => 1,
1247                 runningmachine => 'pc-i440fx-5.0+pve0',
1248             },
1249         },
1250     },
1251     {
1252         name => '105_replicated',
1253         target => 'pve2',
1254         vmid => 105,
1255         vm_status => {
1256             running => 0,
1257         },
1258         target_volids => local_volids_for_vm(105),
1259         expected_calls => $replicated_expected_calls_offline,
1260         expected => {
1261             source_volids => local_volids_for_vm(105),
1262             target_volids => local_volids_for_vm(105),
1263             vm_config => $vm_configs->{105},
1264             vm_status => {
1265                 running => 0,
1266             },
1267         },
1268     },
1269     {
1270         name => '105_running_replicated_without_snapshot',
1271         target => 'pve2',
1272         vmid => 105,
1273         vm_status => {
1274             running => 1,
1275             runningmachine => 'pc-i440fx-5.0+pve0',
1276         },
1277         config_patch => {
1278             snapshots => undef,
1279         },
1280         opts => {
1281             online => 1,
1282             'with-local-disks' => 1,
1283         },
1284         target_volids => local_volids_for_vm(105),
1285         expected_calls => {
1286             %{$replicated_expected_calls_online},
1287             'block-dirty-bitmap-add-drive-scsi0' => 1,
1288             'block-dirty-bitmap-add-drive-ide0' => 1,
1289         },
1290         expected => {
1291             source_volids => local_volids_for_vm(105),
1292             target_volids => local_volids_for_vm(105),
1293             vm_config => get_patched_config(105, {
1294                 snapshots => {},
1295             }),
1296             vm_status => {
1297                 running => 1,
1298                 runningmachine => 'pc-i440fx-5.0+pve0',
1299             },
1300         },
1301     },
1302     {
1303         name => '105_replicated_without_snapshot',
1304         target => 'pve2',
1305         vmid => 105,
1306         vm_status => {
1307             running => 0,
1308         },
1309         config_patch => {
1310             snapshots => undef,
1311         },
1312         opts => {
1313             online => 1,
1314         },
1315         target_volids => local_volids_for_vm(105),
1316         expected_calls => $replicated_expected_calls_offline,
1317         expected => {
1318             source_volids => local_volids_for_vm(105),
1319             target_volids => local_volids_for_vm(105),
1320             vm_config => get_patched_config(105, {
1321                 snapshots => {},
1322             }),
1323             vm_status => {
1324                 running => 0,
1325             },
1326         },
1327     },
1328     {
1329         name => '1033_running',
1330         target => 'pve2',
1331         vmid => 1033,
1332         vm_status => {
1333             running => 1,
1334             runningmachine => 'pc-q35-5.0+pve0',
1335         },
1336         opts => {
1337             online => 1,
1338         },
1339         expected_calls => $default_expected_calls_online,
1340         expected => {
1341             source_volids => {},
1342             target_volids => {},
1343             vm_config => $vm_configs->{1033},
1344             vm_status => {
1345                 running => 1,
1346                 runningmachine => 'pc-q35-5.0+pve0',
1347             },
1348         },
1349     },
1350     {
1351         name => '149_locked',
1352         target => 'pve2',
1353         vmid => 149,
1354         vm_status => {
1355             running => 0,
1356         },
1357         config_patch => {
1358             lock => 'locked',
1359         },
1360         expected_calls => {},
1361         expect_die => "VM is locked",
1362         expected => {
1363             source_volids => local_volids_for_vm(149),
1364             target_volids => {},
1365             vm_config => get_patched_config(149, {
1366                 lock => 'locked',
1367             }),
1368             vm_status => {
1369                 running => 0,
1370             },
1371         },
1372     },
1373     {
1374         name => '149_storage_not_available',
1375         target => 'pve2',
1376         vmid => 149,
1377         vm_status => {
1378             running => 0,
1379         },
1380         expected_calls => {},
1381         expect_die => "storage 'local-lvm' is not available on node 'pve2'",
1382         expected => {
1383             source_volids => local_volids_for_vm(149),
1384             target_volids => {},
1385             vm_config => $vm_configs->{149},
1386             vm_status => {
1387                 running => 0,
1388             },
1389         },
1390     },
1391     {
1392         name => '149_running',
1393         target => 'pve1',
1394         vmid => 149,
1395         vm_status => {
1396             running => 1,
1397             runningmachine => 'pc-q35-5.0+pve0',
1398         },
1399         opts => {
1400             online => 1,
1401             'with-local-disks' => 1,
1402         },
1403         expected_calls => $default_expected_calls_online,
1404         expected => {
1405             source_volids => {},
1406             target_volids => {
1407                 'local-lvm:vm-149-disk-10' => 1,
1408                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
1409             },
1410             vm_config => get_patched_config(149, {
1411                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
1412                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
1413             }),
1414             vm_status => {
1415                 running => 1,
1416                 runningmachine => 'pc-q35-5.0+pve0',
1417             },
1418         },
1419     },
1420     {
1421         name => '149_running_drive_mirror_fail',
1422         target => 'pve1',
1423         vmid => 149,
1424         vm_status => {
1425             running => 1,
1426             runningmachine => 'pc-q35-5.0+pve0',
1427         },
1428         opts => {
1429             online => 1,
1430             'with-local-disks' => 1,
1431         },
1432         expected_calls => {},
1433         expect_die => "qemu_drive_mirror 'scsi1' error",
1434         fail_config => {
1435             'qemu_drive_mirror' => 'scsi1',
1436         },
1437         expected => {
1438             source_volids => local_volids_for_vm(149),
1439             target_volids => {},
1440             vm_config => $vm_configs->{149},
1441             vm_status => {
1442                 running => 1,
1443                 runningmachine => 'pc-q35-5.0+pve0',
1444             },
1445         },
1446     },
1447     {
1448         name => '149_running_unused_block_job_cancel_fail',
1449         target => 'pve1',
1450         vmid => 149,
1451         vm_status => {
1452             running => 1,
1453             runningmachine => 'pc-q35-5.0+pve0',
1454         },
1455         opts => {
1456             online => 1,
1457             'with-local-disks' => 1,
1458         },
1459         config_patch => {
1460             scsi1 => undef,
1461             unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1462         },
1463         expected_calls => {},
1464         expect_die => "qemu_drive_mirror_monitor 'cancel' error",
1465         # note that 'cancel' is also used to finish and that's what this test is about
1466         fail_config => {
1467             'qemu_drive_mirror_monitor' => 'cancel',
1468         },
1469         expected => {
1470             source_volids => local_volids_for_vm(149),
1471             target_volids => {},
1472             vm_config => get_patched_config(149, {
1473                 scsi1 => undef,
1474                 unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1475             }),
1476             vm_status => {
1477                 running => 1,
1478                 runningmachine => 'pc-q35-5.0+pve0',
1479             },
1480         },
1481     },
1482     {
1483         name => '149_offline',
1484         target => 'pve1',
1485         vmid => 149,
1486         vm_status => {
1487             running => 0,
1488         },
1489         opts => {
1490             'with-local-disks' => 1,
1491         },
1492         expected_calls => $default_expected_calls_offline,
1493         expected => {
1494             source_volids => {},
1495             target_volids => local_volids_for_vm(149),
1496             vm_config => $vm_configs->{149},
1497             vm_status => {
1498                 running => 0,
1499             },
1500         },
1501     },
1502     {
1503         name => '149_storage_migrate_fail',
1504         target => 'pve1',
1505         vmid => 149,
1506         vm_status => {
1507             running => 0,
1508         },
1509         opts => {
1510             'with-local-disks' => 1,
1511         },
1512         fail_config => {
1513             'storage_migrate' => 'local-lvm:vm-149-disk-0',
1514         },
1515         expected_calls => {},
1516         expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error",
1517         expected => {
1518             source_volids => local_volids_for_vm(149),
1519             target_volids => {},
1520             vm_config => $vm_configs->{149},
1521             vm_status => {
1522                 running => 0,
1523             },
1524         },
1525     },
1526 ];
1527
1528 my $single_test_name = shift;
1529
1530 if (defined($single_test_name) && $single_test_name eq 'DUMP_NAMES') {
1531     my $output = '';
1532     foreach my $test (@{$tests}) {
1533         $output .= $test->{name} . ' ';
1534     }
1535     print "$output\n";
1536     exit 0;
1537 }
1538
1539 mkdir $RUN_DIR_PATH;
1540
1541 foreach my $test (@{$tests}) {
1542     my $name = $test->{name};
1543     next if defined($single_test_name) && $name ne $single_test_name;
1544
1545     my $run_dir = "${RUN_DIR_PATH}/${name}";
1546
1547     mkdir $run_dir;
1548     file_set_contents("${run_dir}/replication_config", to_json($replication_config));
1549     file_set_contents("${run_dir}/storage_config", to_json($storage_config));
1550     file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks));
1551
1552     my $expect_die = $test->{expect_die};
1553     my $expected = $test->{expected};
1554
1555     my $source_volids = local_volids_for_vm($test->{vmid});
1556     my $target_volids = $test->{target_volids} // {};
1557
1558     my $config_patch = $test->{config_patch};
1559     my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch});
1560
1561     my $fail_config = $test->{fail_config} // {};
1562     my $storage_migrate_map = $test->{storage_migrate_map} // {};
1563
1564     if (my $targetstorage = $test->{opts}->{targetstorage}) {
1565         $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id');
1566     }
1567
1568     my $migrate_params = {
1569         target => $test->{target},
1570         vmid => $test->{vmid},
1571         opts => $test->{opts},
1572     };
1573
1574     file_set_contents("${run_dir}/nbd_info", to_json({}));
1575     file_set_contents("${run_dir}/source_volids", to_json($source_volids));
1576     file_set_contents("${run_dir}/target_volids", to_json($target_volids));
1577     file_set_contents("${run_dir}/vm_config", to_json($vm_config));
1578     file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status}));
1579     file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls}));
1580     file_set_contents("${run_dir}/fail_config", to_json($fail_config));
1581     file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map));
1582     file_set_contents("${run_dir}/migrate_params", to_json($migrate_params));
1583
1584     $ENV{QM_LIB_PATH} = $QM_LIB_PATH;
1585     $ENV{RUN_DIR_PATH} = $run_dir;
1586     my $exitcode = run_command([
1587         '/usr/bin/perl',
1588         "-I${MIGRATE_LIB_PATH}",
1589         "-I${MIGRATE_LIB_PATH}/test",
1590         "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm",
1591     ], noerr => 1, errfunc => sub {print "#$name - $_[0]\n"} );
1592
1593     if (defined($expect_die) && $exitcode) {
1594         my $log = file_get_contents("${run_dir}/log");
1595         my @lines = split /\n/, $log;
1596
1597         my $matched = 0;
1598         foreach my $line (@lines) {
1599             $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/;
1600             $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/;
1601         }
1602         if (!$matched) {
1603             fail($name);
1604             note("expected error message is not present in log");
1605         }
1606     } elsif (defined($expect_die) && !$exitcode) {
1607         fail($name);
1608         note("mocked migrate call didn't fail, but it was expected to - check log");
1609     } elsif (!defined($expect_die) && $exitcode) {
1610         fail($name);
1611         note("mocked migrate call failed, but it was not expected - check log");
1612     }
1613
1614     my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls"));
1615     foreach my $call (keys %{$expected_calls}) {
1616         fail($name);
1617         note("expected call '$call' was not made");
1618     }
1619
1620     if (!defined($expect_die)) {
1621         my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info"));
1622         foreach my $drive (keys %{$nbd_info}) {
1623             fail($name);
1624             note("drive '$drive' was not mirrored");
1625         }
1626     }
1627
1628     my $actual = {
1629         source_volids => decode_json(file_get_contents("${run_dir}/source_volids")),
1630         target_volids => decode_json(file_get_contents("${run_dir}/target_volids")),
1631         vm_config => decode_json(file_get_contents("${run_dir}/vm_config")),
1632         vm_status => decode_json(file_get_contents("${run_dir}/vm_status")),
1633     };
1634
1635     is_deeply($actual, $expected, $name);
1636 }
1637
1638 done_testing();