test/run_qemu_migrate_tests.pl

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use JSON;
   7 use Test::More;
   8 use Test::MockModule;
   9
  10 use PVE::JSONSchema;
  11 use PVE::Tools qw(file_set_contents file_get_contents run_command);
  12
  13 my $QM_LIB_PATH = '..';
  14 my $MIGRATE_LIB_PATH = '..';
  15 my $RUN_DIR_PATH = './MigrationTest/run/';
  16
  17 # test configuration shared by all tests
  18
  19 my $replication_config = {
  20     'ids' => {
  21         '105-0' => {
  22             'guest' => '105',
  23             'id' => '105-0',
  24             'jobnum' => '0',
  25             'source' => 'pve0',
  26             'target' => 'pve2',
  27             'type' => 'local'
  28         },
  29     },
  30     'order' => {
  31         '105-0' => 1,
  32     }
  33 };
  34
  35 my $storage_config = {
  36     ids => {
  37         local => {
  38             content => {
  39                 images => 1,
  40             },
  41             path => "/var/lib/vz",
  42             type => "dir",
  43             shared => 0,
  44         },
  45         "local-lvm" => {
  46             content => {
  47                 images => 1,
  48             },
  49             nodes => {
  50                 pve0 => 1,
  51                 pve1 => 1,
  52             },
  53             type => "lvmthin",
  54             thinpool => "data",
  55             vgname => "pve",
  56         },
  57         "local-zfs" => {
  58             content => {
  59                 images => 1,
  60                 rootdir => 1,
  61             },
  62             pool => "rpool/data",
  63             sparse => 1,
  64             type => "zfspool",
  65         },
  66         "rbd-store" => {
  67             monhost => "127.0.0.42,127.0.0.21,::1",
  68             fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
  69             content => {
  70                 images => 1,
  71             },
  72             type => "rbd",
  73             pool => "cpool",
  74             username => "admin",
  75             shared => 1,
  76         },
  77         "local-dir" => {
  78             content => {
  79                 images => 1,
  80             },
  81             path => "/some/dir/",
  82             type => "dir",
  83         },
  84         "other-dir" => {
  85             content => {
  86                 images => 1,
  87             },
  88             path => "/some/other/dir/",
  89             type => "dir",
  90         },
  91     },
  92 };
  93
  94 my $vm_configs = {
  95      105 => {
  96         'bootdisk' => 'scsi0',
  97         'cores' => 1,
  98         'ide0' => 'local-zfs:vm-105-disk-1,size=103M',
  99         'ide2' => 'none,media=cdrom',
 100         'memory' => 512,
 101         'name' => 'Copy-of-VM-newapache',
 102         'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 103         'numa' => 0,
 104         'ostype' => 'l26',
 105         'parent' => 'ohsnap',
 106         'pending' => {},
 107         'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 108         'scsihw' => 'virtio-scsi-pci',
 109         'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 110         'snapshots' => {
 111             'ohsnap' => {
 112                 'bootdisk' => 'scsi0',
 113                 'cores' => 1,
 114                 'ide2' => 'none,media=cdrom',
 115                 'memory' => 512,
 116                 'name' => 'Copy-of-VM-newapache',
 117                 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 118                 'numa' => 0,
 119                 'ostype' => 'l26',
 120                 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 121                 'scsihw' => 'virtio-scsi-pci',
 122                 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 123                 'snaptime' => 1580976924,
 124                 'sockets' => 1,
 125                 'startup' => 'order=2',
 126                 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b'
 127             },
 128         },
 129         'sockets' => 1,
 130         'startup' => 'order=2',
 131         'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b',
 132     },
 133     111 => {
 134         'bootdisk' => 'scsi0',
 135         'cores' => 1,
 136         'ide0' => 'local-lvm:vm-111-disk-0,size=4096M',
 137         'ide2' => 'none,media=cdrom',
 138         'memory' => 512,
 139         'name' => 'pending-test',
 140         'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 141         'numa' => 0,
 142         'ostype' => 'l26',
 143         'pending' => {
 144                 'scsi0' => 'local-zfs:vm-111-disk-0,size=103M',
 145         },
 146         'scsihw' => 'virtio-scsi-pci',
 147         'snapshots' => {},
 148         'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5',
 149         'sockets' => 1,
 150         'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22',
 151     },
 152     149 => {
 153         'agent' => '0',
 154         'bootdisk' => 'scsi0',
 155         'cores' => 1,
 156         'hotplug' => 'disk,network,usb,memory,cpu',
 157         'ide2' => 'none,media=cdrom',
 158         'memory' => 4096,
 159         'name' => 'asdf',
 160         'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1',
 161         'numa' => 1,
 162         'ostype' => 'l26',
 163         'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G',
 164         'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G',
 165         'scsihw' => 'virtio-scsi-pci',
 166         'snapshots' => {},
 167         'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460',
 168         'sockets' => 1,
 169         'startup' => 'order=2',
 170         'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0',
 171     },
 172     341 => {
 173         'arch' => 'aarch64',
 174         'bootdisk' => 'scsi0',
 175         'cores' => 1,
 176         'efidisk0' => 'local-lvm:vm-341-disk-0',
 177         'ide2' => 'none,media=cdrom',
 178         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 179         'memory' => 4096,
 180         'name' => 'VM1033',
 181         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 182         'numa' => 0,
 183         'ostype' => 'l26',
 184         'scsi0' => 'rbd-store:vm-341-disk-0,size=1G',
 185         'scsihw' => 'virtio-scsi-pci',
 186         'snapshots' => {},
 187         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 188         'sockets' => 2,
 189         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 190     },
 191     1033 => {
 192         'bootdisk' => 'scsi0',
 193         'cores' => 1,
 194         'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M',
 195         'ide2' => 'none,media=cdrom',
 196         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 197         'memory' => 4096,
 198         'name' => 'VM1033',
 199         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 200         'numa' => 0,
 201         'ostype' => 'l26',
 202         'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G',
 203         'scsihw' => 'virtio-scsi-pci',
 204         'snapshots' => {},
 205         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 206         'sockets' => 2,
 207         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 208     },
 209     4567 => {
 210         'bootdisk' => 'scsi0',
 211         'cores' => 1,
 212         'ide2' => 'none,media=cdrom',
 213         'memory' => 512,
 214         'name' => 'snapme',
 215         'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 216         'numa' => 0,
 217         'ostype' => 'l26',
 218         'parent' => 'snap1',
 219         'pending' => {},
 220         'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 221         'scsihw' => 'virtio-scsi-pci',
 222         'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 223         'snapshots' => {
 224             'snap1' => {
 225                 'bootdisk' => 'scsi0',
 226                 'cores' => 1,
 227                 'ide2' => 'none,media=cdrom',
 228                 'memory' => 512,
 229                 'name' => 'snapme',
 230                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 231                 'numa' => 0,
 232                 'ostype' => 'l26',
 233                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 234                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 235                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 236                 'scsihw' => 'virtio-scsi-pci',
 237                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 238                 'snaptime' => 1595928799,
 239                 'sockets' => 1,
 240                 'startup' => 'order=2',
 241                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 242                 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw',
 243             },
 244             'snap2' => {
 245                 'bootdisk' => 'scsi0',
 246                 'cores' => 1,
 247                 'ide2' => 'none,media=cdrom',
 248                 'memory' => 512,
 249                 'name' => 'snapme',
 250                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 251                 'numa' => 0,
 252                 'ostype' => 'l26',
 253                 'parent' => 'snap1',
 254                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 255                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 256                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 257                 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 258                 'scsihw' => 'virtio-scsi-pci',
 259                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 260                 'snaptime' => 1595928871,
 261                 'sockets' => 1,
 262                 'startup' => 'order=2',
 263                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 264                 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw',
 265             },
 266         },
 267         'sockets' => 1,
 268         'startup' => 'order=2',
 269         'unused0' => 'local-zfs:vm-4567-disk-0',
 270         'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a',
 271         },
 272 };
 273
 274 my $source_vdisks = {
 275     'local-dir' => [
 276         {
 277             'ctime' => 1589439681,
 278             'format' => 'qcow2',
 279             'parent' => undef,
 280             'size' => 1073741824,
 281             'used' => 335872,
 282             'vmid' => '149',
 283             'volid' => 'local-dir:149/vm-149-disk-0.qcow2',
 284         },
 285         {
 286             'ctime' => 1595928898,
 287             'format' => 'qcow2',
 288             'parent' => undef,
 289             'size' => 4294967296,
 290             'used' => 1811664896,
 291             'vmid' => '4567',
 292             'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2',
 293         },
 294         {
 295             'ctime' => 1595928800,
 296             'format' => 'raw',
 297             'parent' => undef,
 298             'size' => 274666496,
 299             'used' => 274669568,
 300             'vmid' => '4567',
 301             'volid' => 'local-dir:4567/vm-4567-state-snap1.raw',
 302         },
 303         {
 304             'ctime' => 1595928872,
 305             'format' => 'raw',
 306             'parent' => undef,
 307             'size' => 273258496,
 308             'used' => 273260544,
 309             'vmid' => '4567',
 310             'volid' => 'local-dir:4567/vm-4567-state-snap2.raw',
 311         },
 312     ],
 313     'local-lvm' => [
 314         {
 315             'ctime' => '1589277334',
 316             'format' => 'raw',
 317             'size' => 4294967296,
 318             'vmid' => '149',
 319             'volid' => 'local-lvm:vm-149-disk-0',
 320         },
 321         {
 322             'ctime' => '1589277334',
 323             'format' => 'raw',
 324             'size' => 4194304,
 325             'vmid' => '341',
 326             'volid' => 'local-lvm:vm-341-disk-0',
 327         },
 328         {
 329             'ctime' => '1589277334',
 330             'format' => 'raw',
 331             'size' =>  4294967296,
 332             'vmid' => '111',
 333             'volid' => 'local-lvm:vm-111-disk-0',
 334         },
 335     ],
 336     'local-zfs' => [
 337         {
 338             'ctime' => '1589277334',
 339             'format' => 'raw',
 340             'size' => 4294967296,
 341             'vmid' => '105',
 342             'volid' => 'local-zfs:vm-105-disk-0',
 343         },
 344         {
 345             'ctime' => '1589277334',
 346             'format' => 'raw',
 347             'size' => 108003328,
 348             'vmid' => '105',
 349             'volid' => 'local-zfs:vm-105-disk-1',
 350         },
 351         {
 352             'ctime' => '1589277334',
 353             'format' => 'raw',
 354             'size' => 108003328,
 355             'vmid' => '111',
 356             'volid' => 'local-zfs:vm-111-disk-0',
 357         },
 358         {
 359             'format' => 'raw',
 360             'name' => 'vm-4567-disk-0',
 361             'parent' => undef,
 362             'size' => 1073741824,
 363             'vmid' => '4567',
 364             'volid' => 'local-zfs:vm-4567-disk-0',
 365         },
 366     ],
 367     'rbd-store' => [
 368         {
 369             'ctime' => '1589277334',
 370             'format' => 'raw',
 371             'size' => 1073741824,
 372             'vmid' => '1033',
 373             'volid' => 'rbd-store:vm-1033-disk-1',
 374         },
 375         {
 376             'ctime' => '1589277334',
 377             'format' => 'raw',
 378             'size' => 1073741824,
 379             'vmid' => '1033',
 380             'volid' => 'rbd-store:vm-1033-cloudinit',
 381         },
 382     ],
 383 };
 384
 385 my $default_expected_calls_online = {
 386     move_config_to_node => 1,
 387     ssh_qm_start => 1,
 388     vm_stop => 1,
 389 };
 390
 391 my $default_expected_calls_offline = {
 392     move_config_to_node => 1,
 393 };
 394
 395 my $replicated_expected_calls_online = {
 396     %{$default_expected_calls_online},
 397     transfer_replication_state => 1,
 398     switch_replication_job_target => 1,
 399 };
 400
 401 my $replicated_expected_calls_offline = {
 402     %{$default_expected_calls_offline},
 403     transfer_replication_state => 1,
 404     switch_replication_job_target => 1,
 405 };
 406
 407 # helpers
 408
 409 sub get_patched_config {
 410     my ($vmid, $patch) = @_;
 411
 412     my $new_config = { %{$vm_configs->{$vmid}} };
 413     patch_config($new_config, $patch) if defined($patch);
 414
 415     return $new_config;
 416 }
 417
 418 sub patch_config {
 419     my ($config, $patch) = @_;
 420
 421     foreach my $key (keys %{$patch}) {
 422         if ($key eq 'snapshots' && defined($patch->{$key})) {
 423             my $new_snapshot_configs = {};
 424             foreach my $snap (keys %{$patch->{snapshots}}) {
 425                 my $new_snapshot_config = { %{$config->{snapshots}->{$snap}} };
 426                 patch_config($new_snapshot_config, $patch->{snapshots}->{$snap});
 427                 $new_snapshot_configs->{$snap} = $new_snapshot_config;
 428             }
 429             $config->{snapshots} = $new_snapshot_configs;
 430         } elsif (defined($patch->{$key})) {
 431             $config->{$key} = $patch->{$key};
 432         } else { # use undef value for deletion
 433             delete $config->{$key};
 434         }
 435     }
 436 }
 437
 438 sub local_volids_for_vm {
 439     my ($vmid) = @_;
 440
 441     my $res = {};
 442     foreach my $storeid (keys %{$source_vdisks}) {
 443         next if $storage_config->{ids}->{$storeid}->{shared};
 444         $res = {
 445             %{$res},
 446             map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{$source_vdisks->{$storeid}}
 447         };
 448     }
 449     return $res;
 450 }
 451
 452 my $tests = [
 453 # each test consists of the following:
 454 # name           - unique name for the test which also serves as a dir name.
 455 #                  NOTE: gets passed to make, so don't use whitespace or slash
 456 #                        and adapt buildsys (regex) on code structure changes
 457 # target         - hostname of target node
 458 # vmid           - ID of the VM to migrate
 459 # opts           - options for the migrate() call
 460 # target_volids  - hash of volids on the target at the beginning
 461 # vm_status      - hash with running, runningmachine and optionally runningcpu
 462 # expected_calls - hash whose keys are calls which are required
 463 #                  to be made if the migration gets far enough
 464 # expect_die     - expect the migration call to fail, and an error message
 465 #                  matching the specified text in the log
 466 # expected       - hash consisting of:
 467 #                  source_volids    - hash of volids expected on the source
 468 #                  target_volids    - hash of volids expected on the target
 469 #                  vm_config        - vm configuration hash
 470 #                  vm_status        - hash with running, runningmachine and optionally runningcpu
 471     {
 472         # NOTE get_efivars_size is mocked and returns 128K
 473         name => '341_running_efidisk_targetstorage_dir',
 474         target => 'pve1',
 475         vmid => 341,
 476         vm_status => {
 477             running => 1,
 478             runningmachine => 'pc-q35-5.0+pve0',
 479         },
 480         opts => {
 481             online => 1,
 482             'with-local-disks' => 1,
 483             targetstorage => 'local-dir',
 484         },
 485         expected_calls => $default_expected_calls_online,
 486         expected => {
 487             source_volids => {},
 488             target_volids => {
 489                 'local-dir:341/vm-341-disk-10.raw' => 1,
 490             },
 491             vm_config => get_patched_config(341, {
 492                 efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K',
 493             }),
 494             vm_status => {
 495                 running => 1,
 496                 runningmachine => 'pc-q35-5.0+pve0',
 497             },
 498         },
 499     },
 500     {
 501         # NOTE get_efivars_size is mocked and returns 128K
 502         name => '341_running_efidisk',
 503         target => 'pve1',
 504         vmid => 341,
 505         vm_status => {
 506             running => 1,
 507             runningmachine => 'pc-q35-5.0+pve0',
 508         },
 509         opts => {
 510             online => 1,
 511             'with-local-disks' => 1,
 512         },
 513         expected_calls => $default_expected_calls_online,
 514         expected => {
 515             source_volids => {},
 516             target_volids => {
 517                 'local-lvm:vm-341-disk-10' => 1,
 518             },
 519             vm_config => get_patched_config(341, {
 520                 efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K',
 521             }),
 522             vm_status => {
 523                 running => 1,
 524                 runningmachine => 'pc-q35-5.0+pve0',
 525             },
 526         },
 527     },
 528     {
 529         name => '149_running_vdisk_alloc_and_pvesm_free_fail',
 530         target => 'pve1',
 531         vmid => 149,
 532         vm_status => {
 533             running => 1,
 534             runningmachine => 'pc-q35-5.0+pve0',
 535         },
 536         opts => {
 537             online => 1,
 538             'with-local-disks' => 1,
 539         },
 540         fail_config => {
 541             vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2',
 542             pvesm_free => 'local-lvm:vm-149-disk-10',
 543         },
 544         expected_calls => {},
 545         expect_die => "remote command failed with exit code",
 546         expected => {
 547             source_volids => local_volids_for_vm(149),
 548             target_volids => {
 549                 'local-lvm:vm-149-disk-10' => 1,
 550             },
 551             vm_config => $vm_configs->{149},
 552             vm_status => {
 553                 running => 1,
 554                 runningmachine => 'pc-q35-5.0+pve0',
 555             },
 556         },
 557     },
 558     {
 559         name => '149_running_vdisk_alloc_fail',
 560         target => 'pve1',
 561         vmid => 149,
 562         vm_status => {
 563             running => 1,
 564             runningmachine => 'pc-q35-5.0+pve0',
 565         },
 566         opts => {
 567             online => 1,
 568             'with-local-disks' => 1,
 569         },
 570         fail_config => {
 571             vdisk_alloc => 'local-lvm:vm-149-disk-10',
 572         },
 573         expected_calls => {},
 574         expect_die => "remote command failed with exit code",
 575         expected => {
 576             source_volids => local_volids_for_vm(149),
 577             target_volids => {},
 578             vm_config => $vm_configs->{149},
 579             vm_status => {
 580                 running => 1,
 581                 runningmachine => 'pc-q35-5.0+pve0',
 582             },
 583         },
 584     },
 585     {
 586         name => '149_vdisk_free_fail',
 587         target => 'pve1',
 588         vmid => 149,
 589         vm_status => {
 590             running => 0,
 591         },
 592         opts => {
 593             'with-local-disks' => 1,
 594         },
 595         fail_config => {
 596             'vdisk_free' => 'local-lvm:vm-149-disk-0',
 597         },
 598         expected_calls => $default_expected_calls_offline,
 599         expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error",
 600         expected => {
 601             source_volids => {
 602                 'local-lvm:vm-149-disk-0' => 1,
 603             },
 604             target_volids => local_volids_for_vm(149),
 605             vm_config => $vm_configs->{149},
 606             vm_status => {
 607                 running => 0,
 608             },
 609         },
 610     },
 611     {
 612         name => '105_replicated_run_replication_fail',
 613         target => 'pve2',
 614         vmid => 105,
 615         vm_status => {
 616             running => 0,
 617         },
 618         target_volids => local_volids_for_vm(105),
 619         fail_config => {
 620             run_replication => 1,
 621         },
 622         expected_calls => {},
 623         expect_die => 'run_replication error',
 624         expected => {
 625             source_volids => local_volids_for_vm(105),
 626             target_volids => local_volids_for_vm(105),
 627             vm_config => $vm_configs->{105},
 628             vm_status => {
 629                 running => 0,
 630             },
 631         },
 632     },
 633     {
 634         name => '1033_running_query_migrate_fail',
 635         target => 'pve2',
 636         vmid => 1033,
 637         vm_status => {
 638             running => 1,
 639             runningmachine => 'pc-q35-5.0+pve0',
 640         },
 641         opts => {
 642             online => 1,
 643         },
 644         fail_config => {
 645             'query-migrate' => 1,
 646         },
 647         expected_calls => {},
 648         expect_die => 'online migrate failure - aborting',
 649         expected => {
 650             source_volids => {},
 651             target_volids => {},
 652             vm_config => $vm_configs->{1033},
 653             vm_status => {
 654                 running => 1,
 655                 runningmachine => 'pc-q35-5.0+pve0',
 656             },
 657         },
 658     },
 659     {
 660         name => '4567_targetstorage_dirotherdir',
 661         target => 'pve1',
 662         vmid => 4567,
 663         vm_status => {
 664             running => 0,
 665         },
 666         opts => {
 667             targetstorage => 'local-dir:other-dir,local-zfs:local-zfs',
 668         },
 669         storage_migrate_map => {
 670             'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2',
 671             'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw',
 672             'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw',
 673         },
 674         expected_calls => $default_expected_calls_offline,
 675         expected => {
 676             source_volids => {},
 677             target_volids => {
 678                 'other-dir:4567/vm-4567-disk-0.qcow2' => 1,
 679                 'other-dir:4567/vm-4567-state-snap1.raw' => 1,
 680                 'other-dir:4567/vm-4567-state-snap2.raw' => 1,
 681                 'local-zfs:vm-4567-disk-0' => 1,
 682             },
 683             vm_config => get_patched_config(4567, {
 684                 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 685                 snapshots => {
 686                     snap1 => {
 687                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 688                         'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw',
 689                     },
 690                     snap2 => {
 691                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 692                         'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 693                         'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw',
 694                     },
 695                 },
 696             }),
 697             vm_status => {
 698                 running => 0,
 699             },
 700         },
 701     },
 702     {
 703         name => '4567_running',
 704         target => 'pve1',
 705         vmid => 4567,
 706         vm_status => {
 707             running => 1,
 708             runningmachine => 'pc-i440fx-5.0+pve0',
 709         },
 710         opts => {
 711             online => 1,
 712             'with-local-disks' => 1,
 713         },
 714         expected_calls => {},
 715         expect_die => 'online storage migration not possible if non-replicated snapshot exists',
 716         expected => {
 717             source_volids => local_volids_for_vm(4567),
 718             target_volids => {},
 719             vm_config => $vm_configs->{4567},
 720             vm_status => {
 721                 running => 1,
 722                 runningmachine => 'pc-i440fx-5.0+pve0',
 723             },
 724         },
 725     },
 726     {
 727         name => '4567_offline',
 728         target => 'pve1',
 729         vmid => 4567,
 730         vm_status => {
 731             running => 0,
 732         },
 733         expected_calls => $default_expected_calls_offline,
 734         expected => {
 735             source_volids => {},
 736             target_volids => local_volids_for_vm(4567),
 737             vm_config => $vm_configs->{4567},
 738             vm_status => {
 739                 running => 0,
 740             },
 741         },
 742     },
 743     {
 744         name => '149_running_orphaned_disk_targetstorage_zfs',
 745         target => 'pve1',
 746         vmid => 149,
 747         vm_status => {
 748             running => 1,
 749             runningmachine => 'pc-q35-5.0+pve0',
 750         },
 751         opts => {
 752             online => 1,
 753             'with-local-disks' => 1,
 754             targetstorage => 'local-zfs',
 755         },
 756         config_patch => {
 757             scsi1 => undef,
 758         },
 759         storage_migrate_map => {
 760             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 761         },
 762         expected_calls => $default_expected_calls_online,
 763         expected => {
 764             source_volids => {
 765                 'local-dir:149/vm-149-disk-0.qcow2' => 1,
 766             },
 767             target_volids => {
 768                 'local-zfs:vm-149-disk-10' => 1,
 769             },
 770             vm_config => get_patched_config(149, {
 771                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 772                 scsi1 => undef,
 773             }),
 774             vm_status => {
 775                 running => 1,
 776                 runningmachine => 'pc-q35-5.0+pve0',
 777             },
 778         },
 779     },
 780     {
 781         name => '149_running_orphaned_disk',
 782         target => 'pve1',
 783         vmid => 149,
 784         vm_status => {
 785             running => 1,
 786             runningmachine => 'pc-q35-5.0+pve0',
 787         },
 788         opts => {
 789             online => 1,
 790             'with-local-disks' => 1,
 791         },
 792         config_patch => {
 793             scsi1 => undef,
 794         },
 795         storage_migrate_map => {
 796             'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2',
 797         },
 798         expected_calls => $default_expected_calls_online,
 799         expected => {
 800             source_volids => {
 801                 'local-dir:149/vm-149-disk-0.qcow2' => 1,
 802             },
 803             target_volids => {
 804                 'local-lvm:vm-149-disk-10' => 1,
 805             },
 806             vm_config => get_patched_config(149, {
 807                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 808                 scsi1 => undef,
 809             }),
 810             vm_status => {
 811                 running => 1,
 812                 runningmachine => 'pc-q35-5.0+pve0',
 813             },
 814         },
 815     },
 816     {
 817         # FIXME: This test is not (yet) a realistic situation, because
 818         # storage_migrate currently never changes the format (AFAICT)
 819         # But if such migrations become possible, we need to either update
 820         # the 'format' property or simply remove it for drives migrated
 821         # with storage_migrate (the property is optional, so it shouldn't be a problem)
 822         name => '149_targetstorage_map_lvmzfs_defaultlvm',
 823         target => 'pve1',
 824         vmid => 149,
 825         vm_status => {
 826             running => 0,
 827         },
 828         opts => {
 829             targetstorage => 'local-lvm:local-zfs,local-lvm',
 830         },
 831         storage_migrate_map => {
 832             'local-lvm:vm-149-disk-0' => 'vm-149-disk-0',
 833             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 834         },
 835         expected_calls => $default_expected_calls_offline,
 836         expected => {
 837             source_volids => {},
 838             target_volids => {
 839                 'local-zfs:vm-149-disk-0' => 1,
 840                 'local-lvm:vm-149-disk-0' => 1,
 841             },
 842             vm_config => get_patched_config(149, {
 843                 scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G',
 844                 scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G',
 845             }),
 846             vm_status => {
 847                 running => 0,
 848             },
 849         },
 850     },
 851     {
 852         # FIXME same as for the previous test
 853         name => '149_targetstorage_map_dirzfs_lvmdir',
 854         target => 'pve1',
 855         vmid => 149,
 856         vm_status => {
 857             running => 0,
 858         },
 859         opts => {
 860             online => 1,
 861             'with-local-disks' => 1,
 862             targetstorage => 'local-dir:local-zfs,local-lvm:local-dir',
 863         },
 864         storage_migrate_map => {
 865             'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw',
 866             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 867         },
 868         expected_calls => $default_expected_calls_offline,
 869         expected => {
 870             source_volids => {},
 871             target_volids => {
 872                 'local-dir:149/vm-149-disk-0.raw' => 1,
 873                 'local-zfs:vm-149-disk-0' => 1,
 874             },
 875             vm_config => get_patched_config(149, {
 876                 scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G',
 877                 scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G',
 878             }),
 879             vm_status => {
 880                 running => 0,
 881             },
 882         },
 883     },
 884     {
 885         name => '149_running_targetstorage_map_lvmzfs_defaultlvm',
 886         target => 'pve1',
 887         vmid => 149,
 888         vm_status => {
 889             running => 1,
 890             runningmachine => 'pc-q35-5.0+pve0',
 891         },
 892         opts => {
 893             online => 1,
 894             'with-local-disks' => 1,
 895             targetstorage => 'local-lvm:local-zfs,local-lvm',
 896         },
 897         expected_calls => $default_expected_calls_online,
 898         expected => {
 899             source_volids => {},
 900             target_volids => {
 901                 'local-zfs:vm-149-disk-10' => 1,
 902                 'local-lvm:vm-149-disk-11' => 1,
 903             },
 904             vm_config => get_patched_config(149, {
 905                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 906                 scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G',
 907             }),
 908             vm_status => {
 909                 running => 1,
 910                 runningmachine => 'pc-q35-5.0+pve0',
 911             },
 912         },
 913     },
 914     {
 915         name => '149_running_targetstorage_map_lvmzfs_dirdir',
 916         target => 'pve1',
 917         vmid => 149,
 918         vm_status => {
 919             running => 1,
 920             runningmachine => 'pc-q35-5.0+pve0',
 921         },
 922         opts => {
 923             online => 1,
 924             'with-local-disks' => 1,
 925             targetstorage => 'local-lvm:local-zfs,local-dir:local-dir',
 926         },
 927         expected_calls => $default_expected_calls_online,
 928         expected => {
 929             source_volids => {},
 930             target_volids => {
 931                 'local-zfs:vm-149-disk-10' => 1,
 932                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 933             },
 934             vm_config => get_patched_config(149, {
 935                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 936                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 937             }),
 938             vm_status => {
 939                 running => 1,
 940                 runningmachine => 'pc-q35-5.0+pve0',
 941             },
 942         },
 943     },
 944     {
 945         name => '149_running_targetstorage_zfs',
 946         target => 'pve1',
 947         vmid => 149,
 948         vm_status => {
 949             running => 1,
 950             runningmachine => 'pc-q35-5.0+pve0',
 951         },
 952         opts => {
 953             online => 1,
 954             'with-local-disks' => 1,
 955             targetstorage => 'local-zfs',
 956         },
 957         expected_calls => $default_expected_calls_online,
 958         expected => {
 959             source_volids => {},
 960             target_volids => {
 961                 'local-zfs:vm-149-disk-10' => 1,
 962                 'local-zfs:vm-149-disk-11' => 1,
 963             },
 964             vm_config => get_patched_config(149, {
 965                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 966                 scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G',
 967             }),
 968             vm_status => {
 969                 running => 1,
 970                 runningmachine => 'pc-q35-5.0+pve0',
 971             },
 972         },
 973     },
 974     {
 975         name => '149_running_wrong_size',
 976         target => 'pve1',
 977         vmid => 149,
 978         vm_status => {
 979             running => 1,
 980             runningmachine => 'pc-q35-5.0+pve0',
 981         },
 982         opts => {
 983             online => 1,
 984             'with-local-disks' => 1,
 985         },
 986         config_patch => {
 987             scsi0 => 'local-lvm:vm-149-disk-0,size=123T',
 988         },
 989         expected_calls => $default_expected_calls_online,
 990         expected => {
 991             source_volids => {},
 992             target_volids => {
 993                 'local-lvm:vm-149-disk-10' => 1,
 994                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 995             },
 996             vm_config => get_patched_config(149, {
 997                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 998                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 999             }),
1000             vm_status => {
1001                 running => 1,
1002                 runningmachine => 'pc-q35-5.0+pve0',
1003             },
1004         },
1005     },
1006     {
1007         name => '149_running_missing_size',
1008         target => 'pve1',
1009         vmid => 149,
1010         vm_status => {
1011             running => 1,
1012             runningmachine => 'pc-q35-5.0+pve0',
1013         },
1014         opts => {
1015             online => 1,
1016             'with-local-disks' => 1,
1017         },
1018         config_patch => {
1019             scsi0 => 'local-lvm:vm-149-disk-0',
1020         },
1021         expected_calls => $default_expected_calls_online,
1022         expected => {
1023             source_volids => {},
1024             target_volids => {
1025                 'local-lvm:vm-149-disk-10' => 1,
1026                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
1027             },
1028             vm_config => get_patched_config(149, {
1029                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
1030                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
1031             }),
1032             vm_status => {
1033                 running => 1,
1034                 runningmachine => 'pc-q35-5.0+pve0',
1035             },
1036         },
1037     },
1038     {
1039         name => '105_local_device_shared',
1040         target => 'pve1',
1041         vmid => 105,
1042         vm_status => {
1043             running => 0,
1044         },
1045         config_patch => {
1046             ide2 => '/dev/sde,shared=1',
1047         },
1048         expected_calls => $default_expected_calls_offline,
1049         expected => {
1050             source_volids => {},
1051             target_volids => local_volids_for_vm(105),
1052             vm_config => get_patched_config(105, {
1053                 ide2 => '/dev/sde,shared=1',
1054             }),
1055             vm_status => {
1056                 running => 0,
1057             },
1058         },
1059     },
1060     {
1061         name => '105_local_device_in_snapshot',
1062         target => 'pve1',
1063         vmid => 105,
1064         vm_status => {
1065             running => 0,
1066         },
1067         config_patch => {
1068             snapshots => {
1069                 ohsnap => {
1070                     ide2 => '/dev/sde',
1071                 },
1072             },
1073         },
1074         expected_calls => {},
1075         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1076         expected => {
1077             source_volids => local_volids_for_vm(105),
1078             target_volids => {},
1079             vm_config => get_patched_config(105, {
1080                 snapshots => {
1081                     ohsnap => {
1082                         ide2 => '/dev/sde',
1083                     },
1084                 },
1085             }),
1086             vm_status => {
1087                 running => 0,
1088             },
1089         },
1090     },
1091     {
1092         name => '105_local_device',
1093         target => 'pve1',
1094         vmid => 105,
1095         vm_status => {
1096             running => 0,
1097         },
1098         config_patch => {
1099             ide2 => '/dev/sde',
1100         },
1101         expected_calls => {},
1102         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1103         expected => {
1104             source_volids => local_volids_for_vm(105),
1105             target_volids => {},
1106             vm_config => get_patched_config(105, {
1107                 ide2 => '/dev/sde',
1108             }),
1109             vm_status => {
1110                 running => 0,
1111             },
1112         },
1113     },
1114     {
1115         name => '105_cdrom_in_snapshot',
1116         target => 'pve1',
1117         vmid => 105,
1118         vm_status => {
1119             running => 0,
1120         },
1121         config_patch => {
1122             snapshots => {
1123                 ohsnap => {
1124                     ide2 => 'cdrom,media=cdrom',
1125                 },
1126             },
1127         },
1128         expected_calls => {},
1129         expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap",
1130         expected => {
1131             source_volids => local_volids_for_vm(105),
1132             target_volids => {},
1133             vm_config => get_patched_config(105, {
1134                 snapshots => {
1135                     ohsnap => {
1136                         ide2 => 'cdrom,media=cdrom',
1137                     },
1138                 },
1139             }),
1140             vm_status => {
1141                 running => 0,
1142             },
1143         },
1144     },
1145     {
1146         name => '105_cdrom',
1147         target => 'pve1',
1148         vmid => 105,
1149         vm_status => {
1150             running => 0,
1151         },
1152         config_patch => {
1153             ide2 => 'cdrom,media=cdrom',
1154         },
1155         expected_calls => {},
1156         expect_die => "can't migrate local cdrom drive",
1157         expected => {
1158             source_volids => local_volids_for_vm(105),
1159             target_volids => {},
1160             vm_config => get_patched_config(105, {
1161                 ide2 => 'cdrom,media=cdrom',
1162             }),
1163             vm_status => {
1164                 running => 0,
1165             },
1166         },
1167     },
1168     {
1169         name => '149_running_missing_option_withlocaldisks',
1170         target => 'pve1',
1171         vmid => 149,
1172         vm_status => {
1173             running => 1,
1174             runningmachine => 'pc-q35-5.0+pve0',
1175         },
1176         opts => {
1177             online => 1,
1178         },
1179         expected_calls => {},
1180         expect_die => "can't live migrate attached local disks without with-local-disks option",
1181         expected => {
1182             source_volids => local_volids_for_vm(149),
1183             target_volids => {},
1184             vm_config => $vm_configs->{149},
1185             vm_status => {
1186                 running => 1,
1187                 runningmachine => 'pc-q35-5.0+pve0',
1188             },
1189         },
1190     },
1191     {
1192         name => '149_running_missing_option_online',
1193         target => 'pve1',
1194         vmid => 149,
1195         vm_status => {
1196             running => 1,
1197             runningmachine => 'pc-q35-5.0+pve0',
1198         },
1199         opts => {
1200             'with-local-disks' => 1,
1201         },
1202         expected_calls => {},
1203         expect_die => "can't migrate running VM without --online",
1204         expected => {
1205             source_volids => local_volids_for_vm(149),
1206             target_volids => {},
1207             vm_config => $vm_configs->{149},
1208             vm_status => {
1209                 running => 1,
1210                 runningmachine => 'pc-q35-5.0+pve0',
1211             },
1212         },
1213     },
1214     {
1215         name => '1033_running_customcpu',
1216         target => 'pve1',
1217         vmid => 1033,
1218         vm_status => {
1219             running => 1,
1220             runningmachine => 'pc-q35-5.0+pve0',
1221             runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1222         },
1223         opts => {
1224             online => 1,
1225         },
1226         config_patch => {
1227             cpu => 'custom-mycpu',
1228         },
1229         expected_calls => $default_expected_calls_online,
1230         expected => {
1231             source_volids => {},
1232             target_volids => {},
1233             vm_config => get_patched_config(1033, {
1234                 cpu => 'custom-mycpu',
1235             }),
1236             vm_status => {
1237                 running => 1,
1238                 runningmachine => 'pc-q35-5.0+pve0',
1239                 runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1240             },
1241         },
1242     },
1243     {
1244         name => '105_replicated_to_non_replication_target',
1245         target => 'pve1',
1246         vmid => 105,
1247         vm_status => {
1248             running => 0,
1249         },
1250         target_volids => {},
1251         expected_calls => $replicated_expected_calls_offline,
1252         expected => {
1253             source_volids => {},
1254             target_volids => local_volids_for_vm(105),
1255             vm_config => $vm_configs->{105},
1256             vm_status => {
1257                 running => 0,
1258             },
1259         },
1260     },
1261     {
1262         name => '105_running_replicated',
1263         target => 'pve2',
1264         vmid => 105,
1265         vm_status => {
1266             running => 1,
1267             runningmachine => 'pc-i440fx-5.0+pve0',
1268         },
1269         opts => {
1270             online => 1,
1271             'with-local-disks' => 1,
1272         },
1273         target_volids => local_volids_for_vm(105),
1274         expected_calls => {
1275             %{$replicated_expected_calls_online},
1276             'block-dirty-bitmap-add-drive-scsi0' => 1,
1277             'block-dirty-bitmap-add-drive-ide0' => 1,
1278         },
1279         expected => {
1280             source_volids => local_volids_for_vm(105),
1281             target_volids => local_volids_for_vm(105),
1282             vm_config => $vm_configs->{105},
1283             vm_status => {
1284                 running => 1,
1285                 runningmachine => 'pc-i440fx-5.0+pve0',
1286             },
1287         },
1288     },
1289     {
1290         name => '105_replicated',
1291         target => 'pve2',
1292         vmid => 105,
1293         vm_status => {
1294             running => 0,
1295         },
1296         target_volids => local_volids_for_vm(105),
1297         expected_calls => $replicated_expected_calls_offline,
1298         expected => {
1299             source_volids => local_volids_for_vm(105),
1300             target_volids => local_volids_for_vm(105),
1301             vm_config => $vm_configs->{105},
1302             vm_status => {
1303                 running => 0,
1304             },
1305         },
1306     },
1307     {
1308         name => '105_running_replicated_without_snapshot',
1309         target => 'pve2',
1310         vmid => 105,
1311         vm_status => {
1312             running => 1,
1313             runningmachine => 'pc-i440fx-5.0+pve0',
1314         },
1315         config_patch => {
1316             snapshots => undef,
1317         },
1318         opts => {
1319             online => 1,
1320             'with-local-disks' => 1,
1321         },
1322         target_volids => local_volids_for_vm(105),
1323         expected_calls => {
1324             %{$replicated_expected_calls_online},
1325             'block-dirty-bitmap-add-drive-scsi0' => 1,
1326             'block-dirty-bitmap-add-drive-ide0' => 1,
1327         },
1328         expected => {
1329             source_volids => local_volids_for_vm(105),
1330             target_volids => local_volids_for_vm(105),
1331             vm_config => get_patched_config(105, {
1332                 snapshots => {},
1333             }),
1334             vm_status => {
1335                 running => 1,
1336                 runningmachine => 'pc-i440fx-5.0+pve0',
1337             },
1338         },
1339     },
1340     {
1341         name => '105_replicated_without_snapshot',
1342         target => 'pve2',
1343         vmid => 105,
1344         vm_status => {
1345             running => 0,
1346         },
1347         config_patch => {
1348             snapshots => undef,
1349         },
1350         opts => {
1351             online => 1,
1352         },
1353         target_volids => local_volids_for_vm(105),
1354         expected_calls => $replicated_expected_calls_offline,
1355         expected => {
1356             source_volids => local_volids_for_vm(105),
1357             target_volids => local_volids_for_vm(105),
1358             vm_config => get_patched_config(105, {
1359                 snapshots => {},
1360             }),
1361             vm_status => {
1362                 running => 0,
1363             },
1364         },
1365     },
1366     {
1367         name => '1033_running',
1368         target => 'pve2',
1369         vmid => 1033,
1370         vm_status => {
1371             running => 1,
1372             runningmachine => 'pc-q35-5.0+pve0',
1373         },
1374         opts => {
1375             online => 1,
1376         },
1377         expected_calls => $default_expected_calls_online,
1378         expected => {
1379             source_volids => {},
1380             target_volids => {},
1381             vm_config => $vm_configs->{1033},
1382             vm_status => {
1383                 running => 1,
1384                 runningmachine => 'pc-q35-5.0+pve0',
1385             },
1386         },
1387     },
1388     {
1389         name => '149_locked',
1390         target => 'pve2',
1391         vmid => 149,
1392         vm_status => {
1393             running => 0,
1394         },
1395         config_patch => {
1396             lock => 'locked',
1397         },
1398         expected_calls => {},
1399         expect_die => "VM is locked",
1400         expected => {
1401             source_volids => local_volids_for_vm(149),
1402             target_volids => {},
1403             vm_config => get_patched_config(149, {
1404                 lock => 'locked',
1405             }),
1406             vm_status => {
1407                 running => 0,
1408             },
1409         },
1410     },
1411     {
1412         name => '149_storage_not_available',
1413         target => 'pve2',
1414         vmid => 149,
1415         vm_status => {
1416             running => 0,
1417         },
1418         expected_calls => {},
1419         expect_die => "storage 'local-lvm' is not available on node 'pve2'",
1420         expected => {
1421             source_volids => local_volids_for_vm(149),
1422             target_volids => {},
1423             vm_config => $vm_configs->{149},
1424             vm_status => {
1425                 running => 0,
1426             },
1427         },
1428     },
1429     {
1430         name => '149_running',
1431         target => 'pve1',
1432         vmid => 149,
1433         vm_status => {
1434             running => 1,
1435             runningmachine => 'pc-q35-5.0+pve0',
1436         },
1437         opts => {
1438             online => 1,
1439             'with-local-disks' => 1,
1440         },
1441         expected_calls => $default_expected_calls_online,
1442         expected => {
1443             source_volids => {},
1444             target_volids => {
1445                 'local-lvm:vm-149-disk-10' => 1,
1446                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
1447             },
1448             vm_config => get_patched_config(149, {
1449                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
1450                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
1451             }),
1452             vm_status => {
1453                 running => 1,
1454                 runningmachine => 'pc-q35-5.0+pve0',
1455             },
1456         },
1457     },
1458     {
1459         name => '149_running_drive_mirror_fail',
1460         target => 'pve1',
1461         vmid => 149,
1462         vm_status => {
1463             running => 1,
1464             runningmachine => 'pc-q35-5.0+pve0',
1465         },
1466         opts => {
1467             online => 1,
1468             'with-local-disks' => 1,
1469         },
1470         expected_calls => {},
1471         expect_die => "qemu_drive_mirror 'scsi1' error",
1472         fail_config => {
1473             'qemu_drive_mirror' => 'scsi1',
1474         },
1475         expected => {
1476             source_volids => local_volids_for_vm(149),
1477             target_volids => {},
1478             vm_config => $vm_configs->{149},
1479             vm_status => {
1480                 running => 1,
1481                 runningmachine => 'pc-q35-5.0+pve0',
1482             },
1483         },
1484     },
1485     {
1486         name => '149_running_unused_block_job_cancel_fail',
1487         target => 'pve1',
1488         vmid => 149,
1489         vm_status => {
1490             running => 1,
1491             runningmachine => 'pc-q35-5.0+pve0',
1492         },
1493         opts => {
1494             online => 1,
1495             'with-local-disks' => 1,
1496         },
1497         config_patch => {
1498             scsi1 => undef,
1499             unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1500         },
1501         expected_calls => {},
1502         expect_die => "qemu_drive_mirror_monitor 'cancel' error",
1503         # note that 'cancel' is also used to finish and that's what this test is about
1504         fail_config => {
1505             'qemu_drive_mirror_monitor' => 'cancel',
1506         },
1507         expected => {
1508             source_volids => local_volids_for_vm(149),
1509             target_volids => {},
1510             vm_config => get_patched_config(149, {
1511                 scsi1 => undef,
1512                 unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1513             }),
1514             vm_status => {
1515                 running => 1,
1516                 runningmachine => 'pc-q35-5.0+pve0',
1517             },
1518         },
1519     },
1520     {
1521         name => '149_offline',
1522         target => 'pve1',
1523         vmid => 149,
1524         vm_status => {
1525             running => 0,
1526         },
1527         opts => {
1528             'with-local-disks' => 1,
1529         },
1530         expected_calls => $default_expected_calls_offline,
1531         expected => {
1532             source_volids => {},
1533             target_volids => local_volids_for_vm(149),
1534             vm_config => $vm_configs->{149},
1535             vm_status => {
1536                 running => 0,
1537             },
1538         },
1539     },
1540     {
1541         name => '149_storage_migrate_fail',
1542         target => 'pve1',
1543         vmid => 149,
1544         vm_status => {
1545             running => 0,
1546         },
1547         opts => {
1548             'with-local-disks' => 1,
1549         },
1550         fail_config => {
1551             'storage_migrate' => 'local-lvm:vm-149-disk-0',
1552         },
1553         expected_calls => {},
1554         expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error",
1555         expected => {
1556             source_volids => local_volids_for_vm(149),
1557             target_volids => {},
1558             vm_config => $vm_configs->{149},
1559             vm_status => {
1560                 running => 0,
1561             },
1562         },
1563     },
1564     {
1565         name => '111_running_pending',
1566         target => 'pve1',
1567         vmid => 111,
1568         vm_status => {
1569             running => 1,
1570             runningmachine => 'pc-q35-5.0+pve0',
1571         },
1572         opts => {
1573             online => 1,
1574             'with-local-disks' => 1,
1575         },
1576         expected_calls => $default_expected_calls_online,
1577         expected => {
1578             source_volids => {},
1579             target_volids => {
1580                 'local-zfs:vm-111-disk-0' => 1,
1581                 'local-lvm:vm-111-disk-10' => 1,
1582             },
1583             vm_config => get_patched_config(111, {
1584                 ide0 => 'local-lvm:vm-111-disk-10,format=raw,size=4G',
1585                 pending => {
1586                     scsi0 => 'local-zfs:vm-111-disk-0,size=103M',
1587                 },
1588             }),
1589             vm_status => {
1590                 running => 1,
1591                 runningmachine => 'pc-q35-5.0+pve0',
1592             },
1593         },
1594     },
1595 ];
1596
1597 my $single_test_name = shift;
1598
1599 mkdir $RUN_DIR_PATH;
1600
1601 foreach my $test (@{$tests}) {
1602     my $name = $test->{name};
1603     next if defined($single_test_name) && $name ne $single_test_name;
1604
1605     my $run_dir = "${RUN_DIR_PATH}/${name}";
1606
1607     mkdir $run_dir;
1608     file_set_contents("${run_dir}/replication_config", to_json($replication_config));
1609     file_set_contents("${run_dir}/storage_config", to_json($storage_config));
1610     file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks));
1611
1612     my $expect_die = $test->{expect_die};
1613     my $expected = $test->{expected};
1614
1615     my $source_volids = local_volids_for_vm($test->{vmid});
1616     my $target_volids = $test->{target_volids} // {};
1617
1618     my $config_patch = $test->{config_patch};
1619     my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch});
1620
1621     my $fail_config = $test->{fail_config} // {};
1622     my $storage_migrate_map = $test->{storage_migrate_map} // {};
1623
1624     if (my $targetstorage = $test->{opts}->{targetstorage}) {
1625         $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id');
1626     }
1627
1628     my $migrate_params = {
1629         target => $test->{target},
1630         vmid => $test->{vmid},
1631         opts => $test->{opts},
1632     };
1633
1634     file_set_contents("${run_dir}/nbd_info", to_json({}));
1635     file_set_contents("${run_dir}/source_volids", to_json($source_volids));
1636     file_set_contents("${run_dir}/target_volids", to_json($target_volids));
1637     file_set_contents("${run_dir}/vm_config", to_json($vm_config));
1638     file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status}));
1639     file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls}));
1640     file_set_contents("${run_dir}/fail_config", to_json($fail_config));
1641     file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map));
1642     file_set_contents("${run_dir}/migrate_params", to_json($migrate_params));
1643
1644     $ENV{QM_LIB_PATH} = $QM_LIB_PATH;
1645     $ENV{RUN_DIR_PATH} = $run_dir;
1646     my $exitcode = run_command([
1647         '/usr/bin/perl',
1648         "-I${MIGRATE_LIB_PATH}",
1649         "-I${MIGRATE_LIB_PATH}/test",
1650         "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm",
1651     ], noerr => 1, errfunc => sub {print "#$name - $_[0]\n"} );
1652
1653     if (defined($expect_die) && $exitcode) {
1654         my $log = file_get_contents("${run_dir}/log");
1655         my @lines = split /\n/, $log;
1656
1657         my $matched = 0;
1658         foreach my $line (@lines) {
1659             $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/;
1660             $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/;
1661         }
1662         if (!$matched) {
1663             fail($name);
1664             note("expected error message is not present in log");
1665         }
1666     } elsif (defined($expect_die) && !$exitcode) {
1667         fail($name);
1668         note("mocked migrate call didn't fail, but it was expected to - check log");
1669     } elsif (!defined($expect_die) && $exitcode) {
1670         fail($name);
1671         note("mocked migrate call failed, but it was not expected - check log");
1672     }
1673
1674     my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls"));
1675     foreach my $call (keys %{$expected_calls}) {
1676         fail($name);
1677         note("expected call '$call' was not made");
1678     }
1679
1680     if (!defined($expect_die)) {
1681         my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info"));
1682         foreach my $drive (keys %{$nbd_info}) {
1683             fail($name);
1684             note("drive '$drive' was not mirrored");
1685         }
1686     }
1687
1688     my $actual = {
1689         source_volids => decode_json(file_get_contents("${run_dir}/source_volids")),
1690         target_volids => decode_json(file_get_contents("${run_dir}/target_volids")),
1691         vm_config => decode_json(file_get_contents("${run_dir}/vm_config")),
1692         vm_status => decode_json(file_get_contents("${run_dir}/vm_status")),
1693     };
1694
1695     is_deeply($actual, $expected, $name);
1696 }
1697
1698 done_testing();