test/run_qemu_migrate_tests.pl

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use JSON;
   7 use Test::More;
   8 use Test::MockModule;
   9
  10 use PVE::JSONSchema;
  11 use PVE::Tools qw(file_set_contents file_get_contents run_command);
  12
  13 my $QM_LIB_PATH = '..';
  14 my $MIGRATE_LIB_PATH = '..';
  15 my $RUN_DIR_PATH = './MigrationTest/run/';
  16
  17 # test configuration shared by all tests
  18
  19 my $replication_config = {
  20     'ids' => {
  21         '105-0' => {
  22             'guest' => '105',
  23             'id' => '105-0',
  24             'jobnum' => '0',
  25             'source' => 'pve0',
  26             'target' => 'pve2',
  27             'type' => 'local'
  28         },
  29     },
  30     'order' => {
  31         '105-0' => 1,
  32     }
  33 };
  34
  35 my $storage_config = {
  36     ids => {
  37         local => {
  38             content => {
  39                 images => 1,
  40             },
  41             path => "/var/lib/vz",
  42             type => "dir",
  43             shared => 0,
  44         },
  45         "local-lvm" => {
  46             content => {
  47                 images => 1,
  48             },
  49             nodes => {
  50                 pve0 => 1,
  51                 pve1 => 1,
  52             },
  53             type => "lvmthin",
  54             thinpool => "data",
  55             vgname => "pve",
  56         },
  57         "local-zfs" => {
  58             content => {
  59                 images => 1,
  60                 rootdir => 1,
  61             },
  62             pool => "rpool/data",
  63             sparse => 1,
  64             type => "zfspool",
  65         },
  66         "rbd-store" => {
  67             monhost => "127.0.0.42,127.0.0.21,::1",
  68             fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
  69             content => {
  70                 images => 1,
  71             },
  72             type => "rbd",
  73             pool => "cpool",
  74             username => "admin",
  75             shared => 1,
  76         },
  77         "local-dir" => {
  78             content => {
  79                 images => 1,
  80             },
  81             path => "/some/dir/",
  82             type => "dir",
  83         },
  84         "other-dir" => {
  85             content => {
  86                 images => 1,
  87             },
  88             path => "/some/other/dir/",
  89             type => "dir",
  90         },
  91     },
  92 };
  93
  94 my $vm_configs = {
  95      105 => {
  96         'bootdisk' => 'scsi0',
  97         'cores' => 1,
  98         'ide0' => 'local-zfs:vm-105-disk-1,size=103M',
  99         'ide2' => 'none,media=cdrom',
 100         'memory' => 512,
 101         'name' => 'Copy-of-VM-newapache',
 102         'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 103         'numa' => 0,
 104         'ostype' => 'l26',
 105         'parent' => 'ohsnap',
 106         'pending' => {},
 107         'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 108         'scsihw' => 'virtio-scsi-pci',
 109         'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 110         'snapshots' => {
 111             'ohsnap' => {
 112                 'bootdisk' => 'scsi0',
 113                 'cores' => 1,
 114                 'ide2' => 'none,media=cdrom',
 115                 'memory' => 512,
 116                 'name' => 'Copy-of-VM-newapache',
 117                 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
 118                 'numa' => 0,
 119                 'ostype' => 'l26',
 120                 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
 121                 'scsihw' => 'virtio-scsi-pci',
 122                 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
 123                 'snaptime' => 1580976924,
 124                 'sockets' => 1,
 125                 'startup' => 'order=2',
 126                 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b'
 127             },
 128         },
 129         'sockets' => 1,
 130         'startup' => 'order=2',
 131         'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b',
 132     },
 133     149 => {
 134         'agent' => '0',
 135         'bootdisk' => 'scsi0',
 136         'cores' => 1,
 137         'hotplug' => 'disk,network,usb,memory,cpu',
 138         'ide2' => 'none,media=cdrom',
 139         'memory' => 4096,
 140         'name' => 'asdf',
 141         'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1',
 142         'numa' => 1,
 143         'ostype' => 'l26',
 144         'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G',
 145         'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G',
 146         'scsihw' => 'virtio-scsi-pci',
 147         'snapshots' => {},
 148         'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460',
 149         'sockets' => 1,
 150         'startup' => 'order=2',
 151         'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0',
 152     },
 153     341 => {
 154         'arch' => 'aarch64',
 155         'bootdisk' => 'scsi0',
 156         'cores' => 1,
 157         'efidisk0' => 'local-lvm:vm-341-disk-0',
 158         'ide2' => 'none,media=cdrom',
 159         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 160         'memory' => 4096,
 161         'name' => 'VM1033',
 162         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 163         'numa' => 0,
 164         'ostype' => 'l26',
 165         'scsi0' => 'rbd-store:vm-341-disk-0,size=1G',
 166         'scsihw' => 'virtio-scsi-pci',
 167         'snapshots' => {},
 168         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 169         'sockets' => 2,
 170         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 171     },
 172     1033 => {
 173         'bootdisk' => 'scsi0',
 174         'cores' => 1,
 175         'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M',
 176         'ide2' => 'none,media=cdrom',
 177         'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
 178         'memory' => 4096,
 179         'name' => 'VM1033',
 180         'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
 181         'numa' => 0,
 182         'ostype' => 'l26',
 183         'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G',
 184         'scsihw' => 'virtio-scsi-pci',
 185         'snapshots' => {},
 186         'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
 187         'sockets' => 2,
 188         'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
 189     },
 190     4567 => {
 191         'bootdisk' => 'scsi0',
 192         'cores' => 1,
 193         'ide2' => 'none,media=cdrom',
 194         'memory' => 512,
 195         'name' => 'snapme',
 196         'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 197         'numa' => 0,
 198         'ostype' => 'l26',
 199         'parent' => 'snap1',
 200         'pending' => {},
 201         'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 202         'scsihw' => 'virtio-scsi-pci',
 203         'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 204         'snapshots' => {
 205             'snap1' => {
 206                 'bootdisk' => 'scsi0',
 207                 'cores' => 1,
 208                 'ide2' => 'none,media=cdrom',
 209                 'memory' => 512,
 210                 'name' => 'snapme',
 211                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 212                 'numa' => 0,
 213                 'ostype' => 'l26',
 214                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 215                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 216                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 217                 'scsihw' => 'virtio-scsi-pci',
 218                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 219                 'snaptime' => 1595928799,
 220                 'sockets' => 1,
 221                 'startup' => 'order=2',
 222                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 223                 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw',
 224             },
 225             'snap2' => {
 226                 'bootdisk' => 'scsi0',
 227                 'cores' => 1,
 228                 'ide2' => 'none,media=cdrom',
 229                 'memory' => 512,
 230                 'name' => 'snapme',
 231                 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
 232                 'numa' => 0,
 233                 'ostype' => 'l26',
 234                 'parent' => 'snap1',
 235                 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
 236                 'runningmachine' => 'pc-i440fx-5.0+pve0',
 237                 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 238                 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 239                 'scsihw' => 'virtio-scsi-pci',
 240                 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
 241                 'snaptime' => 1595928871,
 242                 'sockets' => 1,
 243                 'startup' => 'order=2',
 244                 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
 245                 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw',
 246             },
 247         },
 248         'sockets' => 1,
 249         'startup' => 'order=2',
 250         'unused0' => 'local-zfs:vm-4567-disk-0',
 251         'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a',
 252         },
 253 };
 254
 255 my $source_vdisks = {
 256     'local-dir' => [
 257         {
 258             'ctime' => 1589439681,
 259             'format' => 'qcow2',
 260             'parent' => undef,
 261             'size' => 1073741824,
 262             'used' => 335872,
 263             'vmid' => '149',
 264             'volid' => 'local-dir:149/vm-149-disk-0.qcow2',
 265         },
 266         {
 267             'ctime' => 1595928898,
 268             'format' => 'qcow2',
 269             'parent' => undef,
 270             'size' => 4294967296,
 271             'used' => 1811664896,
 272             'vmid' => '4567',
 273             'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2',
 274         },
 275         {
 276             'ctime' => 1595928800,
 277             'format' => 'raw',
 278             'parent' => undef,
 279             'size' => 274666496,
 280             'used' => 274669568,
 281             'vmid' => '4567',
 282             'volid' => 'local-dir:4567/vm-4567-state-snap1.raw',
 283         },
 284         {
 285             'ctime' => 1595928872,
 286             'format' => 'raw',
 287             'parent' => undef,
 288             'size' => 273258496,
 289             'used' => 273260544,
 290             'vmid' => '4567',
 291             'volid' => 'local-dir:4567/vm-4567-state-snap2.raw',
 292         },
 293     ],
 294     'local-lvm' => [
 295         {
 296             'ctime' => '1589277334',
 297             'format' => 'raw',
 298             'size' => 4294967296,
 299             'vmid' => '149',
 300             'volid' => 'local-lvm:vm-149-disk-0',
 301         },
 302         {
 303             'ctime' => '1589277334',
 304             'format' => 'raw',
 305             'size' => 4194304,
 306             'vmid' => '341',
 307             'volid' => 'local-lvm:vm-341-disk-0',
 308         },
 309     ],
 310     'local-zfs' => [
 311         {
 312             'ctime' => '1589277334',
 313             'format' => 'raw',
 314             'size' => 4294967296,
 315             'vmid' => '105',
 316             'volid' => 'local-zfs:vm-105-disk-0',
 317         },
 318         {
 319             'ctime' => '1589277334',
 320             'format' => 'raw',
 321             'size' => 108003328,
 322             'vmid' => '105',
 323             'volid' => 'local-zfs:vm-105-disk-1',
 324         },
 325         {
 326             'format' => 'raw',
 327             'name' => 'vm-4567-disk-0',
 328             'parent' => undef,
 329             'size' => 1073741824,
 330             'vmid' => '4567',
 331             'volid' => 'local-zfs:vm-4567-disk-0',
 332         },
 333     ],
 334     'rbd-store' => [
 335         {
 336             'ctime' => '1589277334',
 337             'format' => 'raw',
 338             'size' => 1073741824,
 339             'vmid' => '1033',
 340             'volid' => 'rbd-store:vm-1033-disk-1',
 341         },
 342         {
 343             'ctime' => '1589277334',
 344             'format' => 'raw',
 345             'size' => 1073741824,
 346             'vmid' => '1033',
 347             'volid' => 'rbd-store:vm-1033-cloudinit',
 348         },
 349     ],
 350 };
 351
 352 my $default_expected_calls_online = {
 353     move_config_to_node => 1,
 354     ssh_qm_start => 1,
 355     vm_stop => 1,
 356 };
 357
 358 my $default_expected_calls_offline = {
 359     move_config_to_node => 1,
 360 };
 361
 362 my $replicated_expected_calls_online = {
 363     %{$default_expected_calls_online},
 364     transfer_replication_state => 1,
 365     switch_replication_job_target => 1,
 366 };
 367
 368 my $replicated_expected_calls_offline = {
 369     %{$default_expected_calls_offline},
 370     transfer_replication_state => 1,
 371     switch_replication_job_target => 1,
 372 };
 373
 374 # helpers
 375
 376 sub get_patched_config {
 377     my ($vmid, $patch) = @_;
 378
 379     my $new_config = { %{$vm_configs->{$vmid}} };
 380     patch_config($new_config, $patch) if defined($patch);
 381
 382     return $new_config;
 383 }
 384
 385 sub patch_config {
 386     my ($config, $patch) = @_;
 387
 388     foreach my $key (keys %{$patch}) {
 389         if ($key eq 'snapshots' && defined($patch->{$key})) {
 390             my $new_snapshot_configs = {};
 391             foreach my $snap (keys %{$patch->{snapshots}}) {
 392                 my $new_snapshot_config = { %{$config->{snapshots}->{$snap}} };
 393                 patch_config($new_snapshot_config, $patch->{snapshots}->{$snap});
 394                 $new_snapshot_configs->{$snap} = $new_snapshot_config;
 395             }
 396             $config->{snapshots} = $new_snapshot_configs;
 397         } elsif (defined($patch->{$key})) {
 398             $config->{$key} = $patch->{$key};
 399         } else { # use undef value for deletion
 400             delete $config->{$key};
 401         }
 402     }
 403 }
 404
 405 sub local_volids_for_vm {
 406     my ($vmid) = @_;
 407
 408     my $res = {};
 409     foreach my $storeid (keys %{$source_vdisks}) {
 410         next if $storage_config->{ids}->{$storeid}->{shared};
 411         $res = {
 412             %{$res},
 413             map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{$source_vdisks->{$storeid}}
 414         };
 415     }
 416     return $res;
 417 }
 418
 419 my $tests = [
 420 # each test consists of the following:
 421 # name           - unique name for the test which also serves as a dir name and
 422 #                  gets passed to make, so don't use whitespace or slash
 423 # target         - hostname of target node
 424 # vmid           - ID of the VM to migrate
 425 # opts           - options for the migrate() call
 426 # target_volids  - hash of volids on the target at the beginning
 427 # vm_status      - hash with running, runningmachine and optionally runningcpu
 428 # expected_calls - hash whose keys are calls which are required
 429 #                  to be made if the migration gets far enough
 430 # expect_die     - expect the migration call to fail, and an error message
 431 #                  matching the specified text in the log
 432 # expected       - hash consisting of:
 433 #                  source_volids    - hash of volids expected on the source
 434 #                  target_volids    - hash of volids expected on the target
 435 #                  vm_config        - vm configuration hash
 436 #                  vm_status        - hash with running, runningmachine and optionally runningcpu
 437     {
 438         # NOTE get_efivars_size is mocked and returns 128K
 439         name => '341_running_efidisk_targetstorage_dir',
 440         target => 'pve1',
 441         vmid => 341,
 442         vm_status => {
 443             running => 1,
 444             runningmachine => 'pc-q35-5.0+pve0',
 445         },
 446         opts => {
 447             online => 1,
 448             'with-local-disks' => 1,
 449             targetstorage => 'local-dir',
 450         },
 451         expected_calls => $default_expected_calls_online,
 452         expected => {
 453             source_volids => {},
 454             target_volids => {
 455                 'local-dir:341/vm-341-disk-10.raw' => 1,
 456             },
 457             vm_config => get_patched_config(341, {
 458                 efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K',
 459             }),
 460             vm_status => {
 461                 running => 1,
 462                 runningmachine => 'pc-q35-5.0+pve0',
 463             },
 464         },
 465     },
 466     {
 467         # NOTE get_efivars_size is mocked and returns 128K
 468         name => '341_running_efidisk',
 469         target => 'pve1',
 470         vmid => 341,
 471         vm_status => {
 472             running => 1,
 473             runningmachine => 'pc-q35-5.0+pve0',
 474         },
 475         opts => {
 476             online => 1,
 477             'with-local-disks' => 1,
 478         },
 479         expected_calls => $default_expected_calls_online,
 480         expected => {
 481             source_volids => {},
 482             target_volids => {
 483                 'local-lvm:vm-341-disk-10' => 1,
 484             },
 485             vm_config => get_patched_config(341, {
 486                 efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K',
 487             }),
 488             vm_status => {
 489                 running => 1,
 490                 runningmachine => 'pc-q35-5.0+pve0',
 491             },
 492         },
 493     },
 494     {
 495         name => '149_running_vdisk_alloc_and_pvesm_free_fail',
 496         target => 'pve1',
 497         vmid => 149,
 498         vm_status => {
 499             running => 1,
 500             runningmachine => 'pc-q35-5.0+pve0',
 501         },
 502         opts => {
 503             online => 1,
 504             'with-local-disks' => 1,
 505         },
 506         fail_config => {
 507             vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2',
 508             pvesm_free => 'local-lvm:vm-149-disk-10',
 509         },
 510         expected_calls => {},
 511         expect_die => "remote command failed with exit code",
 512         expected => {
 513             source_volids => local_volids_for_vm(149),
 514             target_volids => {
 515                 'local-lvm:vm-149-disk-10' => 1,
 516             },
 517             vm_config => $vm_configs->{149},
 518             vm_status => {
 519                 running => 1,
 520                 runningmachine => 'pc-q35-5.0+pve0',
 521             },
 522         },
 523     },
 524     {
 525         name => '149_running_vdisk_alloc_fail',
 526         target => 'pve1',
 527         vmid => 149,
 528         vm_status => {
 529             running => 1,
 530             runningmachine => 'pc-q35-5.0+pve0',
 531         },
 532         opts => {
 533             online => 1,
 534             'with-local-disks' => 1,
 535         },
 536         fail_config => {
 537             vdisk_alloc => 'local-lvm:vm-149-disk-10',
 538         },
 539         expected_calls => {},
 540         expect_die => "remote command failed with exit code",
 541         expected => {
 542             source_volids => local_volids_for_vm(149),
 543             target_volids => {},
 544             vm_config => $vm_configs->{149},
 545             vm_status => {
 546                 running => 1,
 547                 runningmachine => 'pc-q35-5.0+pve0',
 548             },
 549         },
 550     },
 551     {
 552         name => '149_vdisk_free_fail',
 553         target => 'pve1',
 554         vmid => 149,
 555         vm_status => {
 556             running => 0,
 557         },
 558         opts => {
 559             'with-local-disks' => 1,
 560         },
 561         fail_config => {
 562             'vdisk_free' => 'local-lvm:vm-149-disk-0',
 563         },
 564         expected_calls => $default_expected_calls_offline,
 565         expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error",
 566         expected => {
 567             source_volids => {
 568                 'local-lvm:vm-149-disk-0' => 1,
 569             },
 570             target_volids => local_volids_for_vm(149),
 571             vm_config => $vm_configs->{149},
 572             vm_status => {
 573                 running => 0,
 574             },
 575         },
 576     },
 577     {
 578         name => '105_replicated_run_replication_fail',
 579         target => 'pve2',
 580         vmid => 105,
 581         vm_status => {
 582             running => 0,
 583         },
 584         target_volids => local_volids_for_vm(105),
 585         fail_config => {
 586             run_replication => 1,
 587         },
 588         expected_calls => {},
 589         expect_die => 'run_replication error',
 590         expected => {
 591             source_volids => local_volids_for_vm(105),
 592             target_volids => local_volids_for_vm(105),
 593             vm_config => $vm_configs->{105},
 594             vm_status => {
 595                 running => 0,
 596             },
 597         },
 598     },
 599     {
 600         name => '1033_running_query_migrate_fail',
 601         target => 'pve2',
 602         vmid => 1033,
 603         vm_status => {
 604             running => 1,
 605             runningmachine => 'pc-q35-5.0+pve0',
 606         },
 607         opts => {
 608             online => 1,
 609         },
 610         fail_config => {
 611             'query-migrate' => 1,
 612         },
 613         expected_calls => {},
 614         expect_die => 'online migrate failure - aborting',
 615         expected => {
 616             source_volids => {},
 617             target_volids => {},
 618             vm_config => $vm_configs->{1033},
 619             vm_status => {
 620                 running => 1,
 621                 runningmachine => 'pc-q35-5.0+pve0',
 622             },
 623         },
 624     },
 625     {
 626         name => '4567_targetstorage_dirotherdir',
 627         target => 'pve1',
 628         vmid => 4567,
 629         vm_status => {
 630             running => 0,
 631         },
 632         opts => {
 633             targetstorage => 'local-dir:other-dir,local-zfs:local-zfs',
 634         },
 635         storage_migrate_map => {
 636             'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2',
 637             'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw',
 638             'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw',
 639         },
 640         expected_calls => $default_expected_calls_offline,
 641         expected => {
 642             source_volids => {},
 643             target_volids => {
 644                 'other-dir:4567/vm-4567-disk-0.qcow2' => 1,
 645                 'other-dir:4567/vm-4567-state-snap1.raw' => 1,
 646                 'other-dir:4567/vm-4567-state-snap2.raw' => 1,
 647                 'local-zfs:vm-4567-disk-0' => 1,
 648             },
 649             vm_config => get_patched_config(4567, {
 650                 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 651                 snapshots => {
 652                     snap1 => {
 653                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 654                         'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw',
 655                     },
 656                     snap2 => {
 657                         'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
 658                         'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
 659                         'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw',
 660                     },
 661                 },
 662             }),
 663             vm_status => {
 664                 running => 0,
 665             },
 666         },
 667     },
 668     {
 669         name => '4567_running',
 670         target => 'pve1',
 671         vmid => 4567,
 672         vm_status => {
 673             running => 1,
 674             runningmachine => 'pc-i440fx-5.0+pve0',
 675         },
 676         opts => {
 677             online => 1,
 678             'with-local-disks' => 1,
 679         },
 680         expected_calls => {},
 681         expect_die => 'online storage migration not possible if snapshot exists',
 682         expected => {
 683             source_volids => local_volids_for_vm(4567),
 684             target_volids => {},
 685             vm_config => $vm_configs->{4567},
 686             vm_status => {
 687                 running => 1,
 688                 runningmachine => 'pc-i440fx-5.0+pve0',
 689             },
 690         },
 691     },
 692     {
 693         name => '4567_offline',
 694         target => 'pve1',
 695         vmid => 4567,
 696         vm_status => {
 697             running => 0,
 698         },
 699         expected_calls => $default_expected_calls_offline,
 700         expected => {
 701             source_volids => {},
 702             target_volids => local_volids_for_vm(4567),
 703             vm_config => $vm_configs->{4567},
 704             vm_status => {
 705                 running => 0,
 706             },
 707         },
 708     },
 709     {
 710         # FIXME: Maybe add orphaned drives as unused?
 711         name => '149_running_orphaned_disk_targetstorage_zfs',
 712         target => 'pve1',
 713         vmid => 149,
 714         vm_status => {
 715             running => 1,
 716             runningmachine => 'pc-q35-5.0+pve0',
 717         },
 718         opts => {
 719             online => 1,
 720             'with-local-disks' => 1,
 721             targetstorage => 'local-zfs',
 722         },
 723         config_patch => {
 724             scsi1 => undef,
 725         },
 726         storage_migrate_map => {
 727             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 728         },
 729         expected_calls => $default_expected_calls_online,
 730         expected => {
 731             source_volids => {},
 732             target_volids => {
 733                 'local-zfs:vm-149-disk-10' => 1,
 734                 'local-zfs:vm-149-disk-0' => 1,
 735             },
 736             vm_config => get_patched_config(149, {
 737                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 738                 scsi1 => undef,
 739             }),
 740             vm_status => {
 741                 running => 1,
 742                 runningmachine => 'pc-q35-5.0+pve0',
 743             },
 744         },
 745     },
 746     {
 747         # FIXME: Maybe add orphaned drives as unused?
 748         name => '149_running_orphaned_disk',
 749         target => 'pve1',
 750         vmid => 149,
 751         vm_status => {
 752             running => 1,
 753             runningmachine => 'pc-q35-5.0+pve0',
 754         },
 755         opts => {
 756             online => 1,
 757             'with-local-disks' => 1,
 758         },
 759         config_patch => {
 760             scsi1 => undef,
 761         },
 762         storage_migrate_map => {
 763             'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2',
 764         },
 765         expected_calls => $default_expected_calls_online,
 766         expected => {
 767             source_volids => {},
 768             target_volids => {
 769                 'local-lvm:vm-149-disk-10' => 1,
 770                 'local-dir:149/vm-149-disk-0.qcow2' => 1,
 771             },
 772             vm_config => get_patched_config(149, {
 773                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 774                 scsi1 => undef,
 775             }),
 776             vm_status => {
 777                 running => 1,
 778                 runningmachine => 'pc-q35-5.0+pve0',
 779             },
 780         },
 781     },
 782     {
 783         # FIXME: This test is not (yet) a realistic situation, because
 784         # storage_migrate currently never changes the format (AFAICT)
 785         # But if such migrations become possible, we need to either update
 786         # the 'format' property or simply remove it for drives migrated
 787         # with storage_migrate (the property is optional, so it shouldn't be a problem)
 788         name => '149_targetstorage_map_lvmzfs_defaultlvm',
 789         target => 'pve1',
 790         vmid => 149,
 791         vm_status => {
 792             running => 0,
 793         },
 794         opts => {
 795             targetstorage => 'local-lvm:local-zfs,local-lvm',
 796         },
 797         storage_migrate_map => {
 798             'local-lvm:vm-149-disk-0' => 'vm-149-disk-0',
 799             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 800         },
 801         expected_calls => $default_expected_calls_offline,
 802         expected => {
 803             source_volids => {},
 804             target_volids => {
 805                 'local-zfs:vm-149-disk-0' => 1,
 806                 'local-lvm:vm-149-disk-0' => 1,
 807             },
 808             vm_config => get_patched_config(149, {
 809                 scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G',
 810                 scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G',
 811             }),
 812             vm_status => {
 813                 running => 0,
 814             },
 815         },
 816     },
 817     {
 818         # FIXME same as for the previous test
 819         name => '149_targetstorage_map_dirzfs_lvmdir',
 820         target => 'pve1',
 821         vmid => 149,
 822         vm_status => {
 823             running => 0,
 824         },
 825         opts => {
 826             online => 1,
 827             'with-local-disks' => 1,
 828             targetstorage => 'local-dir:local-zfs,local-lvm:local-dir',
 829         },
 830         storage_migrate_map => {
 831             'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw',
 832             'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
 833         },
 834         expected_calls => $default_expected_calls_offline,
 835         expected => {
 836             source_volids => {},
 837             target_volids => {
 838                 'local-dir:149/vm-149-disk-0.raw' => 1,
 839                 'local-zfs:vm-149-disk-0' => 1,
 840             },
 841             vm_config => get_patched_config(149, {
 842                 scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G',
 843                 scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G',
 844             }),
 845             vm_status => {
 846                 running => 0,
 847             },
 848         },
 849     },
 850     {
 851         name => '149_running_targetstorage_map_lvmzfs_defaultlvm',
 852         target => 'pve1',
 853         vmid => 149,
 854         vm_status => {
 855             running => 1,
 856             runningmachine => 'pc-q35-5.0+pve0',
 857         },
 858         opts => {
 859             online => 1,
 860             'with-local-disks' => 1,
 861             targetstorage => 'local-lvm:local-zfs,local-lvm',
 862         },
 863         expected_calls => $default_expected_calls_online,
 864         expected => {
 865             source_volids => {},
 866             target_volids => {
 867                 'local-zfs:vm-149-disk-10' => 1,
 868                 'local-lvm:vm-149-disk-11' => 1,
 869             },
 870             vm_config => get_patched_config(149, {
 871                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 872                 scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G',
 873             }),
 874             vm_status => {
 875                 running => 1,
 876                 runningmachine => 'pc-q35-5.0+pve0',
 877             },
 878         },
 879     },
 880     {
 881         name => '149_running_targetstorage_map_lvmzfs_dirdir',
 882         target => 'pve1',
 883         vmid => 149,
 884         vm_status => {
 885             running => 1,
 886             runningmachine => 'pc-q35-5.0+pve0',
 887         },
 888         opts => {
 889             online => 1,
 890             'with-local-disks' => 1,
 891             targetstorage => 'local-lvm:local-zfs,local-dir:local-dir',
 892         },
 893         expected_calls => $default_expected_calls_online,
 894         expected => {
 895             source_volids => {},
 896             target_volids => {
 897                 'local-zfs:vm-149-disk-10' => 1,
 898                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 899             },
 900             vm_config => get_patched_config(149, {
 901                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 902                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 903             }),
 904             vm_status => {
 905                 running => 1,
 906                 runningmachine => 'pc-q35-5.0+pve0',
 907             },
 908         },
 909     },
 910     {
 911         name => '149_running_targetstorage_zfs',
 912         target => 'pve1',
 913         vmid => 149,
 914         vm_status => {
 915             running => 1,
 916             runningmachine => 'pc-q35-5.0+pve0',
 917         },
 918         opts => {
 919             online => 1,
 920             'with-local-disks' => 1,
 921             targetstorage => 'local-zfs',
 922         },
 923         expected_calls => $default_expected_calls_online,
 924         expected => {
 925             source_volids => {},
 926             target_volids => {
 927                 'local-zfs:vm-149-disk-10' => 1,
 928                 'local-zfs:vm-149-disk-11' => 1,
 929             },
 930             vm_config => get_patched_config(149, {
 931                 scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
 932                 scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G',
 933             }),
 934             vm_status => {
 935                 running => 1,
 936                 runningmachine => 'pc-q35-5.0+pve0',
 937             },
 938         },
 939     },
 940     {
 941         name => '149_running_wrong_size',
 942         target => 'pve1',
 943         vmid => 149,
 944         vm_status => {
 945             running => 1,
 946             runningmachine => 'pc-q35-5.0+pve0',
 947         },
 948         opts => {
 949             online => 1,
 950             'with-local-disks' => 1,
 951         },
 952         config_patch => {
 953             scsi0 => 'local-lvm:vm-149-disk-0,size=123T',
 954         },
 955         expected_calls => $default_expected_calls_online,
 956         expected => {
 957             source_volids => {},
 958             target_volids => {
 959                 'local-lvm:vm-149-disk-10' => 1,
 960                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 961             },
 962             vm_config => get_patched_config(149, {
 963                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 964                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 965             }),
 966             vm_status => {
 967                 running => 1,
 968                 runningmachine => 'pc-q35-5.0+pve0',
 969             },
 970         },
 971     },
 972     {
 973         name => '149_running_missing_size',
 974         target => 'pve1',
 975         vmid => 149,
 976         vm_status => {
 977             running => 1,
 978             runningmachine => 'pc-q35-5.0+pve0',
 979         },
 980         opts => {
 981             online => 1,
 982             'with-local-disks' => 1,
 983         },
 984         config_patch => {
 985             scsi0 => 'local-lvm:vm-149-disk-0',
 986         },
 987         expected_calls => $default_expected_calls_online,
 988         expected => {
 989             source_volids => {},
 990             target_volids => {
 991                 'local-lvm:vm-149-disk-10' => 1,
 992                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
 993             },
 994             vm_config => get_patched_config(149, {
 995                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
 996                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
 997             }),
 998             vm_status => {
 999                 running => 1,
1000                 runningmachine => 'pc-q35-5.0+pve0',
1001             },
1002         },
1003     },
1004     {
1005         name => '105_local_device_shared',
1006         target => 'pve1',
1007         vmid => 105,
1008         vm_status => {
1009             running => 0,
1010         },
1011         config_patch => {
1012             ide2 => '/dev/sde,shared=1',
1013         },
1014         expected_calls => $default_expected_calls_offline,
1015         expected => {
1016             source_volids => {},
1017             target_volids => local_volids_for_vm(105),
1018             vm_config => get_patched_config(105, {
1019                 ide2 => '/dev/sde,shared=1',
1020             }),
1021             vm_status => {
1022                 running => 0,
1023             },
1024         },
1025     },
1026     {
1027         name => '105_local_device_in_snapshot',
1028         target => 'pve1',
1029         vmid => 105,
1030         vm_status => {
1031             running => 0,
1032         },
1033         config_patch => {
1034             snapshots => {
1035                 ohsnap => {
1036                     ide2 => '/dev/sde',
1037                 },
1038             },
1039         },
1040         expected_calls => {},
1041         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1042         expected => {
1043             source_volids => local_volids_for_vm(105),
1044             target_volids => {},
1045             vm_config => get_patched_config(105, {
1046                 snapshots => {
1047                     ohsnap => {
1048                         ide2 => '/dev/sde',
1049                     },
1050                 },
1051             }),
1052             vm_status => {
1053                 running => 0,
1054             },
1055         },
1056     },
1057     {
1058         name => '105_local_device',
1059         target => 'pve1',
1060         vmid => 105,
1061         vm_status => {
1062             running => 0,
1063         },
1064         config_patch => {
1065             ide2 => '/dev/sde',
1066         },
1067         expected_calls => {},
1068         expect_die => "can't migrate local disk '/dev/sde': local file/device",
1069         expected => {
1070             source_volids => local_volids_for_vm(105),
1071             target_volids => {},
1072             vm_config => get_patched_config(105, {
1073                 ide2 => '/dev/sde',
1074             }),
1075             vm_status => {
1076                 running => 0,
1077             },
1078         },
1079     },
1080     {
1081         name => '105_cdrom_in_snapshot',
1082         target => 'pve1',
1083         vmid => 105,
1084         vm_status => {
1085             running => 0,
1086         },
1087         config_patch => {
1088             snapshots => {
1089                 ohsnap => {
1090                     ide2 => 'cdrom,media=cdrom',
1091                 },
1092             },
1093         },
1094         expected_calls => {},
1095         expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap",
1096         expected => {
1097             source_volids => local_volids_for_vm(105),
1098             target_volids => {},
1099             vm_config => get_patched_config(105, {
1100                 snapshots => {
1101                     ohsnap => {
1102                         ide2 => 'cdrom,media=cdrom',
1103                     },
1104                 },
1105             }),
1106             vm_status => {
1107                 running => 0,
1108             },
1109         },
1110     },
1111     {
1112         name => '105_cdrom',
1113         target => 'pve1',
1114         vmid => 105,
1115         vm_status => {
1116             running => 0,
1117         },
1118         config_patch => {
1119             ide2 => 'cdrom,media=cdrom',
1120         },
1121         expected_calls => {},
1122         expect_die => "can't migrate local cdrom drive",
1123         expected => {
1124             source_volids => local_volids_for_vm(105),
1125             target_volids => {},
1126             vm_config => get_patched_config(105, {
1127                 ide2 => 'cdrom,media=cdrom',
1128             }),
1129             vm_status => {
1130                 running => 0,
1131             },
1132         },
1133     },
1134     {
1135         name => '149_running_missing_option_withlocaldisks',
1136         target => 'pve1',
1137         vmid => 149,
1138         vm_status => {
1139             running => 1,
1140             runningmachine => 'pc-q35-5.0+pve0',
1141         },
1142         opts => {
1143             online => 1,
1144         },
1145         expected_calls => {},
1146         expect_die => "can't live migrate attached local disks without with-local-disks option",
1147         expected => {
1148             source_volids => local_volids_for_vm(149),
1149             target_volids => {},
1150             vm_config => $vm_configs->{149},
1151             vm_status => {
1152                 running => 1,
1153                 runningmachine => 'pc-q35-5.0+pve0',
1154             },
1155         },
1156     },
1157     {
1158         name => '149_running_missing_option_online',
1159         target => 'pve1',
1160         vmid => 149,
1161         vm_status => {
1162             running => 1,
1163             runningmachine => 'pc-q35-5.0+pve0',
1164         },
1165         opts => {
1166             'with-local-disks' => 1,
1167         },
1168         expected_calls => {},
1169         expect_die => "can't migrate running VM without --online",
1170         expected => {
1171             source_volids => local_volids_for_vm(149),
1172             target_volids => {},
1173             vm_config => $vm_configs->{149},
1174             vm_status => {
1175                 running => 1,
1176                 runningmachine => 'pc-q35-5.0+pve0',
1177             },
1178         },
1179     },
1180     {
1181         name => '1033_running_customcpu',
1182         target => 'pve1',
1183         vmid => 1033,
1184         vm_status => {
1185             running => 1,
1186             runningmachine => 'pc-q35-5.0+pve0',
1187             runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1188         },
1189         opts => {
1190             online => 1,
1191         },
1192         config_patch => {
1193             cpu => 'custom-mycpu',
1194         },
1195         expected_calls => $default_expected_calls_online,
1196         expected => {
1197             source_volids => {},
1198             target_volids => {},
1199             vm_config => get_patched_config(1033, {
1200                 cpu => 'custom-mycpu',
1201             }),
1202             vm_status => {
1203                 running => 1,
1204                 runningmachine => 'pc-q35-5.0+pve0',
1205                 runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
1206             },
1207         },
1208     },
1209     {
1210         name => '105_replicated_to_non_replication_target',
1211         target => 'pve1',
1212         vmid => 105,
1213         vm_status => {
1214             running => 0,
1215         },
1216         target_volids => {},
1217         expected_calls => $replicated_expected_calls_offline,
1218         expected => {
1219             source_volids => {},
1220             target_volids => local_volids_for_vm(105),
1221             vm_config => $vm_configs->{105},
1222             vm_status => {
1223                 running => 0,
1224             },
1225         },
1226     },
1227     {
1228         name => '105_running_replicated',
1229         target => 'pve2',
1230         vmid => 105,
1231         vm_status => {
1232             running => 1,
1233             runningmachine => 'pc-i440fx-5.0+pve0',
1234         },
1235         opts => {
1236             online => 1,
1237             'with-local-disks' => 1,
1238         },
1239         target_volids => local_volids_for_vm(105),
1240         expected_calls => {},
1241         expect_die => "online storage migration not possible if snapshot exists",
1242         expected => {
1243             source_volids => local_volids_for_vm(105),
1244             target_volids => local_volids_for_vm(105),
1245             vm_config => $vm_configs->{105},
1246             vm_status => {
1247                 running => 1,
1248                 runningmachine => 'pc-i440fx-5.0+pve0',
1249             },
1250         },
1251     },
1252     {
1253         name => '105_replicated',
1254         target => 'pve2',
1255         vmid => 105,
1256         vm_status => {
1257             running => 0,
1258         },
1259         target_volids => local_volids_for_vm(105),
1260         expected_calls => $replicated_expected_calls_offline,
1261         expected => {
1262             source_volids => local_volids_for_vm(105),
1263             target_volids => local_volids_for_vm(105),
1264             vm_config => $vm_configs->{105},
1265             vm_status => {
1266                 running => 0,
1267             },
1268         },
1269     },
1270     {
1271         name => '105_running_replicated_without_snapshot',
1272         target => 'pve2',
1273         vmid => 105,
1274         vm_status => {
1275             running => 1,
1276             runningmachine => 'pc-i440fx-5.0+pve0',
1277         },
1278         config_patch => {
1279             snapshots => undef,
1280         },
1281         opts => {
1282             online => 1,
1283             'with-local-disks' => 1,
1284         },
1285         target_volids => local_volids_for_vm(105),
1286         expected_calls => {
1287             %{$replicated_expected_calls_online},
1288             'block-dirty-bitmap-add-drive-scsi0' => 1,
1289             'block-dirty-bitmap-add-drive-ide0' => 1,
1290         },
1291         expected => {
1292             source_volids => local_volids_for_vm(105),
1293             target_volids => local_volids_for_vm(105),
1294             vm_config => get_patched_config(105, {
1295                 snapshots => {},
1296             }),
1297             vm_status => {
1298                 running => 1,
1299                 runningmachine => 'pc-i440fx-5.0+pve0',
1300             },
1301         },
1302     },
1303     {
1304         name => '105_replicated_without_snapshot',
1305         target => 'pve2',
1306         vmid => 105,
1307         vm_status => {
1308             running => 0,
1309         },
1310         config_patch => {
1311             snapshots => undef,
1312         },
1313         opts => {
1314             online => 1,
1315         },
1316         target_volids => local_volids_for_vm(105),
1317         expected_calls => $replicated_expected_calls_offline,
1318         expected => {
1319             source_volids => local_volids_for_vm(105),
1320             target_volids => local_volids_for_vm(105),
1321             vm_config => get_patched_config(105, {
1322                 snapshots => {},
1323             }),
1324             vm_status => {
1325                 running => 0,
1326             },
1327         },
1328     },
1329     {
1330         name => '1033_running',
1331         target => 'pve2',
1332         vmid => 1033,
1333         vm_status => {
1334             running => 1,
1335             runningmachine => 'pc-q35-5.0+pve0',
1336         },
1337         opts => {
1338             online => 1,
1339         },
1340         expected_calls => $default_expected_calls_online,
1341         expected => {
1342             source_volids => {},
1343             target_volids => {},
1344             vm_config => $vm_configs->{1033},
1345             vm_status => {
1346                 running => 1,
1347                 runningmachine => 'pc-q35-5.0+pve0',
1348             },
1349         },
1350     },
1351     {
1352         name => '149_locked',
1353         target => 'pve2',
1354         vmid => 149,
1355         vm_status => {
1356             running => 0,
1357         },
1358         config_patch => {
1359             lock => 'locked',
1360         },
1361         expected_calls => {},
1362         expect_die => "VM is locked",
1363         expected => {
1364             source_volids => local_volids_for_vm(149),
1365             target_volids => {},
1366             vm_config => get_patched_config(149, {
1367                 lock => 'locked',
1368             }),
1369             vm_status => {
1370                 running => 0,
1371             },
1372         },
1373     },
1374     {
1375         name => '149_storage_not_available',
1376         target => 'pve2',
1377         vmid => 149,
1378         vm_status => {
1379             running => 0,
1380         },
1381         expected_calls => {},
1382         expect_die => "storage 'local-lvm' is not available on node 'pve2'",
1383         expected => {
1384             source_volids => local_volids_for_vm(149),
1385             target_volids => {},
1386             vm_config => $vm_configs->{149},
1387             vm_status => {
1388                 running => 0,
1389             },
1390         },
1391     },
1392     {
1393         name => '149_running',
1394         target => 'pve1',
1395         vmid => 149,
1396         vm_status => {
1397             running => 1,
1398             runningmachine => 'pc-q35-5.0+pve0',
1399         },
1400         opts => {
1401             online => 1,
1402             'with-local-disks' => 1,
1403         },
1404         expected_calls => $default_expected_calls_online,
1405         expected => {
1406             source_volids => {},
1407             target_volids => {
1408                 'local-lvm:vm-149-disk-10' => 1,
1409                 'local-dir:149/vm-149-disk-11.qcow2' => 1,
1410             },
1411             vm_config => get_patched_config(149, {
1412                 scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
1413                 scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
1414             }),
1415             vm_status => {
1416                 running => 1,
1417                 runningmachine => 'pc-q35-5.0+pve0',
1418             },
1419         },
1420     },
1421     {
1422         name => '149_running_drive_mirror_fail',
1423         target => 'pve1',
1424         vmid => 149,
1425         vm_status => {
1426             running => 1,
1427             runningmachine => 'pc-q35-5.0+pve0',
1428         },
1429         opts => {
1430             online => 1,
1431             'with-local-disks' => 1,
1432         },
1433         expected_calls => {},
1434         expect_die => "qemu_drive_mirror 'scsi1' error",
1435         fail_config => {
1436             'qemu_drive_mirror' => 'scsi1',
1437         },
1438         expected => {
1439             source_volids => local_volids_for_vm(149),
1440             target_volids => {},
1441             vm_config => $vm_configs->{149},
1442             vm_status => {
1443                 running => 1,
1444                 runningmachine => 'pc-q35-5.0+pve0',
1445             },
1446         },
1447     },
1448     {
1449         name => '149_running_unused_block_job_cancel_fail',
1450         target => 'pve1',
1451         vmid => 149,
1452         vm_status => {
1453             running => 1,
1454             runningmachine => 'pc-q35-5.0+pve0',
1455         },
1456         opts => {
1457             online => 1,
1458             'with-local-disks' => 1,
1459         },
1460         config_patch => {
1461             scsi1 => undef,
1462             unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1463         },
1464         expected_calls => {},
1465         expect_die => "qemu_drive_mirror_monitor 'cancel' error",
1466         # note that 'cancel' is also used to finish and that's what this test is about
1467         fail_config => {
1468             'qemu_drive_mirror_monitor' => 'cancel',
1469         },
1470         expected => {
1471             source_volids => local_volids_for_vm(149),
1472             target_volids => {},
1473             vm_config => get_patched_config(149, {
1474                 scsi1 => undef,
1475                 unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
1476             }),
1477             vm_status => {
1478                 running => 1,
1479                 runningmachine => 'pc-q35-5.0+pve0',
1480             },
1481         },
1482     },
1483     {
1484         name => '149_offline',
1485         target => 'pve1',
1486         vmid => 149,
1487         vm_status => {
1488             running => 0,
1489         },
1490         opts => {
1491             'with-local-disks' => 1,
1492         },
1493         expected_calls => $default_expected_calls_offline,
1494         expected => {
1495             source_volids => {},
1496             target_volids => local_volids_for_vm(149),
1497             vm_config => $vm_configs->{149},
1498             vm_status => {
1499                 running => 0,
1500             },
1501         },
1502     },
1503     {
1504         name => '149_storage_migrate_fail',
1505         target => 'pve1',
1506         vmid => 149,
1507         vm_status => {
1508             running => 0,
1509         },
1510         opts => {
1511             'with-local-disks' => 1,
1512         },
1513         fail_config => {
1514             'storage_migrate' => 'local-lvm:vm-149-disk-0',
1515         },
1516         expected_calls => {},
1517         expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error",
1518         expected => {
1519             source_volids => local_volids_for_vm(149),
1520             target_volids => {},
1521             vm_config => $vm_configs->{149},
1522             vm_status => {
1523                 running => 0,
1524             },
1525         },
1526     },
1527 ];
1528
1529 my $single_test_name = shift;
1530
1531 if (defined($single_test_name) && $single_test_name eq 'DUMP_NAMES') {
1532     my $output = '';
1533     foreach my $test (@{$tests}) {
1534         $output .= $test->{name} . ' ';
1535     }
1536     print "$output\n";
1537     exit 0;
1538 }
1539
1540 mkdir $RUN_DIR_PATH;
1541
1542 foreach my $test (@{$tests}) {
1543     my $name = $test->{name};
1544     next if defined($single_test_name) && $name ne $single_test_name;
1545
1546     my $run_dir = "${RUN_DIR_PATH}/${name}";
1547
1548     mkdir $run_dir;
1549     file_set_contents("${run_dir}/replication_config", to_json($replication_config));
1550     file_set_contents("${run_dir}/storage_config", to_json($storage_config));
1551     file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks));
1552
1553     my $expect_die = $test->{expect_die};
1554     my $expected = $test->{expected};
1555
1556     my $source_volids = local_volids_for_vm($test->{vmid});
1557     my $target_volids = $test->{target_volids} // {};
1558
1559     my $config_patch = $test->{config_patch};
1560     my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch});
1561
1562     my $fail_config = $test->{fail_config} // {};
1563     my $storage_migrate_map = $test->{storage_migrate_map} // {};
1564
1565     if (my $targetstorage = $test->{opts}->{targetstorage}) {
1566         $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id');
1567     }
1568
1569     my $migrate_params = {
1570         target => $test->{target},
1571         vmid => $test->{vmid},
1572         opts => $test->{opts},
1573     };
1574
1575     file_set_contents("${run_dir}/nbd_info", to_json({}));
1576     file_set_contents("${run_dir}/source_volids", to_json($source_volids));
1577     file_set_contents("${run_dir}/target_volids", to_json($target_volids));
1578     file_set_contents("${run_dir}/vm_config", to_json($vm_config));
1579     file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status}));
1580     file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls}));
1581     file_set_contents("${run_dir}/fail_config", to_json($fail_config));
1582     file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map));
1583     file_set_contents("${run_dir}/migrate_params", to_json($migrate_params));
1584
1585     $ENV{QM_LIB_PATH} = $QM_LIB_PATH;
1586     $ENV{RUN_DIR_PATH} = $run_dir;
1587     my $exitcode = run_command([
1588         '/usr/bin/perl',
1589         "-I${MIGRATE_LIB_PATH}",
1590         "-I${MIGRATE_LIB_PATH}/test",
1591         "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm",
1592     ], noerr => 1, errfunc => sub {print "#$name - $_[0]\n"} );
1593
1594     if (defined($expect_die) && $exitcode) {
1595         my $log = file_get_contents("${run_dir}/log");
1596         my @lines = split /\n/, $log;
1597
1598         my $matched = 0;
1599         foreach my $line (@lines) {
1600             $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/;
1601             $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/;
1602         }
1603         if (!$matched) {
1604             fail($name);
1605             note("expected error message is not present in log");
1606         }
1607     } elsif (defined($expect_die) && !$exitcode) {
1608         fail($name);
1609         note("mocked migrate call didn't fail, but it was expected to - check log");
1610     } elsif (!defined($expect_die) && $exitcode) {
1611         fail($name);
1612         note("mocked migrate call failed, but it was not expected - check log");
1613     }
1614
1615     my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls"));
1616     foreach my $call (keys %{$expected_calls}) {
1617         fail($name);
1618         note("expected call '$call' was not made");
1619     }
1620
1621     if (!defined($expect_die)) {
1622         my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info"));
1623         foreach my $drive (keys %{$nbd_info}) {
1624             fail($name);
1625             note("drive '$drive' was not mirrored");
1626         }
1627     }
1628
1629     my $actual = {
1630         source_volids => decode_json(file_get_contents("${run_dir}/source_volids")),
1631         target_volids => decode_json(file_get_contents("${run_dir}/target_volids")),
1632         vm_config => decode_json(file_get_contents("${run_dir}/vm_config")),
1633         vm_status => decode_json(file_get_contents("${run_dir}/vm_status")),
1634     };
1635
1636     is_deeply($actual, $expected, $name);
1637 }
1638
1639 done_testing();