PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122
 123 sub map_storage {
 124     my ($map, $source) = @_;
 125
 126     return $source if !defined($map);
 127
 128     return $map->{entries}->{$source}
 129         if $map->{entries} && defined($map->{entries}->{$source});
 130
 131     return $map->{default} if $map->{default};
 132
 133     # identity (fallback)
 134     return $source;
 135 }
 136
 137 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 138     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 139     type => 'string',
 140     format => 'storagepair-list',
 141     optional => 1,
 142 });
 143
 144 #no warnings 'redefine';
 145
 146 my $nodename_cache;
 147 sub nodename {
 148     $nodename_cache //= PVE::INotify::nodename();
 149     return $nodename_cache;
 150 }
 151
 152 my $watchdog_fmt = {
 153     model => {
 154         default_key => 1,
 155         type => 'string',
 156         enum => [qw(i6300esb ib700)],
 157         description => "Watchdog type to emulate.",
 158         default => 'i6300esb',
 159         optional => 1,
 160     },
 161     action => {
 162         type => 'string',
 163         enum => [qw(reset shutdown poweroff pause debug none)],
 164         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 165         optional => 1,
 166     },
 167 };
 168 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 169
 170 my $agent_fmt = {
 171     enabled => {
 172         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 173         type => 'boolean',
 174         default => 0,
 175         default_key => 1,
 176     },
 177     fstrim_cloned_disks => {
 178         description => "Run fstrim after moving a disk or migrating the VM.",
 179         type => 'boolean',
 180         optional => 1,
 181         default => 0
 182     },
 183     type => {
 184         description => "Select the agent type",
 185         type => 'string',
 186         default => 'virtio',
 187         optional => 1,
 188         enum => [qw(virtio isa)],
 189     },
 190 };
 191
 192 my $vga_fmt = {
 193     type => {
 194         description => "Select the VGA type.",
 195         type => 'string',
 196         default => 'std',
 197         optional => 1,
 198         default_key => 1,
 199         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 200     },
 201     memory => {
 202         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 203         type => 'integer',
 204         optional => 1,
 205         minimum => 4,
 206         maximum => 512,
 207     },
 208 };
 209
 210 my $ivshmem_fmt = {
 211     size => {
 212         type => 'integer',
 213         minimum => 1,
 214         description => "The size of the file in MB.",
 215     },
 216     name => {
 217         type => 'string',
 218         pattern => '[a-zA-Z0-9\-]+',
 219         optional => 1,
 220         format_description => 'string',
 221         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 222     },
 223 };
 224
 225 my $audio_fmt = {
 226     device => {
 227         type => 'string',
 228         enum => [qw(ich9-intel-hda intel-hda AC97)],
 229         description =>  "Configure an audio device."
 230     },
 231     driver =>  {
 232         type => 'string',
 233         enum => ['spice', 'none'],
 234         default => 'spice',
 235         optional => 1,
 236         description => "Driver backend for the audio device."
 237     },
 238 };
 239
 240 my $spice_enhancements_fmt = {
 241     foldersharing => {
 242         type => 'boolean',
 243         optional => 1,
 244         default => '0',
 245         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 246     },
 247     videostreaming =>  {
 248         type => 'string',
 249         enum => ['off', 'all', 'filter'],
 250         default => 'off',
 251         optional => 1,
 252         description => "Enable video streaming. Uses compression for detected video streams."
 253     },
 254 };
 255
 256 my $rng_fmt = {
 257     source => {
 258         type => 'string',
 259         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 260         default_key => 1,
 261         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 262             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 263             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 264             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 265             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 266             ." a hardware RNG from the host.",
 267     },
 268     max_bytes => {
 269         type => 'integer',
 270         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 271             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 272             ." `0` to disable limiting (potentially dangerous!).",
 273         optional => 1,
 274
 275         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 276         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 277         # reading from /dev/urandom
 278         default => 1024,
 279     },
 280     period => {
 281         type => 'integer',
 282         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 283             ." the guest to retrieve another 'max_bytes' of entropy.",
 284         optional => 1,
 285         default => 1000,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 2,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710 };
 711
 712 my $cicustom_fmt = {
 713     meta => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all meta data passed to the VM via"
 717             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737 };
 738 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 739
 740 my $confdesc_cloudinit = {
 741     citype => {
 742         optional => 1,
 743         type => 'string',
 744         description => 'Specifies the cloud-init configuration format. The default depends on the'
 745             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 746             .' and `configdrive2` for windows.',
 747         enum => ['configdrive2', 'nocloud', 'opennebula'],
 748     },
 749     ciuser => {
 750         optional => 1,
 751         type => 'string',
 752         description => "cloud-init: User name to change ssh keys and password for instead of the"
 753             ." image's configured default user.",
 754     },
 755     cipassword => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'cloud-init: Password to assign the user. Using this is generally not'
 759             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 760             .' support hashed passwords.',
 761     },
 762     cicustom => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'cloud-init: Specify custom files to replace the automatically generated'
 766             .' ones at start.',
 767         format => 'pve-qm-cicustom',
 768     },
 769     searchdomain => {
 770         optional => 1,
 771         type => 'string',
 772         description => "cloud-init: Sets DNS search domains for a container. Create will'
 773             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 774             .' are set.",
 775     },
 776     nameserver => {
 777         optional => 1,
 778         type => 'string', format => 'address-list',
 779         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 780             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 781             .' are set.",
 782     },
 783     sshkeys => {
 784         optional => 1,
 785         type => 'string',
 786         format => 'urlencoded',
 787         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 788     },
 789 };
 790
 791 # what about other qemu settings ?
 792 #cpu => 'string',
 793 #machine => 'string',
 794 #fda => 'file',
 795 #fdb => 'file',
 796 #mtdblock => 'file',
 797 #sd => 'file',
 798 #pflash => 'file',
 799 #snapshot => 'bool',
 800 #bootp => 'file',
 801 ##tftp => 'dir',
 802 ##smb => 'dir',
 803 #kernel => 'file',
 804 #append => 'string',
 805 #initrd => 'file',
 806 ##soundhw => 'string',
 807
 808 while (my ($k, $v) = each %$confdesc) {
 809     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 810 }
 811
 812 my $MAX_USB_DEVICES = 5;
 813 my $MAX_NETS = 32;
 814 my $MAX_SERIAL_PORTS = 4;
 815 my $MAX_PARALLEL_PORTS = 3;
 816 my $MAX_NUMA = 8;
 817
 818 my $numa_fmt = {
 819     cpus => {
 820         type => "string",
 821         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 822         description => "CPUs accessing this NUMA node.",
 823         format_description => "id[-id];...",
 824     },
 825     memory => {
 826         type => "number",
 827         description => "Amount of memory this NUMA node provides.",
 828         optional => 1,
 829     },
 830     hostnodes => {
 831         type => "string",
 832         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 833         description => "Host NUMA nodes to use.",
 834         format_description => "id[-id];...",
 835         optional => 1,
 836     },
 837     policy => {
 838         type => 'string',
 839         enum => [qw(preferred bind interleave)],
 840         description => "NUMA allocation policy.",
 841         optional => 1,
 842     },
 843 };
 844 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 845 my $numadesc = {
 846     optional => 1,
 847     type => 'string', format => $numa_fmt,
 848     description => "NUMA topology.",
 849 };
 850 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 851
 852 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 853     $confdesc->{"numa$i"} = $numadesc;
 854 }
 855
 856 my $nic_model_list = [
 857     'e1000',
 858     'e1000-82540em',
 859     'e1000-82544gc',
 860     'e1000-82545em',
 861     'e1000e',
 862     'i82551',
 863     'i82557b',
 864     'i82559er',
 865     'ne2k_isa',
 866     'ne2k_pci',
 867     'pcnet',
 868     'rtl8139',
 869     'virtio',
 870     'vmxnet3',
 871 ];
 872 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 873
 874 my $net_fmt_bridge_descr = <<__EOD__;
 875 Bridge to attach the network device to. The Proxmox VE standard bridge
 876 is called 'vmbr0'.
 877
 878 If you do not specify a bridge, we create a kvm user (NATed) network
 879 device, which provides DHCP and DNS services. The following addresses
 880 are used:
 881
 882  10.0.2.2   Gateway
 883  10.0.2.3   DNS Server
 884  10.0.2.4   SMB Server
 885
 886 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 887 __EOD__
 888
 889 my $net_fmt = {
 890     macaddr  => get_standard_option('mac-addr', {
 891         description => "MAC address. That address must be unique withing your network. This is"
 892             ." automatically generated if not specified.",
 893     }),
 894     model => {
 895         type => 'string',
 896         description => "Network Card Model. The 'virtio' model provides the best performance with"
 897             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 898             ." best to use 'e1000'.",
 899         enum => $nic_model_list,
 900         default_key => 1,
 901     },
 902     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 903     bridge => {
 904         type => 'string',
 905         description => $net_fmt_bridge_descr,
 906         format_description => 'bridge',
 907         pattern => '[-_.\w\d]+',
 908         optional => 1,
 909     },
 910     queues => {
 911         type => 'integer',
 912         minimum => 0, maximum => 16,
 913         description => 'Number of packet queues to be used on the device.',
 914         optional => 1,
 915     },
 916     rate => {
 917         type => 'number',
 918         minimum => 0,
 919         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 920         optional => 1,
 921     },
 922     tag => {
 923         type => 'integer',
 924         minimum => 1, maximum => 4094,
 925         description => 'VLAN tag to apply to packets on this interface.',
 926         optional => 1,
 927     },
 928     trunks => {
 929         type => 'string',
 930         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 931         description => 'VLAN trunks to pass through this interface.',
 932         format_description => 'vlanid[;vlanid...]',
 933         optional => 1,
 934     },
 935     firewall => {
 936         type => 'boolean',
 937         description => 'Whether this interface should be protected by the firewall.',
 938         optional => 1,
 939     },
 940     link_down => {
 941         type => 'boolean',
 942         description => 'Whether this interface should be disconnected (like pulling the plug).',
 943         optional => 1,
 944     },
 945     mtu => {
 946         type => 'integer',
 947         minimum => 1, maximum => 65520,
 948         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 949         optional => 1,
 950     },
 951 };
 952
 953 my $netdesc = {
 954     optional => 1,
 955     type => 'string', format => $net_fmt,
 956     description => "Specify network devices.",
 957 };
 958
 959 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 960
 961 my $ipconfig_fmt = {
 962     ip => {
 963         type => 'string',
 964         format => 'pve-ipv4-config',
 965         format_description => 'IPv4Format/CIDR',
 966         description => 'IPv4 address in CIDR format.',
 967         optional => 1,
 968         default => 'dhcp',
 969     },
 970     gw => {
 971         type => 'string',
 972         format => 'ipv4',
 973         format_description => 'GatewayIPv4',
 974         description => 'Default gateway for IPv4 traffic.',
 975         optional => 1,
 976         requires => 'ip',
 977     },
 978     ip6 => {
 979         type => 'string',
 980         format => 'pve-ipv6-config',
 981         format_description => 'IPv6Format/CIDR',
 982         description => 'IPv6 address in CIDR format.',
 983         optional => 1,
 984         default => 'dhcp',
 985     },
 986     gw6 => {
 987         type => 'string',
 988         format => 'ipv6',
 989         format_description => 'GatewayIPv6',
 990         description => 'Default gateway for IPv6 traffic.',
 991         optional => 1,
 992         requires => 'ip6',
 993     },
 994 };
 995 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 996 my $ipconfigdesc = {
 997     optional => 1,
 998     type => 'string', format => 'pve-qm-ipconfig',
 999     description => <<'EODESCR',
1000 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1001
1002 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1003
1004 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1005 gateway should be provided.
1006 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1007 cloud-init 19.4 or newer.
1008
1009 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1010 dhcp on IPv4.
1011 EODESCR
1012 };
1013 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1014
1015 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1016     $confdesc->{"net$i"} = $netdesc;
1017     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1018 }
1019
1020 foreach my $key (keys %$confdesc_cloudinit) {
1021     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1022 }
1023
1024 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1025 sub verify_volume_id_or_qm_path {
1026     my ($volid, $noerr) = @_;
1027
1028     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1029         return $volid;
1030     }
1031
1032     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1033     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1034     if ($@) {
1035         return if $noerr;
1036         die $@;
1037     }
1038     return $volid;
1039 }
1040
1041 my $usb_fmt = {
1042     host => {
1043         default_key => 1,
1044         type => 'string', format => 'pve-qm-usb-device',
1045         format_description => 'HOSTUSBDEVICE|spice',
1046         description => <<EODESCR,
1047 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1048
1049  'bus-port(.port)*' (decimal numbers) or
1050  'vendor_id:product_id' (hexadeciaml numbers) or
1051  'spice'
1052
1053 You can use the 'lsusb -t' command to list existing usb devices.
1054
1055 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1056 machines - use with special care.
1057
1058 The value 'spice' can be used to add a usb redirection devices for spice.
1059 EODESCR
1060     },
1061     usb3 => {
1062         optional => 1,
1063         type => 'boolean',
1064         description => "Specifies whether if given host option is a USB3 device or port.",
1065         default => 0,
1066     },
1067 };
1068
1069 my $usbdesc = {
1070     optional => 1,
1071     type => 'string', format => $usb_fmt,
1072     description => "Configure an USB device (n is 0 to 4).",
1073 };
1074 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1075
1076 my $serialdesc = {
1077         optional => 1,
1078         type => 'string',
1079         pattern => '(/dev/.+|socket)',
1080         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1081         verbose_description =>  <<EODESCR,
1082 Create a serial device inside the VM (n is 0 to 3), and pass through a
1083 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1084 host side (use 'qm terminal' to open a terminal connection).
1085
1086 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1087 use with special care.
1088
1089 CAUTION: Experimental! User reported problems with this option.
1090 EODESCR
1091 };
1092
1093 my $paralleldesc= {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1097         description =>  "Map host parallel devices (n is 0 to 2).",
1098         verbose_description =>  <<EODESCR,
1099 Map host parallel devices (n is 0 to 2).
1100
1101 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1102 machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1118 }
1119
1120 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1121     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1122 }
1123
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1125     $confdesc->{"usb$i"} = $usbdesc;
1126 }
1127
1128 my $boot_fmt = {
1129     legacy => {
1130         optional => 1,
1131         default_key => 1,
1132         type => 'string',
1133         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1134                      . " Deprecated, use 'order=' instead.",
1135         pattern => '[acdn]{1,4}',
1136         format_description => "[acdn]{1,4}",
1137
1138         # note: this is also the fallback if boot: is not given at all
1139         default => 'cdn',
1140     },
1141     order => {
1142         optional => 1,
1143         type => 'string',
1144         format => 'pve-qm-bootdev-list',
1145         format_description => "device[;device...]",
1146         description => <<EODESC,
1147 The guest will attempt to boot from devices in the order they appear here.
1148
1149 Disks, optical drives and passed-through storage USB devices will be directly
1150 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1151 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1152
1153 Note that only devices in this list will be marked as bootable and thus loaded
1154 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1155 (e.g. software-raid), you need to specify all of them here.
1156
1157 Overrides the deprecated 'legacy=[acdn]*' value when given.
1158 EODESC
1159     },
1160 };
1161 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1162
1163 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1164 sub verify_bootdev {
1165     my ($dev, $noerr) = @_;
1166
1167     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1168     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1169
1170     my $check = sub {
1171         my ($base) = @_;
1172         return 0 if $dev !~ m/^$base\d+$/;
1173         return 0 if !$confdesc->{$dev};
1174         return 1;
1175     };
1176
1177     return $dev if $check->("net");
1178     return $dev if $check->("usb");
1179     return $dev if $check->("hostpci");
1180
1181     return if $noerr;
1182     die "invalid boot device '$dev'\n";
1183 }
1184
1185 sub print_bootorder {
1186     my ($devs) = @_;
1187     return "" if !@$devs;
1188     my $data = { order => join(';', @$devs) };
1189     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1190 }
1191
1192 my $kvm_api_version = 0;
1193
1194 sub kvm_version {
1195     return $kvm_api_version if $kvm_api_version;
1196
1197     open my $fh, '<', '/dev/kvm' or return;
1198
1199     # 0xae00 => KVM_GET_API_VERSION
1200     $kvm_api_version = ioctl($fh, 0xae00, 0);
1201     close($fh);
1202
1203     return $kvm_api_version;
1204 }
1205
1206 my $kvm_user_version = {};
1207 my $kvm_mtime = {};
1208
1209 sub kvm_user_version {
1210     my ($binary) = @_;
1211
1212     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1213     my $st = stat($binary);
1214
1215     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1216     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1217         $cachedmtime == $st->mtime;
1218
1219     $kvm_user_version->{$binary} = 'unknown';
1220     $kvm_mtime->{$binary} = $st->mtime;
1221
1222     my $code = sub {
1223         my $line = shift;
1224         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1225             $kvm_user_version->{$binary} = $2;
1226         }
1227     };
1228
1229     eval { run_command([$binary, '--version'], outfunc => $code); };
1230     warn $@ if $@;
1231
1232     return $kvm_user_version->{$binary};
1233
1234 }
1235 my sub extract_version {
1236     my ($machine_type, $version) = @_;
1237     $version = kvm_user_version() if !defined($version);
1238     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1239 }
1240
1241 sub kernel_has_vhost_net {
1242     return -c '/dev/vhost-net';
1243 }
1244
1245 sub option_exists {
1246     my $key = shift;
1247     return defined($confdesc->{$key});
1248 }
1249
1250 my $cdrom_path;
1251 sub get_cdrom_path {
1252
1253     return  $cdrom_path if $cdrom_path;
1254
1255     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1256     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1257     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1258 }
1259
1260 sub get_iso_path {
1261     my ($storecfg, $vmid, $cdrom) = @_;
1262
1263     if ($cdrom eq 'cdrom') {
1264         return get_cdrom_path();
1265     } elsif ($cdrom eq 'none') {
1266         return '';
1267     } elsif ($cdrom =~ m|^/|) {
1268         return $cdrom;
1269     } else {
1270         return PVE::Storage::path($storecfg, $cdrom);
1271     }
1272 }
1273
1274 # try to convert old style file names to volume IDs
1275 sub filename_to_volume_id {
1276     my ($vmid, $file, $media) = @_;
1277
1278      if (!($file eq 'none' || $file eq 'cdrom' ||
1279           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1280
1281         return if $file =~ m|/|;
1282
1283         if ($media && $media eq 'cdrom') {
1284             $file = "local:iso/$file";
1285         } else {
1286             $file = "local:$vmid/$file";
1287         }
1288     }
1289
1290     return $file;
1291 }
1292
1293 sub verify_media_type {
1294     my ($opt, $vtype, $media) = @_;
1295
1296     return if !$media;
1297
1298     my $etype;
1299     if ($media eq 'disk') {
1300         $etype = 'images';
1301     } elsif ($media eq 'cdrom') {
1302         $etype = 'iso';
1303     } else {
1304         die "internal error";
1305     }
1306
1307     return if ($vtype eq $etype);
1308
1309     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1310 }
1311
1312 sub cleanup_drive_path {
1313     my ($opt, $storecfg, $drive) = @_;
1314
1315     # try to convert filesystem paths to volume IDs
1316
1317     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1318         ($drive->{file} !~ m|^/dev/.+|) &&
1319         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1320         ($drive->{file} !~ m/^\d+$/)) {
1321         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1322         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1323             if !$vtype;
1324         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1325         verify_media_type($opt, $vtype, $drive->{media});
1326         $drive->{file} = $volid;
1327     }
1328
1329     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1330 }
1331
1332 sub parse_hotplug_features {
1333     my ($data) = @_;
1334
1335     my $res = {};
1336
1337     return $res if $data eq '0';
1338
1339     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1340
1341     foreach my $feature (PVE::Tools::split_list($data)) {
1342         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1343             $res->{$1} = 1;
1344         } else {
1345             die "invalid hotplug feature '$feature'\n";
1346         }
1347     }
1348     return $res;
1349 }
1350
1351 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1352 sub pve_verify_hotplug_features {
1353     my ($value, $noerr) = @_;
1354
1355     return $value if parse_hotplug_features($value);
1356
1357     return if $noerr;
1358
1359     die "unable to parse hotplug option\n";
1360 }
1361
1362 sub scsi_inquiry {
1363     my($fh, $noerr) = @_;
1364
1365     my $SG_IO = 0x2285;
1366     my $SG_GET_VERSION_NUM = 0x2282;
1367
1368     my $versionbuf = "\x00" x 8;
1369     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1370     if (!$ret) {
1371         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1372         return;
1373     }
1374     my $version = unpack("I", $versionbuf);
1375     if ($version < 30000) {
1376         die "scsi generic interface too old\n"  if !$noerr;
1377         return;
1378     }
1379
1380     my $buf = "\x00" x 36;
1381     my $sensebuf = "\x00" x 8;
1382     my $cmd = pack("C x3 C x1", 0x12, 36);
1383
1384     # see /usr/include/scsi/sg.h
1385     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1386
1387     my $packet = pack(
1388         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1389     );
1390
1391     $ret = ioctl($fh, $SG_IO, $packet);
1392     if (!$ret) {
1393         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1394         return;
1395     }
1396
1397     my @res = unpack($sg_io_hdr_t, $packet);
1398     if ($res[17] || $res[18]) {
1399         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1400         return;
1401     }
1402
1403     my $res = {};
1404     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1405
1406     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1407     $res->{type} &= 0x1F;
1408
1409     return $res;
1410 }
1411
1412 sub path_is_scsi {
1413     my ($path) = @_;
1414
1415     my $fh = IO::File->new("+<$path") || return;
1416     my $res = scsi_inquiry($fh, 1);
1417     close($fh);
1418
1419     return $res;
1420 }
1421
1422 sub print_tabletdevice_full {
1423     my ($conf, $arch) = @_;
1424
1425     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1426
1427     # we use uhci for old VMs because tablet driver was buggy in older qemu
1428     my $usbbus;
1429     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1430         $usbbus = 'ehci';
1431     } else {
1432         $usbbus = 'uhci';
1433     }
1434
1435     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1436 }
1437
1438 sub print_keyboarddevice_full {
1439     my ($conf, $arch) = @_;
1440
1441     return if $arch ne 'aarch64';
1442
1443     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1444 }
1445
1446 my sub get_drive_id {
1447     my ($drive) = @_;
1448     return "$drive->{interface}$drive->{index}";
1449 }
1450
1451 sub print_drivedevice_full {
1452     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1453
1454     my $device = '';
1455     my $maxdev = 0;
1456
1457     my $drive_id = get_drive_id($drive);
1458     if ($drive->{interface} eq 'virtio') {
1459         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1460         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1461         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1462     } elsif ($drive->{interface} eq 'scsi') {
1463
1464         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1465         my $unit = $drive->{index} % $maxdev;
1466         my $devicetype = 'hd';
1467         my $path = '';
1468         if (drive_is_cdrom($drive)) {
1469             $devicetype = 'cd';
1470         } else {
1471             if ($drive->{file} =~ m|^/|) {
1472                 $path = $drive->{file};
1473                 if (my $info = path_is_scsi($path)) {
1474                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1475                         $devicetype = 'block';
1476                     } elsif ($info->{type} == 1) { # tape
1477                         $devicetype = 'generic';
1478                     }
1479                 }
1480             } else {
1481                  $path = PVE::Storage::path($storecfg, $drive->{file});
1482             }
1483
1484             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1485             my $version = extract_version($machine_type, kvm_user_version());
1486             if ($path =~ m/^iscsi\:\/\// &&
1487                !min_version($version, 4, 1)) {
1488                 $devicetype = 'generic';
1489             }
1490         }
1491
1492         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1493             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1494         } else {
1495             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1496                 .",lun=$drive->{index}";
1497         }
1498         $device .= ",drive=drive-$drive_id,id=$drive_id";
1499
1500         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1501             $device .= ",rotation_rate=1";
1502         }
1503         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1504
1505     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1506         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1507         my $controller = int($drive->{index} / $maxdev);
1508         my $unit = $drive->{index} % $maxdev;
1509         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1510
1511         $device = "ide-$devicetype";
1512         if ($drive->{interface} eq 'ide') {
1513             $device .= ",bus=ide.$controller,unit=$unit";
1514         } else {
1515             $device .= ",bus=ahci$controller.$unit";
1516         }
1517         $device .= ",drive=drive-$drive_id,id=$drive_id";
1518
1519         if ($devicetype eq 'hd') {
1520             if (my $model = $drive->{model}) {
1521                 $model = URI::Escape::uri_unescape($model);
1522                 $device .= ",model=$model";
1523             }
1524             if ($drive->{ssd}) {
1525                 $device .= ",rotation_rate=1";
1526             }
1527         }
1528         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1529     } elsif ($drive->{interface} eq 'usb') {
1530         die "implement me";
1531         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1532     } else {
1533         die "unsupported interface type";
1534     }
1535
1536     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1537
1538     if (my $serial = $drive->{serial}) {
1539         $serial = URI::Escape::uri_unescape($serial);
1540         $device .= ",serial=$serial";
1541     }
1542
1543
1544     return $device;
1545 }
1546
1547 sub get_initiator_name {
1548     my $initiator;
1549
1550     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1551     while (defined(my $line = <$fh>)) {
1552         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1553         $initiator = $1;
1554         last;
1555     }
1556     $fh->close();
1557
1558     return $initiator;
1559 }
1560
1561 sub print_drive_commandline_full {
1562     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1563
1564     my $path;
1565     my $volid = $drive->{file};
1566     my $format = $drive->{format};
1567     my $drive_id = get_drive_id($drive);
1568
1569     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1570     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1571
1572     if (drive_is_cdrom($drive)) {
1573         $path = get_iso_path($storecfg, $vmid, $volid);
1574         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1575     } else {
1576         if ($storeid) {
1577             $path = PVE::Storage::path($storecfg, $volid);
1578             $format //= qemu_img_format($scfg, $volname);
1579         } else {
1580             $path = $volid;
1581             $format //= "raw";
1582         }
1583    }
1584
1585    my $is_rbd = $path =~ m/^rbd:/;
1586
1587     my $opts = '';
1588     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1589     foreach my $o (@qemu_drive_options) {
1590         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1591     }
1592
1593     # snapshot only accepts on|off
1594     if (defined($drive->{snapshot})) {
1595         my $v = $drive->{snapshot} ? 'on' : 'off';
1596         $opts .= ",snapshot=$v";
1597     }
1598
1599     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1600         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1601     }
1602
1603     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1604         my ($dir, $qmpname) = @$type;
1605         if (my $v = $drive->{"mbps$dir"}) {
1606             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1607         }
1608         if (my $v = $drive->{"mbps${dir}_max"}) {
1609             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1610         }
1611         if (my $v = $drive->{"bps${dir}_max_length"}) {
1612             $opts .= ",throttling.bps$qmpname-max-length=$v";
1613         }
1614         if (my $v = $drive->{"iops${dir}"}) {
1615             $opts .= ",throttling.iops$qmpname=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}_max"}) {
1618             $opts .= ",throttling.iops$qmpname-max=$v";
1619         }
1620         if (my $v = $drive->{"iops${dir}_max_length"}) {
1621             $opts .= ",throttling.iops$qmpname-max-length=$v";
1622         }
1623     }
1624
1625     if ($pbs_name) {
1626         $format = "rbd" if $is_rbd;
1627         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1628             if !$format;
1629         $opts .= ",format=alloc-track,file.driver=$format";
1630     } elsif ($format) {
1631         $opts .= ",format=$format";
1632     }
1633
1634     my $cache_direct = 0;
1635
1636     if (my $cache = $drive->{cache}) {
1637         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1638     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1639         $opts .= ",cache=none";
1640         $cache_direct = 1;
1641     }
1642
1643     # io_uring with cache mode writeback or writethrough on krbd will hang...
1644     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1645
1646     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1647     # sometimes, just plain disable...
1648     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1649
1650     if (!$drive->{aio}) {
1651         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1652             # io_uring supports all cache modes
1653             $opts .= ",aio=io_uring";
1654         } else {
1655             # aio native works only with O_DIRECT
1656             if($cache_direct) {
1657                 $opts .= ",aio=native";
1658             } else {
1659                 $opts .= ",aio=threads";
1660             }
1661         }
1662     }
1663
1664     if (!drive_is_cdrom($drive)) {
1665         my $detectzeroes;
1666         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1667             $detectzeroes = 'off';
1668         } elsif ($drive->{discard}) {
1669             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1670         } else {
1671             # This used to be our default with discard not being specified:
1672             $detectzeroes = 'on';
1673         }
1674
1675         # note: 'detect-zeroes' works per blockdev and we want it to persist
1676         # after the alloc-track is removed, so put it on 'file' directly
1677         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1678         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1679     }
1680
1681     if ($pbs_name) {
1682         $opts .= ",backing=$pbs_name";
1683         $opts .= ",auto-remove=on";
1684     }
1685
1686     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1687     my $file_param = "file";
1688     if ($pbs_name) {
1689         # non-rbd drivers require the underlying file to be a seperate block
1690         # node, so add a second .file indirection
1691         $file_param .= ".file" if !$is_rbd;
1692         $file_param .= ".filename";
1693     }
1694     my $pathinfo = $path ? "$file_param=$path," : '';
1695
1696     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1697 }
1698
1699 sub print_pbs_blockdev {
1700     my ($pbs_conf, $pbs_name) = @_;
1701     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1702     $blockdev .= ",repository=$pbs_conf->{repository}";
1703     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1704     $blockdev .= ",archive=$pbs_conf->{archive}";
1705     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1706     return $blockdev;
1707 }
1708
1709 sub print_netdevice_full {
1710     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1711
1712     my $device = $net->{model};
1713     if ($net->{model} eq 'virtio') {
1714          $device = 'virtio-net-pci';
1715      };
1716
1717     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1718     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1719     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1720         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1721         # and out of each queue plus one config interrupt and control vector queue
1722         my $vectors = $net->{queues} * 2 + 2;
1723         $tmpstr .= ",vectors=$vectors,mq=on";
1724     }
1725     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1726
1727     if (my $mtu = $net->{mtu}) {
1728         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1729             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1730             if ($mtu == 1) {
1731                  $mtu = $bridge_mtu;
1732             } elsif ($mtu < 576) {
1733                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1734             } elsif ($mtu > $bridge_mtu) {
1735                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1736             }
1737             $tmpstr .= ",host_mtu=$mtu";
1738         } else {
1739             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1740         }
1741     }
1742
1743     if ($use_old_bios_files) {
1744         my $romfile;
1745         if ($device eq 'virtio-net-pci') {
1746             $romfile = 'pxe-virtio.rom';
1747         } elsif ($device eq 'e1000') {
1748             $romfile = 'pxe-e1000.rom';
1749         } elsif ($device eq 'e1000e') {
1750             $romfile = 'pxe-e1000e.rom';
1751         } elsif ($device eq 'ne2k') {
1752             $romfile = 'pxe-ne2k_pci.rom';
1753         } elsif ($device eq 'pcnet') {
1754             $romfile = 'pxe-pcnet.rom';
1755         } elsif ($device eq 'rtl8139') {
1756             $romfile = 'pxe-rtl8139.rom';
1757         }
1758         $tmpstr .= ",romfile=$romfile" if $romfile;
1759     }
1760
1761     return $tmpstr;
1762 }
1763
1764 sub print_netdev_full {
1765     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1766
1767     my $i = '';
1768     if ($netid =~ m/^net(\d+)$/) {
1769         $i = int($1);
1770     }
1771
1772     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1773
1774     my $ifname = "tap${vmid}i$i";
1775
1776     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1777     die "interface name '$ifname' is too long (max 15 character)\n"
1778         if length($ifname) >= 16;
1779
1780     my $vhostparam = '';
1781     if (is_native($arch)) {
1782         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1783     }
1784
1785     my $vmname = $conf->{name} || "vm$vmid";
1786
1787     my $netdev = "";
1788     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1789
1790     if ($net->{bridge}) {
1791         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1792             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1793     } else {
1794         $netdev = "type=user,id=$netid,hostname=$vmname";
1795     }
1796
1797     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1798
1799     return $netdev;
1800 }
1801
1802 my $vga_map = {
1803     'cirrus' => 'cirrus-vga',
1804     'std' => 'VGA',
1805     'vmware' => 'vmware-svga',
1806     'virtio' => 'virtio-vga',
1807 };
1808
1809 sub print_vga_device {
1810     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1811
1812     my $type = $vga_map->{$vga->{type}};
1813     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1814         $type = 'virtio-gpu';
1815     }
1816     my $vgamem_mb = $vga->{memory};
1817
1818     my $max_outputs = '';
1819     if ($qxlnum) {
1820         $type = $id ? 'qxl' : 'qxl-vga';
1821
1822         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1823             # set max outputs so linux can have up to 4 qxl displays with one device
1824             if (min_version($machine_version, 4, 1)) {
1825                 $max_outputs = ",max_outputs=4";
1826             }
1827         }
1828     }
1829
1830     die "no devicetype for $vga->{type}\n" if !$type;
1831
1832     my $memory = "";
1833     if ($vgamem_mb) {
1834         if ($vga->{type} eq 'virtio') {
1835             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1836             $memory = ",max_hostmem=$bytes";
1837         } elsif ($qxlnum) {
1838             # from https://www.spice-space.org/multiple-monitors.html
1839             $memory = ",vgamem_mb=$vga->{memory}";
1840             my $ram = $vgamem_mb * 4;
1841             my $vram = $vgamem_mb * 2;
1842             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1843         } else {
1844             $memory = ",vgamem_mb=$vga->{memory}";
1845         }
1846     } elsif ($qxlnum && $id) {
1847         $memory = ",ram_size=67108864,vram_size=33554432";
1848     }
1849
1850     my $edidoff = "";
1851     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1852         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1853     }
1854
1855     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1856     my $vgaid = "vga" . ($id // '');
1857     my $pciaddr;
1858     if ($q35 && $vgaid eq 'vga') {
1859         # the first display uses pcie.0 bus on q35 machines
1860         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1861     } else {
1862         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1863     }
1864
1865     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1866 }
1867
1868 sub parse_number_sets {
1869     my ($set) = @_;
1870     my $res = [];
1871     foreach my $part (split(/;/, $set)) {
1872         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1873             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1874             push @$res, [ $1, $2 ];
1875         } else {
1876             die "invalid range: $part\n";
1877         }
1878     }
1879     return $res;
1880 }
1881
1882 sub parse_numa {
1883     my ($data) = @_;
1884
1885     my $res = parse_property_string($numa_fmt, $data);
1886     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1887     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1888     return $res;
1889 }
1890
1891 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1892 sub parse_net {
1893     my ($data) = @_;
1894
1895     my $res = eval { parse_property_string($net_fmt, $data) };
1896     if ($@) {
1897         warn $@;
1898         return;
1899     }
1900     if (!defined($res->{macaddr})) {
1901         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1902         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1903     }
1904     return $res;
1905 }
1906
1907 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1908 sub parse_ipconfig {
1909     my ($data) = @_;
1910
1911     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1912     if ($@) {
1913         warn $@;
1914         return;
1915     }
1916
1917     if ($res->{gw} && !$res->{ip}) {
1918         warn 'gateway specified without specifying an IP address';
1919         return;
1920     }
1921     if ($res->{gw6} && !$res->{ip6}) {
1922         warn 'IPv6 gateway specified without specifying an IPv6 address';
1923         return;
1924     }
1925     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1926         warn 'gateway specified together with DHCP';
1927         return;
1928     }
1929     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1930         # gw6 + auto/dhcp
1931         warn "IPv6 gateway specified together with $res->{ip6} address";
1932         return;
1933     }
1934
1935     if (!$res->{ip} && !$res->{ip6}) {
1936         return { ip => 'dhcp', ip6 => 'dhcp' };
1937     }
1938
1939     return $res;
1940 }
1941
1942 sub print_net {
1943     my $net = shift;
1944
1945     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1946 }
1947
1948 sub add_random_macs {
1949     my ($settings) = @_;
1950
1951     foreach my $opt (keys %$settings) {
1952         next if $opt !~ m/^net(\d+)$/;
1953         my $net = parse_net($settings->{$opt});
1954         next if !$net;
1955         $settings->{$opt} = print_net($net);
1956     }
1957 }
1958
1959 sub vm_is_volid_owner {
1960     my ($storecfg, $vmid, $volid) = @_;
1961
1962     if ($volid !~  m|^/|) {
1963         my ($path, $owner);
1964         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1965         if ($owner && ($owner == $vmid)) {
1966             return 1;
1967         }
1968     }
1969
1970     return;
1971 }
1972
1973 sub vmconfig_register_unused_drive {
1974     my ($storecfg, $vmid, $conf, $drive) = @_;
1975
1976     if (drive_is_cloudinit($drive)) {
1977         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1978         warn $@ if $@;
1979     } elsif (!drive_is_cdrom($drive)) {
1980         my $volid = $drive->{file};
1981         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1982             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1983         }
1984     }
1985 }
1986
1987 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1988 my $smbios1_fmt = {
1989     uuid => {
1990         type => 'string',
1991         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1992         format_description => 'UUID',
1993         description => "Set SMBIOS1 UUID.",
1994         optional => 1,
1995     },
1996     version => {
1997         type => 'string',
1998         pattern => '[A-Za-z0-9+\/]+={0,2}',
1999         format_description => 'Base64 encoded string',
2000         description => "Set SMBIOS1 version.",
2001         optional => 1,
2002     },
2003     serial => {
2004         type => 'string',
2005         pattern => '[A-Za-z0-9+\/]+={0,2}',
2006         format_description => 'Base64 encoded string',
2007         description => "Set SMBIOS1 serial number.",
2008         optional => 1,
2009     },
2010     manufacturer => {
2011         type => 'string',
2012         pattern => '[A-Za-z0-9+\/]+={0,2}',
2013         format_description => 'Base64 encoded string',
2014         description => "Set SMBIOS1 manufacturer.",
2015         optional => 1,
2016     },
2017     product => {
2018         type => 'string',
2019         pattern => '[A-Za-z0-9+\/]+={0,2}',
2020         format_description => 'Base64 encoded string',
2021         description => "Set SMBIOS1 product ID.",
2022         optional => 1,
2023     },
2024     sku => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 SKU string.",
2029         optional => 1,
2030     },
2031     family => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 family string.",
2036         optional => 1,
2037     },
2038     base64 => {
2039         type => 'boolean',
2040         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2041         optional => 1,
2042     },
2043 };
2044
2045 sub parse_smbios1 {
2046     my ($data) = @_;
2047
2048     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2049     warn $@ if $@;
2050     return $res;
2051 }
2052
2053 sub print_smbios1 {
2054     my ($smbios1) = @_;
2055     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2056 }
2057
2058 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2059
2060 sub parse_watchdog {
2061     my ($value) = @_;
2062
2063     return if !$value;
2064
2065     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2066     warn $@ if $@;
2067     return $res;
2068 }
2069
2070 sub parse_guest_agent {
2071     my ($conf) = @_;
2072
2073     return {} if !defined($conf->{agent});
2074
2075     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2076     warn $@ if $@;
2077
2078     # if the agent is disabled ignore the other potentially set properties
2079     return {} if !$res->{enabled};
2080     return $res;
2081 }
2082
2083 sub get_qga_key {
2084     my ($conf, $key) = @_;
2085     return undef if !defined($conf->{agent});
2086
2087     my $agent = parse_guest_agent($conf);
2088     return $agent->{$key};
2089 }
2090
2091 sub parse_vga {
2092     my ($value) = @_;
2093
2094     return {} if !$value;
2095     my $res = eval { parse_property_string($vga_fmt, $value) };
2096     warn $@ if $@;
2097     return $res;
2098 }
2099
2100 sub parse_rng {
2101     my ($value) = @_;
2102
2103     return if !$value;
2104
2105     my $res = eval { parse_property_string($rng_fmt, $value) };
2106     warn $@ if $@;
2107     return $res;
2108 }
2109
2110 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2111 sub verify_usb_device {
2112     my ($value, $noerr) = @_;
2113
2114     return $value if parse_usb_device($value);
2115
2116     return if $noerr;
2117
2118     die "unable to parse usb device\n";
2119 }
2120
2121 # add JSON properties for create and set function
2122 sub json_config_properties {
2123     my $prop = shift;
2124
2125     my $skip_json_config_opts = {
2126         parent => 1,
2127         snaptime => 1,
2128         vmstate => 1,
2129         runningmachine => 1,
2130         runningcpu => 1,
2131     };
2132
2133     foreach my $opt (keys %$confdesc) {
2134         next if $skip_json_config_opts->{$opt};
2135         $prop->{$opt} = $confdesc->{$opt};
2136     }
2137
2138     return $prop;
2139 }
2140
2141 # return copy of $confdesc_cloudinit to generate documentation
2142 sub cloudinit_config_properties {
2143
2144     return dclone($confdesc_cloudinit);
2145 }
2146
2147 sub check_type {
2148     my ($key, $value) = @_;
2149
2150     die "unknown setting '$key'\n" if !$confdesc->{$key};
2151
2152     my $type = $confdesc->{$key}->{type};
2153
2154     if (!defined($value)) {
2155         die "got undefined value\n";
2156     }
2157
2158     if ($value =~ m/[\n\r]/) {
2159         die "property contains a line feed\n";
2160     }
2161
2162     if ($type eq 'boolean') {
2163         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2164         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2165         die "type check ('boolean') failed - got '$value'\n";
2166     } elsif ($type eq 'integer') {
2167         return int($1) if $value =~ m/^(\d+)$/;
2168         die "type check ('integer') failed - got '$value'\n";
2169     } elsif ($type eq 'number') {
2170         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2171         die "type check ('number') failed - got '$value'\n";
2172     } elsif ($type eq 'string') {
2173         if (my $fmt = $confdesc->{$key}->{format}) {
2174             PVE::JSONSchema::check_format($fmt, $value);
2175             return $value;
2176         }
2177         $value =~ s/^\"(.*)\"$/$1/;
2178         return $value;
2179     } else {
2180         die "internal error"
2181     }
2182 }
2183
2184 sub destroy_vm {
2185     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2186
2187     my $conf = PVE::QemuConfig->load_config($vmid);
2188
2189     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2190
2191     if ($conf->{template}) {
2192         # check if any base image is still used by a linked clone
2193         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2194                 my ($ds, $drive) = @_;
2195                 return if drive_is_cdrom($drive);
2196
2197                 my $volid = $drive->{file};
2198                 return if !$volid || $volid =~ m|^/|;
2199
2200                 die "base volume '$volid' is still in use by linked cloned\n"
2201                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2202
2203         });
2204     }
2205
2206     my $volids = {};
2207     my $remove_owned_drive = sub {
2208         my ($ds, $drive) = @_;
2209         return if drive_is_cdrom($drive, 1);
2210
2211         my $volid = $drive->{file};
2212         return if !$volid || $volid =~ m|^/|;
2213         return if $volids->{$volid};
2214
2215         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2216         return if !$path || !$owner || ($owner != $vmid);
2217
2218         $volids->{$volid} = 1;
2219         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2220         warn "Could not remove disk '$volid', check manually: $@" if $@;
2221     };
2222
2223     # only remove disks owned by this VM (referenced in the config)
2224     my $include_opts = {
2225         include_unused => 1,
2226         extra_keys => ['vmstate'],
2227     };
2228     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2229
2230     for my $snap (values %{$conf->{snapshots}}) {
2231         next if !defined($snap->{vmstate});
2232         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2233         next if !defined($drive);
2234         $remove_owned_drive->('vmstate', $drive);
2235     }
2236
2237     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2238
2239     if ($purge_unreferenced) { # also remove unreferenced disk
2240         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2241         PVE::Storage::foreach_volid($vmdisks, sub {
2242             my ($volid, $sid, $volname, $d) = @_;
2243             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2244             warn $@ if $@;
2245         });
2246     }
2247
2248     if (defined $replacement_conf) {
2249         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2250     } else {
2251         PVE::QemuConfig->destroy_config($vmid);
2252     }
2253 }
2254
2255 sub parse_vm_config {
2256     my ($filename, $raw) = @_;
2257
2258     return if !defined($raw);
2259
2260     my $res = {
2261         digest => Digest::SHA::sha1_hex($raw),
2262         snapshots => {},
2263         pending => {},
2264     };
2265
2266     $filename =~ m|/qemu-server/(\d+)\.conf$|
2267         || die "got strange filename '$filename'";
2268
2269     my $vmid = $1;
2270
2271     my $conf = $res;
2272     my $descr;
2273     my $section = '';
2274
2275     my @lines = split(/\n/, $raw);
2276     foreach my $line (@lines) {
2277         next if $line =~ m/^\s*$/;
2278
2279         if ($line =~ m/^\[PENDING\]\s*$/i) {
2280             $section = 'pending';
2281             if (defined($descr)) {
2282                 $descr =~ s/\s+$//;
2283                 $conf->{description} = $descr;
2284             }
2285             $descr = undef;
2286             $conf = $res->{$section} = {};
2287             next;
2288
2289         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2290             $section = $1;
2291             if (defined($descr)) {
2292                 $descr =~ s/\s+$//;
2293                 $conf->{description} = $descr;
2294             }
2295             $descr = undef;
2296             $conf = $res->{snapshots}->{$section} = {};
2297             next;
2298         }
2299
2300         if ($line =~ m/^\#(.*)\s*$/) {
2301             $descr = '' if !defined($descr);
2302             $descr .= PVE::Tools::decode_text($1) . "\n";
2303             next;
2304         }
2305
2306         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2307             $descr = '' if !defined($descr);
2308             $descr .= PVE::Tools::decode_text($2);
2309         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2310             $conf->{snapstate} = $1;
2311         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2312             my $key = $1;
2313             my $value = $2;
2314             $conf->{$key} = $value;
2315         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2316             my $value = $1;
2317             if ($section eq 'pending') {
2318                 $conf->{delete} = $value; # we parse this later
2319             } else {
2320                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2321             }
2322         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2323             my $key = $1;
2324             my $value = $2;
2325             eval { $value = check_type($key, $value); };
2326             if ($@) {
2327                 warn "vm $vmid - unable to parse value of '$key' - $@";
2328             } else {
2329                 $key = 'ide2' if $key eq 'cdrom';
2330                 my $fmt = $confdesc->{$key}->{format};
2331                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2332                     my $v = parse_drive($key, $value);
2333                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2334                         $v->{file} = $volid;
2335                         $value = print_drive($v);
2336                     } else {
2337                         warn "vm $vmid - unable to parse value of '$key'\n";
2338                         next;
2339                     }
2340                 }
2341
2342                 $conf->{$key} = $value;
2343             }
2344         } else {
2345             warn "vm $vmid - unable to parse config: $line\n";
2346         }
2347     }
2348
2349     if (defined($descr)) {
2350         $descr =~ s/\s+$//;
2351         $conf->{description} = $descr;
2352     }
2353     delete $res->{snapstate}; # just to be sure
2354
2355     return $res;
2356 }
2357
2358 sub write_vm_config {
2359     my ($filename, $conf) = @_;
2360
2361     delete $conf->{snapstate}; # just to be sure
2362
2363     if ($conf->{cdrom}) {
2364         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2365         $conf->{ide2} = $conf->{cdrom};
2366         delete $conf->{cdrom};
2367     }
2368
2369     # we do not use 'smp' any longer
2370     if ($conf->{sockets}) {
2371         delete $conf->{smp};
2372     } elsif ($conf->{smp}) {
2373         $conf->{sockets} = $conf->{smp};
2374         delete $conf->{cores};
2375         delete $conf->{smp};
2376     }
2377
2378     my $used_volids = {};
2379
2380     my $cleanup_config = sub {
2381         my ($cref, $pending, $snapname) = @_;
2382
2383         foreach my $key (keys %$cref) {
2384             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2385                 $key eq 'snapstate' || $key eq 'pending';
2386             my $value = $cref->{$key};
2387             if ($key eq 'delete') {
2388                 die "propertry 'delete' is only allowed in [PENDING]\n"
2389                     if !$pending;
2390                 # fixme: check syntax?
2391                 next;
2392             }
2393             eval { $value = check_type($key, $value); };
2394             die "unable to parse value of '$key' - $@" if $@;
2395
2396             $cref->{$key} = $value;
2397
2398             if (!$snapname && is_valid_drivename($key)) {
2399                 my $drive = parse_drive($key, $value);
2400                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2401             }
2402         }
2403     };
2404
2405     &$cleanup_config($conf);
2406
2407     &$cleanup_config($conf->{pending}, 1);
2408
2409     foreach my $snapname (keys %{$conf->{snapshots}}) {
2410         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2411         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2412     }
2413
2414     # remove 'unusedX' settings if we re-add a volume
2415     foreach my $key (keys %$conf) {
2416         my $value = $conf->{$key};
2417         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2418             delete $conf->{$key};
2419         }
2420     }
2421
2422     my $generate_raw_config = sub {
2423         my ($conf, $pending) = @_;
2424
2425         my $raw = '';
2426
2427         # add description as comment to top of file
2428         if (defined(my $descr = $conf->{description})) {
2429             if ($descr) {
2430                 foreach my $cl (split(/\n/, $descr)) {
2431                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2432                 }
2433             } else {
2434                 $raw .= "#\n" if $pending;
2435             }
2436         }
2437
2438         foreach my $key (sort keys %$conf) {
2439             next if $key =~ /^(digest|description|pending|snapshots)$/;
2440             $raw .= "$key: $conf->{$key}\n";
2441         }
2442         return $raw;
2443     };
2444
2445     my $raw = &$generate_raw_config($conf);
2446
2447     if (scalar(keys %{$conf->{pending}})){
2448         $raw .= "\n[PENDING]\n";
2449         $raw .= &$generate_raw_config($conf->{pending}, 1);
2450     }
2451
2452     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2453         $raw .= "\n[$snapname]\n";
2454         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2455     }
2456
2457     return $raw;
2458 }
2459
2460 sub load_defaults {
2461
2462     my $res = {};
2463
2464     # we use static defaults from our JSON schema configuration
2465     foreach my $key (keys %$confdesc) {
2466         if (defined(my $default = $confdesc->{$key}->{default})) {
2467             $res->{$key} = $default;
2468         }
2469     }
2470
2471     return $res;
2472 }
2473
2474 sub config_list {
2475     my $vmlist = PVE::Cluster::get_vmlist();
2476     my $res = {};
2477     return $res if !$vmlist || !$vmlist->{ids};
2478     my $ids = $vmlist->{ids};
2479     my $nodename = nodename();
2480
2481     foreach my $vmid (keys %$ids) {
2482         my $d = $ids->{$vmid};
2483         next if !$d->{node} || $d->{node} ne $nodename;
2484         next if !$d->{type} || $d->{type} ne 'qemu';
2485         $res->{$vmid}->{exists} = 1;
2486     }
2487     return $res;
2488 }
2489
2490 # test if VM uses local resources (to prevent migration)
2491 sub check_local_resources {
2492     my ($conf, $noerr) = @_;
2493
2494     my @loc_res = ();
2495
2496     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2497     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2498
2499     push @loc_res, "ivshmem" if $conf->{ivshmem};
2500
2501     foreach my $k (keys %$conf) {
2502         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2503         # sockets are safe: they will recreated be on the target side post-migrate
2504         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2505         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2506     }
2507
2508     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2509
2510     return \@loc_res;
2511 }
2512
2513 # check if used storages are available on all nodes (use by migrate)
2514 sub check_storage_availability {
2515     my ($storecfg, $conf, $node) = @_;
2516
2517     PVE::QemuConfig->foreach_volume($conf, sub {
2518         my ($ds, $drive) = @_;
2519
2520         my $volid = $drive->{file};
2521         return if !$volid;
2522
2523         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2524         return if !$sid;
2525
2526         # check if storage is available on both nodes
2527         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2528         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2529
2530         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2531
2532         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2533             if !$scfg->{content}->{$vtype};
2534    });
2535 }
2536
2537 # list nodes where all VM images are available (used by has_feature API)
2538 sub shared_nodes {
2539     my ($conf, $storecfg) = @_;
2540
2541     my $nodelist = PVE::Cluster::get_nodelist();
2542     my $nodehash = { map { $_ => 1 } @$nodelist };
2543     my $nodename = nodename();
2544
2545     PVE::QemuConfig->foreach_volume($conf, sub {
2546         my ($ds, $drive) = @_;
2547
2548         my $volid = $drive->{file};
2549         return if !$volid;
2550
2551         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2552         if ($storeid) {
2553             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2554             if ($scfg->{disable}) {
2555                 $nodehash = {};
2556             } elsif (my $avail = $scfg->{nodes}) {
2557                 foreach my $node (keys %$nodehash) {
2558                     delete $nodehash->{$node} if !$avail->{$node};
2559                 }
2560             } elsif (!$scfg->{shared}) {
2561                 foreach my $node (keys %$nodehash) {
2562                     delete $nodehash->{$node} if $node ne $nodename
2563                 }
2564             }
2565         }
2566     });
2567
2568     return $nodehash
2569 }
2570
2571 sub check_local_storage_availability {
2572     my ($conf, $storecfg) = @_;
2573
2574     my $nodelist = PVE::Cluster::get_nodelist();
2575     my $nodehash = { map { $_ => {} } @$nodelist };
2576
2577     PVE::QemuConfig->foreach_volume($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         my $volid = $drive->{file};
2581         return if !$volid;
2582
2583         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2584         if ($storeid) {
2585             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2586
2587             if ($scfg->{disable}) {
2588                 foreach my $node (keys %$nodehash) {
2589                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2590                 }
2591             } elsif (my $avail = $scfg->{nodes}) {
2592                 foreach my $node (keys %$nodehash) {
2593                     if (!$avail->{$node}) {
2594                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2595                     }
2596                 }
2597             }
2598         }
2599     });
2600
2601     foreach my $node (values %$nodehash) {
2602         if (my $unavail = $node->{unavailable_storages}) {
2603             $node->{unavailable_storages} = [ sort keys %$unavail ];
2604         }
2605     }
2606
2607     return $nodehash
2608 }
2609
2610 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2611 sub check_running {
2612     my ($vmid, $nocheck, $node) = @_;
2613
2614     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2615     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2616 }
2617
2618 sub vzlist {
2619
2620     my $vzlist = config_list();
2621
2622     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2623
2624     while (defined(my $de = $fd->read)) {
2625         next if $de !~ m/^(\d+)\.pid$/;
2626         my $vmid = $1;
2627         next if !defined($vzlist->{$vmid});
2628         if (my $pid = check_running($vmid)) {
2629             $vzlist->{$vmid}->{pid} = $pid;
2630         }
2631     }
2632
2633     return $vzlist;
2634 }
2635
2636 our $vmstatus_return_properties = {
2637     vmid => get_standard_option('pve-vmid'),
2638     status => {
2639         description => "Qemu process status.",
2640         type => 'string',
2641         enum => ['stopped', 'running'],
2642     },
2643     maxmem => {
2644         description => "Maximum memory in bytes.",
2645         type => 'integer',
2646         optional => 1,
2647         renderer => 'bytes',
2648     },
2649     maxdisk => {
2650         description => "Root disk size in bytes.",
2651         type => 'integer',
2652         optional => 1,
2653         renderer => 'bytes',
2654     },
2655     name => {
2656         description => "VM name.",
2657         type => 'string',
2658         optional => 1,
2659     },
2660     qmpstatus => {
2661         description => "Qemu QMP agent status.",
2662         type => 'string',
2663         optional => 1,
2664     },
2665     pid => {
2666         description => "PID of running qemu process.",
2667         type => 'integer',
2668         optional => 1,
2669     },
2670     uptime => {
2671         description => "Uptime.",
2672         type => 'integer',
2673         optional => 1,
2674         renderer => 'duration',
2675     },
2676     cpus => {
2677         description => "Maximum usable CPUs.",
2678         type => 'number',
2679         optional => 1,
2680     },
2681     lock => {
2682         description => "The current config lock, if any.",
2683         type => 'string',
2684         optional => 1,
2685     },
2686     tags => {
2687         description  => "The current configured tags, if any",
2688         type => 'string',
2689         optional => 1,
2690     },
2691     'running-machine' => {
2692         description => "The currently running machine type (if running).",
2693         type => 'string',
2694         optional => 1,
2695     },
2696     'running-qemu' => {
2697         description => "The currently running QEMU version (if running).",
2698         type => 'string',
2699         optional => 1,
2700     },
2701 };
2702
2703 my $last_proc_pid_stat;
2704
2705 # get VM status information
2706 # This must be fast and should not block ($full == false)
2707 # We only query KVM using QMP if $full == true (this can be slow)
2708 sub vmstatus {
2709     my ($opt_vmid, $full) = @_;
2710
2711     my $res = {};
2712
2713     my $storecfg = PVE::Storage::config();
2714
2715     my $list = vzlist();
2716     my $defaults = load_defaults();
2717
2718     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2719
2720     my $cpucount = $cpuinfo->{cpus} || 1;
2721
2722     foreach my $vmid (keys %$list) {
2723         next if $opt_vmid && ($vmid ne $opt_vmid);
2724
2725         my $conf = PVE::QemuConfig->load_config($vmid);
2726
2727         my $d = { vmid => int($vmid) };
2728         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2729
2730         # fixme: better status?
2731         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2732
2733         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2734         if (defined($size)) {
2735             $d->{disk} = 0; # no info available
2736             $d->{maxdisk} = $size;
2737         } else {
2738             $d->{disk} = 0;
2739             $d->{maxdisk} = 0;
2740         }
2741
2742         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2743             * ($conf->{cores} || $defaults->{cores});
2744         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2745         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2746
2747         $d->{name} = $conf->{name} || "VM $vmid";
2748         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2749             : $defaults->{memory}*(1024*1024);
2750
2751         if ($conf->{balloon}) {
2752             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2753             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2754                 : $defaults->{shares};
2755         }
2756
2757         $d->{uptime} = 0;
2758         $d->{cpu} = 0;
2759         $d->{mem} = 0;
2760
2761         $d->{netout} = 0;
2762         $d->{netin} = 0;
2763
2764         $d->{diskread} = 0;
2765         $d->{diskwrite} = 0;
2766
2767         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2768
2769         $d->{serial} = 1 if conf_has_serial($conf);
2770         $d->{lock} = $conf->{lock} if $conf->{lock};
2771         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2772
2773         $res->{$vmid} = $d;
2774     }
2775
2776     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2777     foreach my $dev (keys %$netdev) {
2778         next if $dev !~ m/^tap([1-9]\d*)i/;
2779         my $vmid = $1;
2780         my $d = $res->{$vmid};
2781         next if !$d;
2782
2783         $d->{netout} += $netdev->{$dev}->{receive};
2784         $d->{netin} += $netdev->{$dev}->{transmit};
2785
2786         if ($full) {
2787             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2788             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2789         }
2790
2791     }
2792
2793     my $ctime = gettimeofday;
2794
2795     foreach my $vmid (keys %$list) {
2796
2797         my $d = $res->{$vmid};
2798         my $pid = $d->{pid};
2799         next if !$pid;
2800
2801         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2802         next if !$pstat; # not running
2803
2804         my $used = $pstat->{utime} + $pstat->{stime};
2805
2806         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2807
2808         if ($pstat->{vsize}) {
2809             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2810         }
2811
2812         my $old = $last_proc_pid_stat->{$pid};
2813         if (!$old) {
2814             $last_proc_pid_stat->{$pid} = {
2815                 time => $ctime,
2816                 used => $used,
2817                 cpu => 0,
2818             };
2819             next;
2820         }
2821
2822         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2823
2824         if ($dtime > 1000) {
2825             my $dutime = $used -  $old->{used};
2826
2827             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2828             $last_proc_pid_stat->{$pid} = {
2829                 time => $ctime,
2830                 used => $used,
2831                 cpu => $d->{cpu},
2832             };
2833         } else {
2834             $d->{cpu} = $old->{cpu};
2835         }
2836     }
2837
2838     return $res if !$full;
2839
2840     my $qmpclient = PVE::QMPClient->new();
2841
2842     my $ballooncb = sub {
2843         my ($vmid, $resp) = @_;
2844
2845         my $info = $resp->{'return'};
2846         return if !$info->{max_mem};
2847
2848         my $d = $res->{$vmid};
2849
2850         # use memory assigned to VM
2851         $d->{maxmem} = $info->{max_mem};
2852         $d->{balloon} = $info->{actual};
2853
2854         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2855             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2856             $d->{freemem} = $info->{free_mem};
2857         }
2858
2859         $d->{ballooninfo} = $info;
2860     };
2861
2862     my $blockstatscb = sub {
2863         my ($vmid, $resp) = @_;
2864         my $data = $resp->{'return'} || [];
2865         my $totalrdbytes = 0;
2866         my $totalwrbytes = 0;
2867
2868         for my $blockstat (@$data) {
2869             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2870             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2871
2872             $blockstat->{device} =~ s/drive-//;
2873             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2874         }
2875         $res->{$vmid}->{diskread} = $totalrdbytes;
2876         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2877     };
2878
2879     my $machinecb = sub {
2880         my ($vmid, $resp) = @_;
2881         my $data = $resp->{'return'} || [];
2882
2883         $res->{$vmid}->{'running-machine'} =
2884             PVE::QemuServer::Machine::current_from_query_machines($data);
2885     };
2886
2887     my $versioncb = sub {
2888         my ($vmid, $resp) = @_;
2889         my $data = $resp->{'return'} // {};
2890         my $version = 'unknown';
2891
2892         if (my $v = $data->{qemu}) {
2893             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2894         }
2895
2896         $res->{$vmid}->{'running-qemu'} = $version;
2897     };
2898
2899     my $statuscb = sub {
2900         my ($vmid, $resp) = @_;
2901
2902         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2903         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2904         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2905         # this fails if ballon driver is not loaded, so this must be
2906         # the last commnand (following command are aborted if this fails).
2907         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2908
2909         my $status = 'unknown';
2910         if (!defined($status = $resp->{'return'}->{status})) {
2911             warn "unable to get VM status\n";
2912             return;
2913         }
2914
2915         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2916     };
2917
2918     foreach my $vmid (keys %$list) {
2919         next if $opt_vmid && ($vmid ne $opt_vmid);
2920         next if !$res->{$vmid}->{pid}; # not running
2921         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2922     }
2923
2924     $qmpclient->queue_execute(undef, 2);
2925
2926     foreach my $vmid (keys %$list) {
2927         next if $opt_vmid && ($vmid ne $opt_vmid);
2928         next if !$res->{$vmid}->{pid}; #not running
2929
2930         # we can't use the $qmpclient since it might have already aborted on
2931         # 'query-balloon', but this might also fail for older versions...
2932         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2933         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2934     }
2935
2936     foreach my $vmid (keys %$list) {
2937         next if $opt_vmid && ($vmid ne $opt_vmid);
2938         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2939     }
2940
2941     return $res;
2942 }
2943
2944 sub conf_has_serial {
2945     my ($conf) = @_;
2946
2947     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2948         if ($conf->{"serial$i"}) {
2949             return 1;
2950         }
2951     }
2952
2953     return 0;
2954 }
2955
2956 sub conf_has_audio {
2957     my ($conf, $id) = @_;
2958
2959     $id //= 0;
2960     my $audio = $conf->{"audio$id"};
2961     return if !defined($audio);
2962
2963     my $audioproperties = parse_property_string($audio_fmt, $audio);
2964     my $audiodriver = $audioproperties->{driver} // 'spice';
2965
2966     return {
2967         dev => $audioproperties->{device},
2968         dev_id => "audiodev$id",
2969         backend => $audiodriver,
2970         backend_id => "$audiodriver-backend${id}",
2971     };
2972 }
2973
2974 sub audio_devs {
2975     my ($audio, $audiopciaddr, $machine_version) = @_;
2976
2977     my $devs = [];
2978
2979     my $id = $audio->{dev_id};
2980     my $audiodev = "";
2981     if (min_version($machine_version, 4, 2)) {
2982         $audiodev = ",audiodev=$audio->{backend_id}";
2983     }
2984
2985     if ($audio->{dev} eq 'AC97') {
2986         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2987     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2988         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2989         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2990         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2991     } else {
2992         die "unkown audio device '$audio->{dev}', implement me!";
2993     }
2994
2995     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2996
2997     return $devs;
2998 }
2999
3000 sub get_tpm_paths {
3001     my ($vmid) = @_;
3002     return {
3003         socket => "/var/run/qemu-server/$vmid.swtpm",
3004         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3005     };
3006 }
3007
3008 sub add_tpm_device {
3009     my ($vmid, $devices, $conf) = @_;
3010
3011     return if !$conf->{tpmstate0};
3012
3013     my $paths = get_tpm_paths($vmid);
3014
3015     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3016     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3017     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3018 }
3019
3020 sub start_swtpm {
3021     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3022
3023     return if !$tpmdrive;
3024
3025     my $state;
3026     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3027     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3028     if ($storeid) {
3029         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3030     } else {
3031         $state = $tpm->{file};
3032     }
3033
3034     my $paths = get_tpm_paths($vmid);
3035
3036     # during migration, we will get state from remote
3037     #
3038     if (!$migration) {
3039         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3040         my $setup_cmd = [
3041             "swtpm_setup",
3042             "--tpmstate",
3043             "file://$state",
3044             "--createek",
3045             "--create-ek-cert",
3046             "--create-platform-cert",
3047             "--lock-nvram",
3048             "--config",
3049             "/etc/swtpm_setup.conf", # do not use XDG configs
3050             "--runas",
3051             "0", # force creation as root, error if not possible
3052             "--not-overwrite", # ignore existing state, do not modify
3053         ];
3054
3055         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3056         # TPM 2.0 supports ECC crypto, use if possible
3057         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3058
3059         run_command($setup_cmd, outfunc => sub {
3060             print "swtpm_setup: $1\n";
3061         });
3062     }
3063
3064     my $emulator_cmd = [
3065         "swtpm",
3066         "socket",
3067         "--tpmstate",
3068         "backend-uri=file://$state,mode=0600",
3069         "--ctrl",
3070         "type=unixio,path=$paths->{socket},mode=0600",
3071         "--pid",
3072         "file=$paths->{pid}",
3073         "--terminate", # terminate on QEMU disconnect
3074         "--daemon",
3075     ];
3076     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3077     run_command($emulator_cmd, outfunc => sub { print $1; });
3078
3079     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3080     while (! -e $paths->{pid}) {
3081         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3082         usleep(50_000);
3083     }
3084
3085     # return untainted PID of swtpm daemon so it can be killed on error
3086     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3087     return $1;
3088 }
3089
3090 sub vga_conf_has_spice {
3091     my ($vga) = @_;
3092
3093     my $vgaconf = parse_vga($vga);
3094     my $vgatype = $vgaconf->{type};
3095     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3096
3097     return $1 || 1;
3098 }
3099
3100 sub is_native($) {
3101     my ($arch) = @_;
3102     return get_host_arch() eq $arch;
3103 }
3104
3105 sub get_vm_arch {
3106     my ($conf) = @_;
3107     return $conf->{arch} // get_host_arch();
3108 }
3109
3110 my $default_machines = {
3111     x86_64 => 'pc',
3112     aarch64 => 'virt',
3113 };
3114
3115 sub get_installed_machine_version {
3116     my ($kvmversion) = @_;
3117     $kvmversion = kvm_user_version() if !defined($kvmversion);
3118     $kvmversion =~ m/^(\d+\.\d+)/;
3119     return $1;
3120 }
3121
3122 sub windows_get_pinned_machine_version {
3123     my ($machine, $base_version, $kvmversion) = @_;
3124
3125     my $pin_version = $base_version;
3126     if (!defined($base_version) ||
3127         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3128     ) {
3129         $pin_version = get_installed_machine_version($kvmversion);
3130     }
3131     if (!$machine || $machine eq 'pc') {
3132         $machine = "pc-i440fx-$pin_version";
3133     } elsif ($machine eq 'q35') {
3134         $machine = "pc-q35-$pin_version";
3135     } elsif ($machine eq 'virt') {
3136         $machine = "virt-$pin_version";
3137     } else {
3138         warn "unknown machine type '$machine', not touching that!\n";
3139     }
3140
3141     return $machine;
3142 }
3143
3144 sub get_vm_machine {
3145     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3146
3147     my $machine = $forcemachine || $conf->{machine};
3148
3149     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3150         $kvmversion //= kvm_user_version();
3151         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3152         # layout which confuses windows quite a bit and may result in various regressions..
3153         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3154         if (windows_version($conf->{ostype})) {
3155             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3156         }
3157         $arch //= 'x86_64';
3158         $machine ||= $default_machines->{$arch};
3159         if ($add_pve_version) {
3160             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3161             $machine .= "+pve$pvever";
3162         }
3163     }
3164
3165     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3166         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3167         $machine = $1 if $is_pxe;
3168
3169         # for version-pinned machines that do not include a pve-version (e.g.
3170         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3171         $machine .= '+pve0';
3172
3173         $machine .= '.pxe' if $is_pxe;
3174     }
3175
3176     return $machine;
3177 }
3178
3179 sub get_ovmf_files($$$) {
3180     my ($arch, $efidisk, $smm) = @_;
3181
3182     my $types = $OVMF->{$arch}
3183         or die "no OVMF images known for architecture '$arch'\n";
3184
3185     my $type = 'default';
3186     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3187         $type = $smm ? "4m" : "4m-no-smm";
3188         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3189     }
3190
3191     return $types->{$type}->@*;
3192 }
3193
3194 my $Arch2Qemu = {
3195     aarch64 => '/usr/bin/qemu-system-aarch64',
3196     x86_64 => '/usr/bin/qemu-system-x86_64',
3197 };
3198 sub get_command_for_arch($) {
3199     my ($arch) = @_;
3200     return '/usr/bin/kvm' if is_native($arch);
3201
3202     my $cmd = $Arch2Qemu->{$arch}
3203         or die "don't know how to emulate architecture '$arch'\n";
3204     return $cmd;
3205 }
3206
3207 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3208 # to use in a QEMU command line (-cpu element), first array_intersect the result
3209 # of query_supported_ with query_understood_. This is necessary because:
3210 #
3211 # a) query_understood_ returns flags the host cannot use and
3212 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3213 #    flags, but CPU settings - with most of them being flags. Those settings
3214 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3215 #
3216 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3217 # expensive. If you need the value returned from this, you can get it much
3218 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3219 # $accel being 'kvm' or 'tcg'.
3220 #
3221 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3222 # changes, automatically populating pmxcfs.
3223 #
3224 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3225 # since kvm and tcg machines support different flags
3226 #
3227 sub query_supported_cpu_flags {
3228     my ($arch) = @_;
3229
3230     $arch //= get_host_arch();
3231     my $default_machine = $default_machines->{$arch};
3232
3233     my $flags = {};
3234
3235     # FIXME: Once this is merged, the code below should work for ARM as well:
3236     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3237     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3238         $arch eq "aarch64";
3239
3240     my $kvm_supported = defined(kvm_version());
3241     my $qemu_cmd = get_command_for_arch($arch);
3242     my $fakevmid = -1;
3243     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3244
3245     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3246     my $query_supported_run_qemu = sub {
3247         my ($kvm) = @_;
3248
3249         my $flags = {};
3250         my $cmd = [
3251             $qemu_cmd,
3252             '-machine', $default_machine,
3253             '-display', 'none',
3254             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3255             '-mon', 'chardev=qmp,mode=control',
3256             '-pidfile', $pidfile,
3257             '-S', '-daemonize'
3258         ];
3259
3260         if (!$kvm) {
3261             push @$cmd, '-accel', 'tcg';
3262         }
3263
3264         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3265         die "QEMU flag querying VM exited with code " . $rc if $rc;
3266
3267         eval {
3268             my $cmd_result = mon_cmd(
3269                 $fakevmid,
3270                 'query-cpu-model-expansion',
3271                 type => 'full',
3272                 model => { name => 'host' }
3273             );
3274
3275             my $props = $cmd_result->{model}->{props};
3276             foreach my $prop (keys %$props) {
3277                 next if $props->{$prop} ne '1';
3278                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3279                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3280                 # We only keep those with underscores, to match /proc/cpuinfo
3281                 $prop =~ s/\.|-/_/g;
3282                 $flags->{$prop} = 1;
3283             }
3284         };
3285         my $err = $@;
3286
3287         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3288         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3289
3290         die $err if $err;
3291
3292         return [ sort keys %$flags ];
3293     };
3294
3295     # We need to query QEMU twice, since KVM and TCG have different supported flags
3296     PVE::QemuConfig->lock_config($fakevmid, sub {
3297         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3298         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3299
3300         if ($kvm_supported) {
3301             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3302             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3303         }
3304     });
3305
3306     return $flags;
3307 }
3308
3309 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3310 my $understood_cpu_flag_dir = "/usr/share/kvm";
3311 sub query_understood_cpu_flags {
3312     my $arch = get_host_arch();
3313     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3314
3315     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3316         if ! -e $filepath;
3317
3318     my $raw = file_get_contents($filepath);
3319     $raw =~ s/^\s+|\s+$//g;
3320     my @flags = split(/\s+/, $raw);
3321
3322     return \@flags;
3323 }
3324
3325 my sub get_cpuunits {
3326     my ($conf) = @_;
3327     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3328 }
3329 sub config_to_command {
3330     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3331         $pbs_backing) = @_;
3332
3333     my $cmd = [];
3334     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3335     my $devices = [];
3336     my $bridges = {};
3337     my $ostype = $conf->{ostype};
3338     my $winversion = windows_version($ostype);
3339     my $kvm = $conf->{kvm};
3340     my $nodename = nodename();
3341
3342     my $arch = get_vm_arch($conf);
3343     my $kvm_binary = get_command_for_arch($arch);
3344     my $kvmver = kvm_user_version($kvm_binary);
3345
3346     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3347         $kvmver //= "undefined";
3348         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3349     }
3350
3351     my $add_pve_version = min_version($kvmver, 4, 1);
3352
3353     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3354     my $machine_version = extract_version($machine_type, $kvmver);
3355     $kvm //= 1 if is_native($arch);
3356
3357     $machine_version =~ m/(\d+)\.(\d+)/;
3358     my ($machine_major, $machine_minor) = ($1, $2);
3359
3360     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3361         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3362     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3363         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3364             ." please upgrade node '$nodename'\n"
3365     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3366         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3367         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3368             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3369             ." node '$nodename'\n";
3370     }
3371
3372     # if a specific +pve version is required for a feature, use $version_guard
3373     # instead of min_version to allow machines to be run with the minimum
3374     # required version
3375     my $required_pve_version = 0;
3376     my $version_guard = sub {
3377         my ($major, $minor, $pve) = @_;
3378         return 0 if !min_version($machine_version, $major, $minor, $pve);
3379         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3380         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3381         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3382         return 1;
3383     };
3384
3385     if ($kvm && !defined kvm_version()) {
3386         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3387             ." or enable in BIOS.\n";
3388     }
3389
3390     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3391     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3392     my $use_old_bios_files = undef;
3393     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3394
3395     my $cpuunits = get_cpuunits($conf);
3396
3397     push @$cmd, $kvm_binary;
3398
3399     push @$cmd, '-id', $vmid;
3400
3401     my $vmname = $conf->{name} || "vm$vmid";
3402
3403     push @$cmd, '-name', $vmname;
3404
3405     push @$cmd, '-no-shutdown';
3406
3407     my $use_virtio = 0;
3408
3409     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3410     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3411     push @$cmd, '-mon', "chardev=qmp,mode=control";
3412
3413     if (min_version($machine_version, 2, 12)) {
3414         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3415         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3416     }
3417
3418     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3419
3420     push @$cmd, '-daemonize';
3421
3422     if ($conf->{smbios1}) {
3423         my $smbios_conf = parse_smbios1($conf->{smbios1});
3424         if ($smbios_conf->{base64}) {
3425             # Do not pass base64 flag to qemu
3426             delete $smbios_conf->{base64};
3427             my $smbios_string = "";
3428             foreach my $key (keys %$smbios_conf) {
3429                 my $value;
3430                 if ($key eq "uuid") {
3431                     $value = $smbios_conf->{uuid}
3432                 } else {
3433                     $value = decode_base64($smbios_conf->{$key});
3434                 }
3435                 # qemu accepts any binary data, only commas need escaping by double comma
3436                 $value =~ s/,/,,/g;
3437                 $smbios_string .= "," . $key . "=" . $value if $value;
3438             }
3439             push @$cmd, '-smbios', "type=1" . $smbios_string;
3440         } else {
3441             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3442         }
3443     }
3444
3445     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3446         my $d;
3447         if (my $efidisk = $conf->{efidisk0}) {
3448             $d = parse_drive('efidisk0', $efidisk);
3449         }
3450
3451         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3452         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3453
3454         my ($path, $format);
3455         my $read_only_str = '';
3456         if ($d) {
3457             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3458             $format = $d->{format};
3459             if ($storeid) {
3460                 $path = PVE::Storage::path($storecfg, $d->{file});
3461                 if (!defined($format)) {
3462                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3463                     $format = qemu_img_format($scfg, $volname);
3464                 }
3465             } else {
3466                 $path = $d->{file};
3467                 die "efidisk format must be specified\n"
3468                     if !defined($format);
3469             }
3470
3471             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3472         } else {
3473             warn "no efidisk configured! Using temporary efivars disk.\n";
3474             $path = "/tmp/$vmid-ovmf.fd";
3475             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3476             $format = 'raw';
3477         }
3478
3479         my $size_str = "";
3480
3481         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3482             $size_str = ",size=" . (-s $ovmf_vars);
3483         }
3484
3485         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3486         my $cache = "";
3487         if ($path =~ m/^rbd:/) {
3488                 $cache = ',cache=writeback';
3489                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3490         }
3491
3492         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3493         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3494     }
3495
3496     if ($q35) { # tell QEMU to load q35 config early
3497         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3498         if (min_version($machine_version, 4, 0)) {
3499             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3500         } else {
3501             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3502         }
3503     }
3504
3505     if ($conf->{vmgenid}) {
3506         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3507     }
3508
3509     # add usb controllers
3510     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3511         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3512     push @$devices, @usbcontrollers if @usbcontrollers;
3513     my $vga = parse_vga($conf->{vga});
3514
3515     my $qxlnum = vga_conf_has_spice($conf->{vga});
3516     $vga->{type} = 'qxl' if $qxlnum;
3517
3518     if (!$vga->{type}) {
3519         if ($arch eq 'aarch64') {
3520             $vga->{type} = 'virtio';
3521         } elsif (min_version($machine_version, 2, 9)) {
3522             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3523         } else {
3524             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3525         }
3526     }
3527
3528     # enable absolute mouse coordinates (needed by vnc)
3529     my $tablet = $conf->{tablet};
3530     if (!defined($tablet)) {
3531         $tablet = $defaults->{tablet};
3532         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3533         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3534     }
3535
3536     if ($tablet) {
3537         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3538         my $kbd = print_keyboarddevice_full($conf, $arch);
3539         push @$devices, '-device', $kbd if defined($kbd);
3540     }
3541
3542     my $bootorder = device_bootorder($conf);
3543
3544     # host pci device passthrough
3545     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3546         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3547
3548     # usb devices
3549     my $usb_dev_features = {};
3550     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3551
3552     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3553         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3554     push @$devices, @usbdevices if @usbdevices;
3555
3556     # serial devices
3557     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3558         my $path = $conf->{"serial$i"} or next;
3559         if ($path eq 'socket') {
3560             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3561             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3562             # On aarch64, serial0 is the UART device. Qemu only allows
3563             # connecting UART devices via the '-serial' command line, as
3564             # the device has a fixed slot on the hardware...
3565             if ($arch eq 'aarch64' && $i == 0) {
3566                 push @$devices, '-serial', "chardev:serial$i";
3567             } else {
3568                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3569             }
3570         } else {
3571             die "no such serial device\n" if ! -c $path;
3572             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3573             push @$devices, '-device', "isa-serial,chardev=serial$i";
3574         }
3575     }
3576
3577     # parallel devices
3578     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3579         if (my $path = $conf->{"parallel$i"}) {
3580             die "no such parallel device\n" if ! -c $path;
3581             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3582             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3583             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3584         }
3585     }
3586
3587     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3588         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3589         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3590         push @$devices, @$audio_devs;
3591     }
3592
3593     add_tpm_device($vmid, $devices, $conf);
3594
3595     my $sockets = 1;
3596     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3597     $sockets = $conf->{sockets} if  $conf->{sockets};
3598
3599     my $cores = $conf->{cores} || 1;
3600
3601     my $maxcpus = $sockets * $cores;
3602
3603     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3604
3605     my $allowed_vcpus = $cpuinfo->{cpus};
3606
3607     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3608
3609     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3610         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3611         for (my $i = 2; $i <= $vcpus; $i++)  {
3612             my $cpustr = print_cpu_device($conf,$i);
3613             push @$cmd, '-device', $cpustr;
3614         }
3615
3616     } else {
3617
3618         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3619     }
3620     push @$cmd, '-nodefaults';
3621
3622     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3623
3624     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3625
3626     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3627
3628     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3629         push @$devices, '-device', print_vga_device(
3630             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3631         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3632         push @$cmd,  '-vnc', "unix:$socket,password=on";
3633     } else {
3634         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3635         push @$cmd, '-nographic';
3636     }
3637
3638     # time drift fix
3639     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3640     my $useLocaltime = $conf->{localtime};
3641
3642     if ($winversion >= 5) { # windows
3643         $useLocaltime = 1 if !defined($conf->{localtime});
3644
3645         # use time drift fix when acpi is enabled
3646         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3647             $tdf = 1 if !defined($conf->{tdf});
3648         }
3649     }
3650
3651     if ($winversion >= 6) {
3652         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3653         push @$cmd, '-no-hpet';
3654     }
3655
3656     push @$rtcFlags, 'driftfix=slew' if $tdf;
3657
3658     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3659         push @$rtcFlags, "base=$conf->{startdate}";
3660     } elsif ($useLocaltime) {
3661         push @$rtcFlags, 'base=localtime';
3662     }
3663
3664     if ($forcecpu) {
3665         push @$cmd, '-cpu', $forcecpu;
3666     } else {
3667         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3668     }
3669
3670     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3671
3672     push @$cmd, '-S' if $conf->{freeze};
3673
3674     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3675
3676     my $guest_agent = parse_guest_agent($conf);
3677
3678     if ($guest_agent->{enabled}) {
3679         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3680         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3681
3682         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3683             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3684             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3685             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3686         } elsif ($guest_agent->{type} eq 'isa') {
3687             push @$devices, '-device', "isa-serial,chardev=qga0";
3688         }
3689     }
3690
3691     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3692     if ($rng && $version_guard->(4, 1, 2)) {
3693         check_rng_source($rng->{source});
3694
3695         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3696         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3697         my $limiter_str = "";
3698         if ($max_bytes) {
3699             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3700         }
3701
3702         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3703         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3704         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3705     }
3706
3707     my $spice_port;
3708
3709     if ($qxlnum) {
3710         if ($qxlnum > 1) {
3711             if ($winversion){
3712                 for (my $i = 1; $i < $qxlnum; $i++){
3713                     push @$devices, '-device', print_vga_device(
3714                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3715                 }
3716             } else {
3717                 # assume other OS works like Linux
3718                 my ($ram, $vram) = ("134217728", "67108864");
3719                 if ($vga->{memory}) {
3720                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3721                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3722                 }
3723                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3724                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3725             }
3726         }
3727
3728         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3729
3730         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3731         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3732         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3733
3734         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3735         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3736         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3737
3738         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3739         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3740
3741         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3742         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3743         if ($spice_enhancement->{foldersharing}) {
3744             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3745             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3746         }
3747
3748         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3749         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3750             if $spice_enhancement->{videostreaming};
3751
3752         push @$devices, '-spice', "$spice_opts";
3753     }
3754
3755     # enable balloon by default, unless explicitly disabled
3756     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3757         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3758         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3759     }
3760
3761     if ($conf->{watchdog}) {
3762         my $wdopts = parse_watchdog($conf->{watchdog});
3763         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3764         my $watchdog = $wdopts->{model} || 'i6300esb';
3765         push @$devices, '-device', "$watchdog$pciaddr";
3766         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3767     }
3768
3769     my $vollist = [];
3770     my $scsicontroller = {};
3771     my $ahcicontroller = {};
3772     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3773
3774     # Add iscsi initiator name if available
3775     if (my $initiator = get_initiator_name()) {
3776         push @$devices, '-iscsi', "initiator-name=$initiator";
3777     }
3778
3779     PVE::QemuConfig->foreach_volume($conf, sub {
3780         my ($ds, $drive) = @_;
3781
3782         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3783             check_volume_storage_type($storecfg, $drive->{file});
3784             push @$vollist, $drive->{file};
3785         }
3786
3787         # ignore efidisk here, already added in bios/fw handling code above
3788         return if $drive->{interface} eq 'efidisk';
3789         # similar for TPM
3790         return if $drive->{interface} eq 'tpmstate';
3791
3792         $use_virtio = 1 if $ds =~ m/^virtio/;
3793
3794         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3795
3796         if ($drive->{interface} eq 'virtio'){
3797            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3798         }
3799
3800         if ($drive->{interface} eq 'scsi') {
3801
3802             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3803
3804             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3805                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3806
3807             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3808             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3809
3810             my $iothread = '';
3811             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3812                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3813                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3814             } elsif ($drive->{iothread}) {
3815                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3816             }
3817
3818             my $queues = '';
3819             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3820                 $queues = ",num_queues=$drive->{queues}";
3821             }
3822
3823             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3824                 if !$scsicontroller->{$controller};
3825             $scsicontroller->{$controller}=1;
3826         }
3827
3828         if ($drive->{interface} eq 'sata') {
3829             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3830             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3831             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3832                 if !$ahcicontroller->{$controller};
3833             $ahcicontroller->{$controller}=1;
3834         }
3835
3836         my $pbs_conf = $pbs_backing->{$ds};
3837         my $pbs_name = undef;
3838         if ($pbs_conf) {
3839             $pbs_name = "drive-$ds-pbs";
3840             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3841         }
3842
3843         my $drive_cmd = print_drive_commandline_full(
3844             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3845
3846         # extra protection for templates, but SATA and IDE don't support it..
3847         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3848
3849         push @$devices, '-drive',$drive_cmd;
3850         push @$devices, '-device', print_drivedevice_full(
3851             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3852     });
3853
3854     for (my $i = 0; $i < $MAX_NETS; $i++) {
3855         my $netname = "net$i";
3856
3857         next if !$conf->{$netname};
3858         my $d = parse_net($conf->{$netname});
3859         next if !$d;
3860
3861         $use_virtio = 1 if $d->{model} eq 'virtio';
3862
3863         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3864
3865         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3866         push @$devices, '-netdev', $netdevfull;
3867
3868         my $netdevicefull = print_netdevice_full(
3869             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3870
3871         push @$devices, '-device', $netdevicefull;
3872     }
3873
3874     if ($conf->{ivshmem}) {
3875         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3876
3877         my $bus;
3878         if ($q35) {
3879             $bus = print_pcie_addr("ivshmem");
3880         } else {
3881             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3882         }
3883
3884         my $ivshmem_name = $ivshmem->{name} // $vmid;
3885         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3886
3887         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3888         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3889             .",size=$ivshmem->{size}M";
3890     }
3891
3892     # pci.4 is nested in pci.1
3893     $bridges->{1} = 1 if $bridges->{4};
3894
3895     if (!$q35) { # add pci bridges
3896         if (min_version($machine_version, 2, 3)) {
3897            $bridges->{1} = 1;
3898            $bridges->{2} = 1;
3899         }
3900         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3901     }
3902
3903     for my $k (sort {$b cmp $a} keys %$bridges) {
3904         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3905
3906         my $k_name = $k;
3907         if ($k == 2 && $legacy_igd) {
3908             $k_name = "$k-igd";
3909         }
3910         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3911         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3912
3913         if ($q35) { # add after -readconfig pve-q35.cfg
3914             splice @$devices, 2, 0, '-device', $devstr;
3915         } else {
3916             unshift @$devices, '-device', $devstr if $k > 0;
3917         }
3918     }
3919
3920     if (!$kvm) {
3921         push @$machineFlags, 'accel=tcg';
3922     }
3923
3924     my $machine_type_min = $machine_type;
3925     if ($add_pve_version) {
3926         $machine_type_min =~ s/\+pve\d+$//;
3927         $machine_type_min .= "+pve$required_pve_version";
3928     }
3929     push @$machineFlags, "type=${machine_type_min}";
3930
3931     push @$cmd, @$devices;
3932     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3933     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3934     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3935
3936     if (my $vmstate = $conf->{vmstate}) {
3937         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3938         push @$vollist, $vmstate;
3939         push @$cmd, '-loadstate', $statepath;
3940         print "activating and using '$vmstate' as vmstate\n";
3941     }
3942
3943     if (PVE::QemuConfig->is_template($conf)) {
3944         # needed to workaround base volumes being read-only
3945         push @$cmd, '-snapshot';
3946     }
3947
3948     # add custom args
3949     if ($conf->{args}) {
3950         my $aa = PVE::Tools::split_args($conf->{args});
3951         push @$cmd, @$aa;
3952     }
3953
3954     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3955 }
3956
3957 sub check_rng_source {
3958     my ($source) = @_;
3959
3960     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3961     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3962         if ! -e $source;
3963
3964     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3965     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3966         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3967         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3968         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3969             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3970             ." to the host.\n";
3971     }
3972 }
3973
3974 sub spice_port {
3975     my ($vmid) = @_;
3976
3977     my $res = mon_cmd($vmid, 'query-spice');
3978
3979     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3980 }
3981
3982 sub vm_devices_list {
3983     my ($vmid) = @_;
3984
3985     my $res = mon_cmd($vmid, 'query-pci');
3986     my $devices_to_check = [];
3987     my $devices = {};
3988     foreach my $pcibus (@$res) {
3989         push @$devices_to_check, @{$pcibus->{devices}},
3990     }
3991
3992     while (@$devices_to_check) {
3993         my $to_check = [];
3994         for my $d (@$devices_to_check) {
3995             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3996             next if !$d->{'pci_bridge'};
3997
3998             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3999             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4000         }
4001         $devices_to_check = $to_check;
4002     }
4003
4004     my $resblock = mon_cmd($vmid, 'query-block');
4005     foreach my $block (@$resblock) {
4006         if($block->{device} =~ m/^drive-(\S+)/){
4007                 $devices->{$1} = 1;
4008         }
4009     }
4010
4011     my $resmice = mon_cmd($vmid, 'query-mice');
4012     foreach my $mice (@$resmice) {
4013         if ($mice->{name} eq 'QEMU HID Tablet') {
4014             $devices->{tablet} = 1;
4015             last;
4016         }
4017     }
4018
4019     # for usb devices there is no query-usb
4020     # but we can iterate over the entries in
4021     # qom-list path=/machine/peripheral
4022     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4023     foreach my $per (@$resperipheral) {
4024         if ($per->{name} =~ m/^usb\d+$/) {
4025             $devices->{$per->{name}} = 1;
4026         }
4027     }
4028
4029     return $devices;
4030 }
4031
4032 sub vm_deviceplug {
4033     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4034
4035     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4036
4037     my $devices_list = vm_devices_list($vmid);
4038     return 1 if defined($devices_list->{$deviceid});
4039
4040     # add PCI bridge if we need it for the device
4041     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4042
4043     if ($deviceid eq 'tablet') {
4044         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4045     } elsif ($deviceid eq 'keyboard') {
4046         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4047     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4048         die "usb hotplug currently not reliable\n";
4049         # since we can't reliably hot unplug all added usb devices and usb
4050         # passthrough breaks live migration we disable usb hotplugging for now
4051         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4052     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4053         qemu_iothread_add($vmid, $deviceid, $device);
4054
4055         qemu_driveadd($storecfg, $vmid, $device);
4056         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4057
4058         qemu_deviceadd($vmid, $devicefull);
4059         eval { qemu_deviceaddverify($vmid, $deviceid); };
4060         if (my $err = $@) {
4061             eval { qemu_drivedel($vmid, $deviceid); };
4062             warn $@ if $@;
4063             die $err;
4064         }
4065     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4066         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4067         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4068         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4069
4070         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4071
4072         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4073             qemu_iothread_add($vmid, $deviceid, $device);
4074             $devicefull .= ",iothread=iothread-$deviceid";
4075         }
4076
4077         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4078             $devicefull .= ",num_queues=$device->{queues}";
4079         }
4080
4081         qemu_deviceadd($vmid, $devicefull);
4082         qemu_deviceaddverify($vmid, $deviceid);
4083     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4084         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4085         qemu_driveadd($storecfg, $vmid, $device);
4086
4087         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4088         eval { qemu_deviceadd($vmid, $devicefull); };
4089         if (my $err = $@) {
4090             eval { qemu_drivedel($vmid, $deviceid); };
4091             warn $@ if $@;
4092             die $err;
4093         }
4094     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4095         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4096
4097         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4098         my $use_old_bios_files = undef;
4099         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4100
4101         my $netdevicefull = print_netdevice_full(
4102             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4103         qemu_deviceadd($vmid, $netdevicefull);
4104         eval {
4105             qemu_deviceaddverify($vmid, $deviceid);
4106             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4107         };
4108         if (my $err = $@) {
4109             eval { qemu_netdevdel($vmid, $deviceid); };
4110             warn $@ if $@;
4111             die $err;
4112         }
4113     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4114         my $bridgeid = $2;
4115         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4116         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4117
4118         qemu_deviceadd($vmid, $devicefull);
4119         qemu_deviceaddverify($vmid, $deviceid);
4120     } else {
4121         die "can't hotplug device '$deviceid'\n";
4122     }
4123
4124     return 1;
4125 }
4126
4127 # fixme: this should raise exceptions on error!
4128 sub vm_deviceunplug {
4129     my ($vmid, $conf, $deviceid) = @_;
4130
4131     my $devices_list = vm_devices_list($vmid);
4132     return 1 if !defined($devices_list->{$deviceid});
4133
4134     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4135     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4136
4137     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4138         qemu_devicedel($vmid, $deviceid);
4139     } elsif ($deviceid =~ m/^usb\d+$/) {
4140         die "usb hotplug currently not reliable\n";
4141         # when unplugging usb devices this way, there may be remaining usb
4142         # controllers/hubs so we disable it for now
4143         #qemu_devicedel($vmid, $deviceid);
4144         #qemu_devicedelverify($vmid, $deviceid);
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         my $device = parse_drive($deviceid, $conf->{$deviceid});
4147
4148         qemu_devicedel($vmid, $deviceid);
4149         qemu_devicedelverify($vmid, $deviceid);
4150         qemu_drivedel($vmid, $deviceid);
4151         qemu_iothread_del($vmid, $deviceid, $device);
4152     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4153         qemu_devicedel($vmid, $deviceid);
4154         qemu_devicedelverify($vmid, $deviceid);
4155     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4156         my $device = parse_drive($deviceid, $conf->{$deviceid});
4157
4158         qemu_devicedel($vmid, $deviceid);
4159         qemu_drivedel($vmid, $deviceid);
4160         qemu_deletescsihw($conf, $vmid, $deviceid);
4161
4162         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4163             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4164     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4165         qemu_devicedel($vmid, $deviceid);
4166         qemu_devicedelverify($vmid, $deviceid);
4167         qemu_netdevdel($vmid, $deviceid);
4168     } else {
4169         die "can't unplug device '$deviceid'\n";
4170     }
4171
4172     return 1;
4173 }
4174
4175 sub qemu_deviceadd {
4176     my ($vmid, $devicefull) = @_;
4177
4178     $devicefull = "driver=".$devicefull;
4179     my %options =  split(/[=,]/, $devicefull);
4180
4181     mon_cmd($vmid, "device_add" , %options);
4182 }
4183
4184 sub qemu_devicedel {
4185     my ($vmid, $deviceid) = @_;
4186
4187     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4188 }
4189
4190 sub qemu_iothread_add {
4191     my ($vmid, $deviceid, $device) = @_;
4192
4193     if ($device->{iothread}) {
4194         my $iothreads = vm_iothreads_list($vmid);
4195         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4196     }
4197 }
4198
4199 sub qemu_iothread_del {
4200     my ($vmid, $deviceid, $device) = @_;
4201
4202     if ($device->{iothread}) {
4203         my $iothreads = vm_iothreads_list($vmid);
4204         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4205     }
4206 }
4207
4208 sub qemu_objectadd {
4209     my ($vmid, $objectid, $qomtype) = @_;
4210
4211     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4212
4213     return 1;
4214 }
4215
4216 sub qemu_objectdel {
4217     my ($vmid, $objectid) = @_;
4218
4219     mon_cmd($vmid, "object-del", id => $objectid);
4220
4221     return 1;
4222 }
4223
4224 sub qemu_driveadd {
4225     my ($storecfg, $vmid, $device) = @_;
4226
4227     my $kvmver = get_running_qemu_version($vmid);
4228     my $io_uring = min_version($kvmver, 6, 0);
4229     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4230     $drive =~ s/\\/\\\\/g;
4231     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4232
4233     # If the command succeeds qemu prints: "OK"
4234     return 1 if $ret =~ m/OK/s;
4235
4236     die "adding drive failed: $ret\n";
4237 }
4238
4239 sub qemu_drivedel {
4240     my ($vmid, $deviceid) = @_;
4241
4242     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4243     $ret =~ s/^\s+//;
4244
4245     return 1 if $ret eq "";
4246
4247     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4248     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4249
4250     die "deleting drive $deviceid failed : $ret\n";
4251 }
4252
4253 sub qemu_deviceaddverify {
4254     my ($vmid, $deviceid) = @_;
4255
4256     for (my $i = 0; $i <= 5; $i++) {
4257          my $devices_list = vm_devices_list($vmid);
4258          return 1 if defined($devices_list->{$deviceid});
4259          sleep 1;
4260     }
4261
4262     die "error on hotplug device '$deviceid'\n";
4263 }
4264
4265
4266 sub qemu_devicedelverify {
4267     my ($vmid, $deviceid) = @_;
4268
4269     # need to verify that the device is correctly removed as device_del
4270     # is async and empty return is not reliable
4271
4272     for (my $i = 0; $i <= 5; $i++) {
4273          my $devices_list = vm_devices_list($vmid);
4274          return 1 if !defined($devices_list->{$deviceid});
4275          sleep 1;
4276     }
4277
4278     die "error on hot-unplugging device '$deviceid'\n";
4279 }
4280
4281 sub qemu_findorcreatescsihw {
4282     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4283
4284     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4285
4286     my $scsihwid="$controller_prefix$controller";
4287     my $devices_list = vm_devices_list($vmid);
4288
4289     if (!defined($devices_list->{$scsihwid})) {
4290         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4291     }
4292
4293     return 1;
4294 }
4295
4296 sub qemu_deletescsihw {
4297     my ($conf, $vmid, $opt) = @_;
4298
4299     my $device = parse_drive($opt, $conf->{$opt});
4300
4301     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4302         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4303         return 1;
4304     }
4305
4306     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4307
4308     my $devices_list = vm_devices_list($vmid);
4309     foreach my $opt (keys %{$devices_list}) {
4310         if (is_valid_drivename($opt)) {
4311             my $drive = parse_drive($opt, $conf->{$opt});
4312             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4313                 return 1;
4314             }
4315         }
4316     }
4317
4318     my $scsihwid="scsihw$controller";
4319
4320     vm_deviceunplug($vmid, $conf, $scsihwid);
4321
4322     return 1;
4323 }
4324
4325 sub qemu_add_pci_bridge {
4326     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4327
4328     my $bridges = {};
4329
4330     my $bridgeid;
4331
4332     print_pci_addr($device, $bridges, $arch, $machine_type);
4333
4334     while (my ($k, $v) = each %$bridges) {
4335         $bridgeid = $k;
4336     }
4337     return 1 if !defined($bridgeid) || $bridgeid < 1;
4338
4339     my $bridge = "pci.$bridgeid";
4340     my $devices_list = vm_devices_list($vmid);
4341
4342     if (!defined($devices_list->{$bridge})) {
4343         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4344     }
4345
4346     return 1;
4347 }
4348
4349 sub qemu_set_link_status {
4350     my ($vmid, $device, $up) = @_;
4351
4352     mon_cmd($vmid, "set_link", name => $device,
4353                up => $up ? JSON::true : JSON::false);
4354 }
4355
4356 sub qemu_netdevadd {
4357     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4358
4359     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4360     my %options =  split(/[=,]/, $netdev);
4361
4362     if (defined(my $vhost = $options{vhost})) {
4363         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4364     }
4365
4366     if (defined(my $queues = $options{queues})) {
4367         $options{queues} = $queues + 0;
4368     }
4369
4370     mon_cmd($vmid, "netdev_add",  %options);
4371     return 1;
4372 }
4373
4374 sub qemu_netdevdel {
4375     my ($vmid, $deviceid) = @_;
4376
4377     mon_cmd($vmid, "netdev_del", id => $deviceid);
4378 }
4379
4380 sub qemu_usb_hotplug {
4381     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4382
4383     return if !$device;
4384
4385     # remove the old one first
4386     vm_deviceunplug($vmid, $conf, $deviceid);
4387
4388     # check if xhci controller is necessary and available
4389     if ($device->{usb3}) {
4390
4391         my $devicelist = vm_devices_list($vmid);
4392
4393         if (!$devicelist->{xhci}) {
4394             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4395             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4396         }
4397     }
4398     my $d = parse_usb_device($device->{host});
4399     $d->{usb3} = $device->{usb3};
4400
4401     # add the new one
4402     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4403 }
4404
4405 sub qemu_cpu_hotplug {
4406     my ($vmid, $conf, $vcpus) = @_;
4407
4408     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4409
4410     my $sockets = 1;
4411     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4412     $sockets = $conf->{sockets} if  $conf->{sockets};
4413     my $cores = $conf->{cores} || 1;
4414     my $maxcpus = $sockets * $cores;
4415
4416     $vcpus = $maxcpus if !$vcpus;
4417
4418     die "you can't add more vcpus than maxcpus\n"
4419         if $vcpus > $maxcpus;
4420
4421     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4422
4423     if ($vcpus < $currentvcpus) {
4424
4425         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4426
4427             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4428                 qemu_devicedel($vmid, "cpu$i");
4429                 my $retry = 0;
4430                 my $currentrunningvcpus = undef;
4431                 while (1) {
4432                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4433                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4434                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4435                     $retry++;
4436                     sleep 1;
4437                 }
4438                 #update conf after each succesfull cpu unplug
4439                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4440                 PVE::QemuConfig->write_config($vmid, $conf);
4441             }
4442         } else {
4443             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4444         }
4445
4446         return;
4447     }
4448
4449     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4450     die "vcpus in running vm does not match its configuration\n"
4451         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4452
4453     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4454
4455         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4456             my $cpustr = print_cpu_device($conf, $i);
4457             qemu_deviceadd($vmid, $cpustr);
4458
4459             my $retry = 0;
4460             my $currentrunningvcpus = undef;
4461             while (1) {
4462                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4463                 last if scalar(@{$currentrunningvcpus}) == $i;
4464                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4465                 sleep 1;
4466                 $retry++;
4467             }
4468             #update conf after each succesfull cpu hotplug
4469             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4470             PVE::QemuConfig->write_config($vmid, $conf);
4471         }
4472     } else {
4473
4474         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4475             mon_cmd($vmid, "cpu-add", id => int($i));
4476         }
4477     }
4478 }
4479
4480 sub qemu_block_set_io_throttle {
4481     my ($vmid, $deviceid,
4482         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4483         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4484         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4485         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4486
4487     return if !check_running($vmid) ;
4488
4489     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4490         bps => int($bps),
4491         bps_rd => int($bps_rd),
4492         bps_wr => int($bps_wr),
4493         iops => int($iops),
4494         iops_rd => int($iops_rd),
4495         iops_wr => int($iops_wr),
4496         bps_max => int($bps_max),
4497         bps_rd_max => int($bps_rd_max),
4498         bps_wr_max => int($bps_wr_max),
4499         iops_max => int($iops_max),
4500         iops_rd_max => int($iops_rd_max),
4501         iops_wr_max => int($iops_wr_max),
4502         bps_max_length => int($bps_max_length),
4503         bps_rd_max_length => int($bps_rd_max_length),
4504         bps_wr_max_length => int($bps_wr_max_length),
4505         iops_max_length => int($iops_max_length),
4506         iops_rd_max_length => int($iops_rd_max_length),
4507         iops_wr_max_length => int($iops_wr_max_length),
4508     );
4509
4510 }
4511
4512 sub qemu_block_resize {
4513     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4514
4515     my $running = check_running($vmid);
4516
4517     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4518
4519     return if !$running;
4520
4521     my $padding = (1024 - $size % 1024) % 1024;
4522     $size = $size + $padding;
4523
4524     mon_cmd(
4525         $vmid,
4526         "block_resize",
4527         device => $deviceid,
4528         size => int($size),
4529         timeout => 60,
4530     );
4531 }
4532
4533 sub qemu_volume_snapshot {
4534     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4535
4536     my $running = check_running($vmid);
4537
4538     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4539         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4540     } else {
4541         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4542     }
4543 }
4544
4545 sub qemu_volume_snapshot_delete {
4546     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4547
4548     my $running = check_running($vmid);
4549
4550     if($running) {
4551
4552         $running = undef;
4553         my $conf = PVE::QemuConfig->load_config($vmid);
4554         PVE::QemuConfig->foreach_volume($conf, sub {
4555             my ($ds, $drive) = @_;
4556             $running = 1 if $drive->{file} eq $volid;
4557         });
4558     }
4559
4560     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4561         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4562     } else {
4563         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4564     }
4565 }
4566
4567 sub set_migration_caps {
4568     my ($vmid, $savevm) = @_;
4569
4570     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4571
4572     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4573     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4574
4575     my $cap_ref = [];
4576
4577     my $enabled_cap = {
4578         "auto-converge" => 1,
4579         "xbzrle" => 1,
4580         "x-rdma-pin-all" => 0,
4581         "zero-blocks" => 0,
4582         "compress" => 0,
4583         "dirty-bitmaps" => $dirty_bitmaps,
4584     };
4585
4586     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4587
4588     for my $supported_capability (@$supported_capabilities) {
4589         push @$cap_ref, {
4590             capability => $supported_capability->{capability},
4591             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4592         };
4593     }
4594
4595     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4596 }
4597
4598 sub foreach_volid {
4599     my ($conf, $func, @param) = @_;
4600
4601     my $volhash = {};
4602
4603     my $test_volid = sub {
4604         my ($key, $drive, $snapname) = @_;
4605
4606         my $volid = $drive->{file};
4607         return if !$volid;
4608
4609         $volhash->{$volid}->{cdrom} //= 1;
4610         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4611
4612         my $replicate = $drive->{replicate} // 1;
4613         $volhash->{$volid}->{replicate} //= 0;
4614         $volhash->{$volid}->{replicate} = 1 if $replicate;
4615
4616         $volhash->{$volid}->{shared} //= 0;
4617         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4618
4619         $volhash->{$volid}->{referenced_in_config} //= 0;
4620         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4621
4622         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4623             if defined($snapname);
4624
4625         my $size = $drive->{size};
4626         $volhash->{$volid}->{size} //= $size if $size;
4627
4628         $volhash->{$volid}->{is_vmstate} //= 0;
4629         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4630
4631         $volhash->{$volid}->{is_tpmstate} //= 0;
4632         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4633
4634         $volhash->{$volid}->{is_unused} //= 0;
4635         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4636
4637         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4638     };
4639
4640     my $include_opts = {
4641         extra_keys => ['vmstate'],
4642         include_unused => 1,
4643     };
4644
4645     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4646     foreach my $snapname (keys %{$conf->{snapshots}}) {
4647         my $snap = $conf->{snapshots}->{$snapname};
4648         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4649     }
4650
4651     foreach my $volid (keys %$volhash) {
4652         &$func($volid, $volhash->{$volid}, @param);
4653     }
4654 }
4655
4656 my $fast_plug_option = {
4657     'lock' => 1,
4658     'name' => 1,
4659     'onboot' => 1,
4660     'shares' => 1,
4661     'startup' => 1,
4662     'description' => 1,
4663     'protection' => 1,
4664     'vmstatestorage' => 1,
4665     'hookscript' => 1,
4666     'tags' => 1,
4667 };
4668
4669 # hotplug changes in [PENDING]
4670 # $selection hash can be used to only apply specified options, for
4671 # example: { cores => 1 } (only apply changed 'cores')
4672 # $errors ref is used to return error messages
4673 sub vmconfig_hotplug_pending {
4674     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4675
4676     my $defaults = load_defaults();
4677     my $arch = get_vm_arch($conf);
4678     my $machine_type = get_vm_machine($conf, undef, $arch);
4679
4680     # commit values which do not have any impact on running VM first
4681     # Note: those option cannot raise errors, we we do not care about
4682     # $selection and always apply them.
4683
4684     my $add_error = sub {
4685         my ($opt, $msg) = @_;
4686         $errors->{$opt} = "hotplug problem - $msg";
4687     };
4688
4689     my $changes = 0;
4690     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4691         if ($fast_plug_option->{$opt}) {
4692             $conf->{$opt} = $conf->{pending}->{$opt};
4693             delete $conf->{pending}->{$opt};
4694             $changes = 1;
4695         }
4696     }
4697
4698     if ($changes) {
4699         PVE::QemuConfig->write_config($vmid, $conf);
4700     }
4701
4702     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4703
4704     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4705     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4706     foreach my $opt (sort keys %$pending_delete_hash) {
4707         next if $selection && !$selection->{$opt};
4708         my $force = $pending_delete_hash->{$opt}->{force};
4709         eval {
4710             if ($opt eq 'hotplug') {
4711                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4712             } elsif ($opt eq 'tablet') {
4713                 die "skip\n" if !$hotplug_features->{usb};
4714                 if ($defaults->{tablet}) {
4715                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4716                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4717                         if $arch eq 'aarch64';
4718                 } else {
4719                     vm_deviceunplug($vmid, $conf, 'tablet');
4720                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4721                 }
4722             } elsif ($opt =~ m/^usb\d+/) {
4723                 die "skip\n";
4724                 # since we cannot reliably hot unplug usb devices we are disabling it
4725                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4726                 #vm_deviceunplug($vmid, $conf, $opt);
4727             } elsif ($opt eq 'vcpus') {
4728                 die "skip\n" if !$hotplug_features->{cpu};
4729                 qemu_cpu_hotplug($vmid, $conf, undef);
4730             } elsif ($opt eq 'balloon') {
4731                 # enable balloon device is not hotpluggable
4732                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4733                 # here we reset the ballooning value to memory
4734                 my $balloon = $conf->{memory} || $defaults->{memory};
4735                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4736             } elsif ($fast_plug_option->{$opt}) {
4737                 # do nothing
4738             } elsif ($opt =~ m/^net(\d+)$/) {
4739                 die "skip\n" if !$hotplug_features->{network};
4740                 vm_deviceunplug($vmid, $conf, $opt);
4741             } elsif (is_valid_drivename($opt)) {
4742                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4743                 vm_deviceunplug($vmid, $conf, $opt);
4744                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4745             } elsif ($opt =~ m/^memory$/) {
4746                 die "skip\n" if !$hotplug_features->{memory};
4747                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4748             } elsif ($opt eq 'cpuunits') {
4749                 $cgroup->change_cpu_shares(undef, 1024);
4750             } elsif ($opt eq 'cpulimit') {
4751                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4752             } else {
4753                 die "skip\n";
4754             }
4755         };
4756         if (my $err = $@) {
4757             &$add_error($opt, $err) if $err ne "skip\n";
4758         } else {
4759             delete $conf->{$opt};
4760             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4761         }
4762     }
4763
4764     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4765     $apply_pending_cloudinit = sub {
4766         return if $apply_pending_cloudinit_done; # once is enough
4767         $apply_pending_cloudinit_done = 1; # once is enough
4768
4769         my ($key, $value) = @_;
4770
4771         my @cloudinit_opts = keys %$confdesc_cloudinit;
4772         foreach my $opt (keys %{$conf->{pending}}) {
4773             next if !grep { $_ eq $opt } @cloudinit_opts;
4774             $conf->{$opt} = delete $conf->{pending}->{$opt};
4775         }
4776
4777         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4778         foreach my $opt (sort keys %$pending_delete_hash) {
4779             next if !grep { $_ eq $opt } @cloudinit_opts;
4780             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4781             delete $conf->{$opt};
4782         }
4783
4784         my $new_conf = { %$conf };
4785         $new_conf->{$key} = $value;
4786         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4787     };
4788
4789     foreach my $opt (keys %{$conf->{pending}}) {
4790         next if $selection && !$selection->{$opt};
4791         my $value = $conf->{pending}->{$opt};
4792         eval {
4793             if ($opt eq 'hotplug') {
4794                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4795             } elsif ($opt eq 'tablet') {
4796                 die "skip\n" if !$hotplug_features->{usb};
4797                 if ($value == 1) {
4798                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4799                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4800                         if $arch eq 'aarch64';
4801                 } elsif ($value == 0) {
4802                     vm_deviceunplug($vmid, $conf, 'tablet');
4803                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4804                 }
4805             } elsif ($opt =~ m/^usb\d+$/) {
4806                 die "skip\n";
4807                 # since we cannot reliably hot unplug usb devices we disable it for now
4808                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4809                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4810                 #die "skip\n" if !$d;
4811                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4812             } elsif ($opt eq 'vcpus') {
4813                 die "skip\n" if !$hotplug_features->{cpu};
4814                 qemu_cpu_hotplug($vmid, $conf, $value);
4815             } elsif ($opt eq 'balloon') {
4816                 # enable/disable balloning device is not hotpluggable
4817                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4818                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4819                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4820
4821                 # allow manual ballooning if shares is set to zero
4822                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4823                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4824                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4825                 }
4826             } elsif ($opt =~ m/^net(\d+)$/) {
4827                 # some changes can be done without hotplug
4828                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4829                                     $vmid, $opt, $value, $arch, $machine_type);
4830             } elsif (is_valid_drivename($opt)) {
4831                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4832                 # some changes can be done without hotplug
4833                 my $drive = parse_drive($opt, $value);
4834                 if (drive_is_cloudinit($drive)) {
4835                     &$apply_pending_cloudinit($opt, $value);
4836                 }
4837                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4838                                      $vmid, $opt, $value, $arch, $machine_type);
4839             } elsif ($opt =~ m/^memory$/) { #dimms
4840                 die "skip\n" if !$hotplug_features->{memory};
4841                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4842             } elsif ($opt eq 'cpuunits') {
4843                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4844             } elsif ($opt eq 'cpulimit') {
4845                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4846                 $cgroup->change_cpu_quota($cpulimit, 100000);
4847             } elsif ($opt eq 'agent') {
4848                 vmconfig_update_agent($conf, $opt, $value);
4849             } else {
4850                 die "skip\n";  # skip non-hot-pluggable options
4851             }
4852         };
4853         if (my $err = $@) {
4854             &$add_error($opt, $err) if $err ne "skip\n";
4855         } else {
4856             $conf->{$opt} = $value;
4857             delete $conf->{pending}->{$opt};
4858         }
4859     }
4860
4861     PVE::QemuConfig->write_config($vmid, $conf);
4862 }
4863
4864 sub try_deallocate_drive {
4865     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4866
4867     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4868         my $volid = $drive->{file};
4869         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4870             my $sid = PVE::Storage::parse_volume_id($volid);
4871             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4872
4873             # check if the disk is really unused
4874             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4875                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4876             PVE::Storage::vdisk_free($storecfg, $volid);
4877             return 1;
4878         } else {
4879             # If vm is not owner of this disk remove from config
4880             return 1;
4881         }
4882     }
4883
4884     return;
4885 }
4886
4887 sub vmconfig_delete_or_detach_drive {
4888     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4889
4890     my $drive = parse_drive($opt, $conf->{$opt});
4891
4892     my $rpcenv = PVE::RPCEnvironment::get();
4893     my $authuser = $rpcenv->get_user();
4894
4895     if ($force) {
4896         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4897         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4898     } else {
4899         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4900     }
4901 }
4902
4903
4904
4905 sub vmconfig_apply_pending {
4906     my ($vmid, $conf, $storecfg, $errors) = @_;
4907
4908     my $add_apply_error = sub {
4909         my ($opt, $msg) = @_;
4910         my $err_msg = "unable to apply pending change $opt : $msg";
4911         $errors->{$opt} = $err_msg;
4912         warn $err_msg;
4913     };
4914
4915     # cold plug
4916
4917     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4918     foreach my $opt (sort keys %$pending_delete_hash) {
4919         my $force = $pending_delete_hash->{$opt}->{force};
4920         eval {
4921             if ($opt =~ m/^unused/) {
4922                 die "internal error";
4923             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4924                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4925             }
4926         };
4927         if (my $err = $@) {
4928             $add_apply_error->($opt, $err);
4929         } else {
4930             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4931             delete $conf->{$opt};
4932         }
4933     }
4934
4935     PVE::QemuConfig->cleanup_pending($conf);
4936
4937     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4938         next if $opt eq 'delete'; # just to be sure
4939         eval {
4940             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4941                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4942             }
4943         };
4944         if (my $err = $@) {
4945             $add_apply_error->($opt, $err);
4946         } else {
4947             $conf->{$opt} = delete $conf->{pending}->{$opt};
4948         }
4949     }
4950
4951     # write all changes at once to avoid unnecessary i/o
4952     PVE::QemuConfig->write_config($vmid, $conf);
4953 }
4954
4955 sub vmconfig_update_net {
4956     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4957
4958     my $newnet = parse_net($value);
4959
4960     if ($conf->{$opt}) {
4961         my $oldnet = parse_net($conf->{$opt});
4962
4963         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4964             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4965             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4966             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4967
4968             # for non online change, we try to hot-unplug
4969             die "skip\n" if !$hotplug;
4970             vm_deviceunplug($vmid, $conf, $opt);
4971         } else {
4972
4973             die "internal error" if $opt !~ m/net(\d+)/;
4974             my $iface = "tap${vmid}i$1";
4975
4976             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4977                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4978                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4979                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4980                 PVE::Network::tap_unplug($iface);
4981
4982                 if ($have_sdn) {
4983                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4984                 } else {
4985                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4986                 }
4987             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4988                 # Rate can be applied on its own but any change above needs to
4989                 # include the rate in tap_plug since OVS resets everything.
4990                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4991             }
4992
4993             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4994                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4995             }
4996
4997             return 1;
4998         }
4999     }
5000
5001     if ($hotplug) {
5002         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5003     } else {
5004         die "skip\n";
5005     }
5006 }
5007
5008 sub vmconfig_update_agent {
5009     my ($conf, $opt, $value) = @_;
5010
5011     die "skip\n" if !$conf->{$opt};
5012
5013     my $hotplug_options = { fstrim_cloned_disks => 1 };
5014
5015     my $old_agent = parse_guest_agent($conf);
5016     my $agent = parse_guest_agent({$opt => $value});
5017
5018     for my $option (keys %$agent) { # added/changed options
5019         next if defined($hotplug_options->{$option});
5020         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5021     }
5022
5023     for my $option (keys %$old_agent) { # removed options
5024         next if defined($hotplug_options->{$option});
5025         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5026     }
5027
5028     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5029 }
5030
5031 sub vmconfig_update_disk {
5032     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5033
5034     my $drive = parse_drive($opt, $value);
5035
5036     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5037         my $media = $drive->{media} || 'disk';
5038         my $oldmedia = $old_drive->{media} || 'disk';
5039         die "unable to change media type\n" if $media ne $oldmedia;
5040
5041         if (!drive_is_cdrom($old_drive)) {
5042
5043             if ($drive->{file} ne $old_drive->{file}) {
5044
5045                 die "skip\n" if !$hotplug;
5046
5047                 # unplug and register as unused
5048                 vm_deviceunplug($vmid, $conf, $opt);
5049                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5050
5051             } else {
5052                 # update existing disk
5053
5054                 # skip non hotpluggable value
5055                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5056                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5057                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5058                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5059                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5060                     die "skip\n";
5061                 }
5062
5063                 # apply throttle
5064                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5065                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5066                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5067                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5068                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5069                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5070                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5071                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5072                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5073                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5074                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5075                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5076                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5077                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5078                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5079                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5080                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5081                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5082
5083                     qemu_block_set_io_throttle(
5084                         $vmid,"drive-$opt",
5085                         ($drive->{mbps} || 0)*1024*1024,
5086                         ($drive->{mbps_rd} || 0)*1024*1024,
5087                         ($drive->{mbps_wr} || 0)*1024*1024,
5088                          $drive->{iops} || 0,
5089                          $drive->{iops_rd} || 0,
5090                          $drive->{iops_wr} || 0,
5091                         ($drive->{mbps_max} || 0)*1024*1024,
5092                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5093                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5094                          $drive->{iops_max} || 0,
5095                          $drive->{iops_rd_max} || 0,
5096                          $drive->{iops_wr_max} || 0,
5097                          $drive->{bps_max_length} || 1,
5098                          $drive->{bps_rd_max_length} || 1,
5099                          $drive->{bps_wr_max_length} || 1,
5100                          $drive->{iops_max_length} || 1,
5101                          $drive->{iops_rd_max_length} || 1,
5102                          $drive->{iops_wr_max_length} || 1,
5103                     );
5104
5105                 }
5106
5107                 return 1;
5108             }
5109
5110         } else { # cdrom
5111
5112             if ($drive->{file} eq 'none') {
5113                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5114                 if (drive_is_cloudinit($old_drive)) {
5115                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5116                 }
5117             } else {
5118                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5119
5120                 # force eject if locked
5121                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5122
5123                 if ($path) {
5124                     mon_cmd($vmid, "blockdev-change-medium",
5125                         id => "$opt", filename => "$path");
5126                 }
5127             }
5128
5129             return 1;
5130         }
5131     }
5132
5133     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5134     # hotplug new disks
5135     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5136     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5137 }
5138
5139 # called in locked context by incoming migration
5140 sub vm_migrate_get_nbd_disks {
5141     my ($storecfg, $conf, $replicated_volumes) = @_;
5142
5143     my $local_volumes = {};
5144     PVE::QemuConfig->foreach_volume($conf, sub {
5145         my ($ds, $drive) = @_;
5146
5147         return if drive_is_cdrom($drive);
5148
5149         my $volid = $drive->{file};
5150
5151         return if !$volid;
5152
5153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5154
5155         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5156         return if $scfg->{shared};
5157
5158         # replicated disks re-use existing state via bitmap
5159         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5160         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5161     });
5162     return $local_volumes;
5163 }
5164
5165 # called in locked context by incoming migration
5166 sub vm_migrate_alloc_nbd_disks {
5167     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5168
5169     my $format = undef;
5170
5171     my $nbd = {};
5172     foreach my $opt (sort keys %$source_volumes) {
5173         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5174
5175         if ($use_existing) {
5176             $nbd->{$opt}->{drivestr} = print_drive($drive);
5177             $nbd->{$opt}->{volid} = $volid;
5178             $nbd->{$opt}->{replicated} = 1;
5179             next;
5180         }
5181
5182         # If a remote storage is specified and the format of the original
5183         # volume is not available there, fall back to the default format.
5184         # Otherwise use the same format as the original.
5185         if (!$storagemap->{identity}) {
5186             $storeid = map_storage($storagemap, $storeid);
5187             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5188             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5189             my $fileFormat = qemu_img_format($scfg, $volname);
5190             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5191         } else {
5192             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5193             $format = qemu_img_format($scfg, $volname);
5194         }
5195
5196         my $size = $drive->{size} / 1024;
5197         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5198         my $newdrive = $drive;
5199         $newdrive->{format} = $format;
5200         $newdrive->{file} = $newvolid;
5201         my $drivestr = print_drive($newdrive);
5202         $nbd->{$opt}->{drivestr} = $drivestr;
5203         $nbd->{$opt}->{volid} = $newvolid;
5204     }
5205
5206     return $nbd;
5207 }
5208
5209 # see vm_start_nolock for parameters, additionally:
5210 # migrate_opts:
5211 #   storagemap = parsed storage map for allocating NBD disks
5212 sub vm_start {
5213     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5214
5215     return PVE::QemuConfig->lock_config($vmid, sub {
5216         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5217
5218         die "you can't start a vm if it's a template\n"
5219             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5220
5221         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5222         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5223
5224         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5225
5226         if ($has_backup_lock && $running) {
5227             # a backup is currently running, attempt to start the guest in the
5228             # existing QEMU instance
5229             return vm_resume($vmid);
5230         }
5231
5232         PVE::QemuConfig->check_lock($conf)
5233             if !($params->{skiplock} || $has_suspended_lock);
5234
5235         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5236
5237         die "VM $vmid already running\n" if $running;
5238
5239         if (my $storagemap = $migrate_opts->{storagemap}) {
5240             my $replicated = $migrate_opts->{replicated_volumes};
5241             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5242             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5243
5244             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5245                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5246             }
5247         }
5248
5249         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5250     });
5251 }
5252
5253
5254 # params:
5255 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5256 #   skiplock => 0/1, skip checking for config lock
5257 #   skiptemplate => 0/1, skip checking whether VM is template
5258 #   forcemachine => to force Qemu machine (rollback/migration)
5259 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5260 #   timeout => in seconds
5261 #   paused => start VM in paused state (backup)
5262 #   resume => resume from hibernation
5263 #   pbs-backing => {
5264 #      sata0 => {
5265 #         repository
5266 #         snapshot
5267 #         keyfile
5268 #         archive
5269 #      },
5270 #      virtio2 => ...
5271 #   }
5272 # migrate_opts:
5273 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5274 #   migratedfrom => source node
5275 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5276 #   network => CIDR of migration network
5277 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5278 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5279 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5280 sub vm_start_nolock {
5281     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5282
5283     my $statefile = $params->{statefile};
5284     my $resume = $params->{resume};
5285
5286     my $migratedfrom = $migrate_opts->{migratedfrom};
5287     my $migration_type = $migrate_opts->{type};
5288
5289     my $res = {};
5290
5291     # clean up leftover reboot request files
5292     eval { clear_reboot_request($vmid); };
5293     warn $@ if $@;
5294
5295     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5296         vmconfig_apply_pending($vmid, $conf, $storecfg);
5297         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5298     }
5299
5300     # don't regenerate the ISO if the VM is started as part of a live migration
5301     # this way we can reuse the old ISO with the correct config
5302     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5303
5304     my $defaults = load_defaults();
5305
5306     # set environment variable useful inside network script
5307     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5308
5309     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5310
5311     my $forcemachine = $params->{forcemachine};
5312     my $forcecpu = $params->{forcecpu};
5313     if ($resume) {
5314         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5315         $forcemachine = $conf->{runningmachine};
5316         $forcecpu = $conf->{runningcpu};
5317         print "Resuming suspended VM\n";
5318     }
5319
5320     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5321         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5322
5323     my $migration_ip;
5324     my $get_migration_ip = sub {
5325         my ($nodename) = @_;
5326
5327         return $migration_ip if defined($migration_ip);
5328
5329         my $cidr = $migrate_opts->{network};
5330
5331         if (!defined($cidr)) {
5332             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5333             $cidr = $dc_conf->{migration}->{network};
5334         }
5335
5336         if (defined($cidr)) {
5337             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5338
5339             die "could not get IP: no address configured on local " .
5340                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5341
5342             die "could not get IP: multiple addresses configured on local " .
5343                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5344
5345             $migration_ip = @$ips[0];
5346         }
5347
5348         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5349             if !defined($migration_ip);
5350
5351         return $migration_ip;
5352     };
5353
5354     my $migrate_uri;
5355     if ($statefile) {
5356         if ($statefile eq 'tcp') {
5357             my $localip = "localhost";
5358             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5359             my $nodename = nodename();
5360
5361             if (!defined($migration_type)) {
5362                 if (defined($datacenterconf->{migration}->{type})) {
5363                     $migration_type = $datacenterconf->{migration}->{type};
5364                 } else {
5365                     $migration_type = 'secure';
5366                 }
5367             }
5368
5369             if ($migration_type eq 'insecure') {
5370                 $localip = $get_migration_ip->($nodename);
5371                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5372             }
5373
5374             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5375             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5376             $migrate_uri = "tcp:${localip}:${migrate_port}";
5377             push @$cmd, '-incoming', $migrate_uri;
5378             push @$cmd, '-S';
5379
5380         } elsif ($statefile eq 'unix') {
5381             # should be default for secure migrations as a ssh TCP forward
5382             # tunnel is not deterministic reliable ready and fails regurarly
5383             # to set up in time, so use UNIX socket forwards
5384             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5385             unlink $socket_addr;
5386
5387             $migrate_uri = "unix:$socket_addr";
5388
5389             push @$cmd, '-incoming', $migrate_uri;
5390             push @$cmd, '-S';
5391
5392         } elsif (-e $statefile) {
5393             push @$cmd, '-loadstate', $statefile;
5394         } else {
5395             my $statepath = PVE::Storage::path($storecfg, $statefile);
5396             push @$vollist, $statefile;
5397             push @$cmd, '-loadstate', $statepath;
5398         }
5399     } elsif ($params->{paused}) {
5400         push @$cmd, '-S';
5401     }
5402
5403     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5404
5405     my $pci_devices = {}; # host pci devices
5406     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5407         my $dev = $conf->{"hostpci$i"} or next;
5408         $pci_devices->{$i} = parse_hostpci($dev);
5409     }
5410
5411     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5412     # reserve all PCI IDs before actually doing anything with them
5413     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5414
5415     eval {
5416         for my $id (sort keys %$pci_devices) {
5417             my $d = $pci_devices->{$id};
5418             for my $dev ($d->{pciid}->@*) {
5419                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5420             }
5421         }
5422     };
5423     if (my $err = $@) {
5424         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5425         warn $@ if $@;
5426         die $err;
5427     }
5428
5429     PVE::Storage::activate_volumes($storecfg, $vollist);
5430
5431     eval {
5432         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5433     };
5434     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5435     # timeout should be more than enough here...
5436     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5437
5438     my $cpuunits = get_cpuunits($conf);
5439
5440     my %run_params = (
5441         timeout => $statefile ? undef : $start_timeout,
5442         umask => 0077,
5443         noerr => 1,
5444     );
5445
5446     # when migrating, prefix QEMU output so other side can pick up any
5447     # errors that might occur and show the user
5448     if ($migratedfrom) {
5449         $run_params{quiet} = 1;
5450         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5451     }
5452
5453     my %systemd_properties = (
5454         Slice => 'qemu.slice',
5455         KillMode => 'process',
5456         SendSIGKILL => 0,
5457         TimeoutStopUSec => ULONG_MAX, # infinity
5458     );
5459
5460     if (PVE::CGroup::cgroup_mode() == 2) {
5461         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5462         $systemd_properties{CPUWeight} = $cpuunits;
5463     } else {
5464         $systemd_properties{CPUShares} = $cpuunits;
5465     }
5466
5467     if (my $cpulimit = $conf->{cpulimit}) {
5468         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5469     }
5470     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5471
5472     my $run_qemu = sub {
5473         PVE::Tools::run_fork sub {
5474             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5475
5476             my $tpmpid;
5477             if (my $tpm = $conf->{tpmstate0}) {
5478                 # start the TPM emulator so QEMU can connect on start
5479                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5480             }
5481
5482             my $exitcode = run_command($cmd, %run_params);
5483             if ($exitcode) {
5484                 if ($tpmpid) {
5485                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5486                     kill 'TERM', $tpmpid;
5487                 }
5488                 die "QEMU exited with code $exitcode\n";
5489             }
5490         };
5491     };
5492
5493     if ($conf->{hugepages}) {
5494
5495         my $code = sub {
5496             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5497             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5498
5499             PVE::QemuServer::Memory::hugepages_mount();
5500             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5501
5502             eval { $run_qemu->() };
5503             if (my $err = $@) {
5504                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5505                     if !$conf->{keephugepages};
5506                 die $err;
5507             }
5508
5509             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5510                 if !$conf->{keephugepages};
5511         };
5512         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5513
5514     } else {
5515         eval { $run_qemu->() };
5516     }
5517
5518     if (my $err = $@) {
5519         # deactivate volumes if start fails
5520         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5521         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5522
5523         die "start failed: $err";
5524     }
5525
5526     # re-reserve all PCI IDs now that we can know the actual VM PID
5527     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5528     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5529     warn $@ if $@;
5530
5531     print "migration listens on $migrate_uri\n" if $migrate_uri;
5532     $res->{migrate_uri} = $migrate_uri;
5533
5534     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5535         eval { mon_cmd($vmid, "cont"); };
5536         warn $@ if $@;
5537     }
5538
5539     #start nbd server for storage migration
5540     if (my $nbd = $migrate_opts->{nbd}) {
5541         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5542
5543         my $migrate_storage_uri;
5544         # nbd_protocol_version > 0 for unix socket support
5545         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5546             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5547             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5548             $migrate_storage_uri = "nbd:unix:$socket_path";
5549         } else {
5550             my $nodename = nodename();
5551             my $localip = $get_migration_ip->($nodename);
5552             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5553             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5554
5555             mon_cmd($vmid, "nbd-server-start", addr => {
5556                 type => 'inet',
5557                 data => {
5558                     host => "${localip}",
5559                     port => "${storage_migrate_port}",
5560                 },
5561             });
5562             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5563             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5564         }
5565
5566         $res->{migrate_storage_uri} = $migrate_storage_uri;
5567
5568         foreach my $opt (sort keys %$nbd) {
5569             my $drivestr = $nbd->{$opt}->{drivestr};
5570             my $volid = $nbd->{$opt}->{volid};
5571             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5572             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5573             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5574             print "re-using replicated volume: $opt - $volid\n"
5575                 if $nbd->{$opt}->{replicated};
5576
5577             $res->{drives}->{$opt} = $nbd->{$opt};
5578             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5579         }
5580     }
5581
5582     if ($migratedfrom) {
5583         eval {
5584             set_migration_caps($vmid);
5585         };
5586         warn $@ if $@;
5587
5588         if ($spice_port) {
5589             print "spice listens on port $spice_port\n";
5590             $res->{spice_port} = $spice_port;
5591             if ($migrate_opts->{spice_ticket}) {
5592                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5593                     $migrate_opts->{spice_ticket});
5594                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5595             }
5596         }
5597
5598     } else {
5599         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5600             if !$statefile && $conf->{balloon};
5601
5602         foreach my $opt (keys %$conf) {
5603             next if $opt !~  m/^net\d+$/;
5604             my $nicconf = parse_net($conf->{$opt});
5605             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5606         }
5607     }
5608
5609     mon_cmd($vmid, 'qom-set',
5610                 path => "machine/peripheral/balloon0",
5611                 property => "guest-stats-polling-interval",
5612                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5613
5614     if ($resume) {
5615         print "Resumed VM, removing state\n";
5616         if (my $vmstate = $conf->{vmstate}) {
5617             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5618             PVE::Storage::vdisk_free($storecfg, $vmstate);
5619         }
5620         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5621         PVE::QemuConfig->write_config($vmid, $conf);
5622     }
5623
5624     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5625
5626     return $res;
5627 }
5628
5629 sub vm_commandline {
5630     my ($storecfg, $vmid, $snapname) = @_;
5631
5632     my $conf = PVE::QemuConfig->load_config($vmid);
5633
5634     my ($forcemachine, $forcecpu);
5635     if ($snapname) {
5636         my $snapshot = $conf->{snapshots}->{$snapname};
5637         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5638
5639         # check for machine or CPU overrides in snapshot
5640         $forcemachine = $snapshot->{runningmachine};
5641         $forcecpu = $snapshot->{runningcpu};
5642
5643         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5644
5645         $conf = $snapshot;
5646     }
5647
5648     my $defaults = load_defaults();
5649
5650     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5651
5652     return PVE::Tools::cmd2string($cmd);
5653 }
5654
5655 sub vm_reset {
5656     my ($vmid, $skiplock) = @_;
5657
5658     PVE::QemuConfig->lock_config($vmid, sub {
5659
5660         my $conf = PVE::QemuConfig->load_config($vmid);
5661
5662         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5663
5664         mon_cmd($vmid, "system_reset");
5665     });
5666 }
5667
5668 sub get_vm_volumes {
5669     my ($conf) = @_;
5670
5671     my $vollist = [];
5672     foreach_volid($conf, sub {
5673         my ($volid, $attr) = @_;
5674
5675         return if $volid =~ m|^/|;
5676
5677         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5678         return if !$sid;
5679
5680         push @$vollist, $volid;
5681     });
5682
5683     return $vollist;
5684 }
5685
5686 sub vm_stop_cleanup {
5687     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5688
5689     eval {
5690
5691         if (!$keepActive) {
5692             my $vollist = get_vm_volumes($conf);
5693             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5694
5695             if (my $tpmdrive = $conf->{tpmstate0}) {
5696                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5697                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5698                 if ($storeid) {
5699                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5700                 }
5701             }
5702         }
5703
5704         foreach my $ext (qw(mon qmp pid vnc qga)) {
5705             unlink "/var/run/qemu-server/${vmid}.$ext";
5706         }
5707
5708         if ($conf->{ivshmem}) {
5709             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5710             # just delete it for now, VMs which have this already open do not
5711             # are affected, but new VMs will get a separated one. If this
5712             # becomes an issue we either add some sort of ref-counting or just
5713             # add a "don't delete on stop" flag to the ivshmem format.
5714             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5715         }
5716
5717         my $ids = [];
5718         foreach my $key (keys %$conf) {
5719             next if $key !~ m/^hostpci(\d+)$/;
5720             my $hostpciindex = $1;
5721             my $d = parse_hostpci($conf->{$key});
5722             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5723
5724             foreach my $pci (@{$d->{pciid}}) {
5725                 my $pciid = $pci->{id};
5726                 push @$ids, $pci->{id};
5727                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5728             }
5729         }
5730         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5731
5732         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5733     };
5734     warn $@ if $@; # avoid errors - just warn
5735 }
5736
5737 # call only in locked context
5738 sub _do_vm_stop {
5739     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5740
5741     my $pid = check_running($vmid, $nocheck);
5742     return if !$pid;
5743
5744     my $conf;
5745     if (!$nocheck) {
5746         $conf = PVE::QemuConfig->load_config($vmid);
5747         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5748         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5749             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5750             $timeout = $opts->{down} if $opts->{down};
5751         }
5752         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5753     }
5754
5755     eval {
5756         if ($shutdown) {
5757             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5758                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5759             } else {
5760                 mon_cmd($vmid, "system_powerdown");
5761             }
5762         } else {
5763             mon_cmd($vmid, "quit");
5764         }
5765     };
5766     my $err = $@;
5767
5768     if (!$err) {
5769         $timeout = 60 if !defined($timeout);
5770
5771         my $count = 0;
5772         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5773             $count++;
5774             sleep 1;
5775         }
5776
5777         if ($count >= $timeout) {
5778             if ($force) {
5779                 warn "VM still running - terminating now with SIGTERM\n";
5780                 kill 15, $pid;
5781             } else {
5782                 die "VM quit/powerdown failed - got timeout\n";
5783             }
5784         } else {
5785             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5786             return;
5787         }
5788     } else {
5789        if (!check_running($vmid, $nocheck)) {
5790             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5791             return;
5792        }
5793        if ($force) {
5794             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5795             kill 15, $pid;
5796         } else {
5797             die "VM quit/powerdown failed\n";
5798         }
5799     }
5800
5801     # wait again
5802     $timeout = 10;
5803
5804     my $count = 0;
5805     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5806         $count++;
5807         sleep 1;
5808     }
5809
5810     if ($count >= $timeout) {
5811         warn "VM still running - terminating now with SIGKILL\n";
5812         kill 9, $pid;
5813         sleep 1;
5814     }
5815
5816     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5817 }
5818
5819 # Note: use $nocheck to skip tests if VM configuration file exists.
5820 # We need that when migration VMs to other nodes (files already moved)
5821 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5822 sub vm_stop {
5823     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5824
5825     $force = 1 if !defined($force) && !$shutdown;
5826
5827     if ($migratedfrom){
5828         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5829         kill 15, $pid if $pid;
5830         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5831         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5832         return;
5833     }
5834
5835     PVE::QemuConfig->lock_config($vmid, sub {
5836         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5837    });
5838 }
5839
5840 sub vm_reboot {
5841     my ($vmid, $timeout) = @_;
5842
5843     PVE::QemuConfig->lock_config($vmid, sub {
5844         eval {
5845
5846             # only reboot if running, as qmeventd starts it again on a stop event
5847             return if !check_running($vmid);
5848
5849             create_reboot_request($vmid);
5850
5851             my $storecfg = PVE::Storage::config();
5852             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5853
5854         };
5855         if (my $err = $@) {
5856             # avoid that the next normal shutdown will be confused for a reboot
5857             clear_reboot_request($vmid);
5858             die $err;
5859         }
5860    });
5861 }
5862
5863 # note: if using the statestorage parameter, the caller has to check privileges
5864 sub vm_suspend {
5865     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5866
5867     my $conf;
5868     my $path;
5869     my $storecfg;
5870     my $vmstate;
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873
5874         $conf = PVE::QemuConfig->load_config($vmid);
5875
5876         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5877         PVE::QemuConfig->check_lock($conf)
5878             if !($skiplock || $is_backing_up);
5879
5880         die "cannot suspend to disk during backup\n"
5881             if $is_backing_up && $includestate;
5882
5883         if ($includestate) {
5884             $conf->{lock} = 'suspending';
5885             my $date = strftime("%Y-%m-%d", localtime(time()));
5886             $storecfg = PVE::Storage::config();
5887             if (!$statestorage) {
5888                 $statestorage = find_vmstate_storage($conf, $storecfg);
5889                 # check permissions for the storage
5890                 my $rpcenv = PVE::RPCEnvironment::get();
5891                 if ($rpcenv->{type} ne 'cli') {
5892                     my $authuser = $rpcenv->get_user();
5893                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5894                 }
5895             }
5896
5897
5898             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5899                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5900             $path = PVE::Storage::path($storecfg, $vmstate);
5901             PVE::QemuConfig->write_config($vmid, $conf);
5902         } else {
5903             mon_cmd($vmid, "stop");
5904         }
5905     });
5906
5907     if ($includestate) {
5908         # save vm state
5909         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5910
5911         eval {
5912             set_migration_caps($vmid, 1);
5913             mon_cmd($vmid, "savevm-start", statefile => $path);
5914             for(;;) {
5915                 my $state = mon_cmd($vmid, "query-savevm");
5916                 if (!$state->{status}) {
5917                     die "savevm not active\n";
5918                 } elsif ($state->{status} eq 'active') {
5919                     sleep(1);
5920                     next;
5921                 } elsif ($state->{status} eq 'completed') {
5922                     print "State saved, quitting\n";
5923                     last;
5924                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5925                     die "query-savevm failed with error '$state->{error}'\n"
5926                 } else {
5927                     die "query-savevm returned status '$state->{status}'\n";
5928                 }
5929             }
5930         };
5931         my $err = $@;
5932
5933         PVE::QemuConfig->lock_config($vmid, sub {
5934             $conf = PVE::QemuConfig->load_config($vmid);
5935             if ($err) {
5936                 # cleanup, but leave suspending lock, to indicate something went wrong
5937                 eval {
5938                     mon_cmd($vmid, "savevm-end");
5939                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5940                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5941                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5942                     PVE::QemuConfig->write_config($vmid, $conf);
5943                 };
5944                 warn $@ if $@;
5945                 die $err;
5946             }
5947
5948             die "lock changed unexpectedly\n"
5949                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5950
5951             mon_cmd($vmid, "quit");
5952             $conf->{lock} = 'suspended';
5953             PVE::QemuConfig->write_config($vmid, $conf);
5954         });
5955     }
5956 }
5957
5958 sub vm_resume {
5959     my ($vmid, $skiplock, $nocheck) = @_;
5960
5961     PVE::QemuConfig->lock_config($vmid, sub {
5962         my $res = mon_cmd($vmid, 'query-status');
5963         my $resume_cmd = 'cont';
5964         my $reset = 0;
5965
5966         if ($res->{status}) {
5967             return if $res->{status} eq 'running'; # job done, go home
5968             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5969             $reset = 1 if $res->{status} eq 'shutdown';
5970         }
5971
5972         if (!$nocheck) {
5973
5974             my $conf = PVE::QemuConfig->load_config($vmid);
5975
5976             PVE::QemuConfig->check_lock($conf)
5977                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5978         }
5979
5980         if ($reset) {
5981             # required if a VM shuts down during a backup and we get a resume
5982             # request before the backup finishes for example
5983             mon_cmd($vmid, "system_reset");
5984         }
5985         mon_cmd($vmid, $resume_cmd);
5986     });
5987 }
5988
5989 sub vm_sendkey {
5990     my ($vmid, $skiplock, $key) = @_;
5991
5992     PVE::QemuConfig->lock_config($vmid, sub {
5993
5994         my $conf = PVE::QemuConfig->load_config($vmid);
5995
5996         # there is no qmp command, so we use the human monitor command
5997         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5998         die $res if $res ne '';
5999     });
6000 }
6001
6002 # vzdump restore implementaion
6003
6004 sub tar_archive_read_firstfile {
6005     my $archive = shift;
6006
6007     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6008
6009     # try to detect archive type first
6010     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6011         die "unable to open file '$archive'\n";
6012     my $firstfile = <$fh>;
6013     kill 15, $pid;
6014     close $fh;
6015
6016     die "ERROR: archive contaions no data\n" if !$firstfile;
6017     chomp $firstfile;
6018
6019     return $firstfile;
6020 }
6021
6022 sub tar_restore_cleanup {
6023     my ($storecfg, $statfile) = @_;
6024
6025     print STDERR "starting cleanup\n";
6026
6027     if (my $fd = IO::File->new($statfile, "r")) {
6028         while (defined(my $line = <$fd>)) {
6029             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6030                 my $volid = $2;
6031                 eval {
6032                     if ($volid =~ m|^/|) {
6033                         unlink $volid || die 'unlink failed\n';
6034                     } else {
6035                         PVE::Storage::vdisk_free($storecfg, $volid);
6036                     }
6037                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6038                 };
6039                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6040             } else {
6041                 print STDERR "unable to parse line in statfile - $line";
6042             }
6043         }
6044         $fd->close();
6045     }
6046 }
6047
6048 sub restore_file_archive {
6049     my ($archive, $vmid, $user, $opts) = @_;
6050
6051     return restore_vma_archive($archive, $vmid, $user, $opts)
6052         if $archive eq '-';
6053
6054     my $info = PVE::Storage::archive_info($archive);
6055     my $format = $opts->{format} // $info->{format};
6056     my $comp = $info->{compression};
6057
6058     # try to detect archive format
6059     if ($format eq 'tar') {
6060         return restore_tar_archive($archive, $vmid, $user, $opts);
6061     } else {
6062         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6063     }
6064 }
6065
6066 # hepler to remove disks that will not be used after restore
6067 my $restore_cleanup_oldconf = sub {
6068     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6069
6070     PVE::QemuConfig->foreach_volume($oldconf, sub {
6071         my ($ds, $drive) = @_;
6072
6073         return if drive_is_cdrom($drive, 1);
6074
6075         my $volid = $drive->{file};
6076         return if !$volid || $volid =~ m|^/|;
6077
6078         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6079         return if !$path || !$owner || ($owner != $vmid);
6080
6081         # Note: only delete disk we want to restore
6082         # other volumes will become unused
6083         if ($virtdev_hash->{$ds}) {
6084             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6085             if (my $err = $@) {
6086                 warn $err;
6087             }
6088         }
6089     });
6090
6091     # delete vmstate files, after the restore we have no snapshots anymore
6092     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6093         my $snap = $oldconf->{snapshots}->{$snapname};
6094         if ($snap->{vmstate}) {
6095             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6096             if (my $err = $@) {
6097                 warn $err;
6098             }
6099         }
6100     }
6101 };
6102
6103 # Helper to parse vzdump backup device hints
6104 #
6105 # $rpcenv: Environment, used to ckeck storage permissions
6106 # $user: User ID, to check storage permissions
6107 # $storecfg: Storage configuration
6108 # $fh: the file handle for reading the configuration
6109 # $devinfo: should contain device sizes for all backu-up'ed devices
6110 # $options: backup options (pool, default storage)
6111 #
6112 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6113 my $parse_backup_hints = sub {
6114     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6115
6116     my $virtdev_hash = {};
6117
6118     while (defined(my $line = <$fh>)) {
6119         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6120             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6121             die "archive does not contain data for drive '$virtdev'\n"
6122                 if !$devinfo->{$devname};
6123
6124             if (defined($options->{storage})) {
6125                 $storeid = $options->{storage} || 'local';
6126             } elsif (!$storeid) {
6127                 $storeid = 'local';
6128             }
6129             $format = 'raw' if !$format;
6130             $devinfo->{$devname}->{devname} = $devname;
6131             $devinfo->{$devname}->{virtdev} = $virtdev;
6132             $devinfo->{$devname}->{format} = $format;
6133             $devinfo->{$devname}->{storeid} = $storeid;
6134
6135             # check permission on storage
6136             my $pool = $options->{pool}; # todo: do we need that?
6137             if ($user ne 'root@pam') {
6138                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6139             }
6140
6141             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6142         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6143             my $virtdev = $1;
6144             my $drive = parse_drive($virtdev, $2);
6145             if (drive_is_cloudinit($drive)) {
6146                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6147                 $storeid = $options->{storage} if defined ($options->{storage});
6148                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6149                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6150
6151                 $virtdev_hash->{$virtdev} = {
6152                     format => $format,
6153                     storeid => $storeid,
6154                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6155                     is_cloudinit => 1,
6156                 };
6157             }
6158         }
6159     }
6160
6161     return $virtdev_hash;
6162 };
6163
6164 # Helper to allocate and activate all volumes required for a restore
6165 #
6166 # $storecfg: Storage configuration
6167 # $virtdev_hash: as returned by parse_backup_hints()
6168 #
6169 # Returns: { $virtdev => $volid }
6170 my $restore_allocate_devices = sub {
6171     my ($storecfg, $virtdev_hash, $vmid) = @_;
6172
6173     my $map = {};
6174     foreach my $virtdev (sort keys %$virtdev_hash) {
6175         my $d = $virtdev_hash->{$virtdev};
6176         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6177         my $storeid = $d->{storeid};
6178         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6179
6180         # test if requested format is supported
6181         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6182         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6183         $d->{format} = $defFormat if !$supported;
6184
6185         my $name;
6186         if ($d->{is_cloudinit}) {
6187             $name = "vm-$vmid-cloudinit";
6188             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6189             if ($scfg->{path}) {
6190                 $name .= ".$d->{format}";
6191             }
6192         }
6193
6194         my $volid = PVE::Storage::vdisk_alloc(
6195             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6196
6197         print STDERR "new volume ID is '$volid'\n";
6198         $d->{volid} = $volid;
6199
6200         PVE::Storage::activate_volumes($storecfg, [$volid]);
6201
6202         $map->{$virtdev} = $volid;
6203     }
6204
6205     return $map;
6206 };
6207
6208 sub restore_update_config_line {
6209     my ($cookie, $map, $line, $unique) = @_;
6210
6211     return '' if $line =~ m/^\#qmdump\#/;
6212     return '' if $line =~ m/^\#vzdump\#/;
6213     return '' if $line =~ m/^lock:/;
6214     return '' if $line =~ m/^unused\d+:/;
6215     return '' if $line =~ m/^parent:/;
6216
6217     my $res = '';
6218
6219     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6220     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6221         # try to convert old 1.X settings
6222         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6223         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6224             my ($model, $macaddr) = split(/\=/, $devconfig);
6225             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6226             my $net = {
6227                 model => $model,
6228                 bridge => "vmbr$ind",
6229                 macaddr => $macaddr,
6230             };
6231             my $netstr = print_net($net);
6232
6233             $res .= "net$cookie->{netcount}: $netstr\n";
6234             $cookie->{netcount}++;
6235         }
6236     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6237         my ($id, $netstr) = ($1, $2);
6238         my $net = parse_net($netstr);
6239         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6240         $netstr = print_net($net);
6241         $res .= "$id: $netstr\n";
6242     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6243         my $virtdev = $1;
6244         my $value = $3;
6245         my $di = parse_drive($virtdev, $value);
6246         if (defined($di->{backup}) && !$di->{backup}) {
6247             $res .= "#$line";
6248         } elsif ($map->{$virtdev}) {
6249             delete $di->{format}; # format can change on restore
6250             $di->{file} = $map->{$virtdev};
6251             $value = print_drive($di);
6252             $res .= "$virtdev: $value\n";
6253         } else {
6254             $res .= $line;
6255         }
6256     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6257         my $vmgenid = $1;
6258         if ($vmgenid ne '0') {
6259             # always generate a new vmgenid if there was a valid one setup
6260             $vmgenid = generate_uuid();
6261         }
6262         $res .= "vmgenid: $vmgenid\n";
6263     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6264         my ($uuid, $uuid_str);
6265         UUID::generate($uuid);
6266         UUID::unparse($uuid, $uuid_str);
6267         my $smbios1 = parse_smbios1($2);
6268         $smbios1->{uuid} = $uuid_str;
6269         $res .= $1.print_smbios1($smbios1)."\n";
6270     } else {
6271         $res .= $line;
6272     }
6273
6274     return $res;
6275 }
6276
6277 my $restore_deactivate_volumes = sub {
6278     my ($storecfg, $devinfo) = @_;
6279
6280     my $vollist = [];
6281     foreach my $devname (keys %$devinfo) {
6282         my $volid = $devinfo->{$devname}->{volid};
6283         push @$vollist, $volid if $volid;
6284     }
6285
6286     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6287 };
6288
6289 my $restore_destroy_volumes = sub {
6290     my ($storecfg, $devinfo) = @_;
6291
6292     foreach my $devname (keys %$devinfo) {
6293         my $volid = $devinfo->{$devname}->{volid};
6294         next if !$volid;
6295         eval {
6296             if ($volid =~ m|^/|) {
6297                 unlink $volid || die 'unlink failed\n';
6298             } else {
6299                 PVE::Storage::vdisk_free($storecfg, $volid);
6300             }
6301             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6302         };
6303         print STDERR "unable to cleanup '$volid' - $@" if $@;
6304     }
6305 };
6306
6307 sub scan_volids {
6308     my ($cfg, $vmid) = @_;
6309
6310     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6311
6312     my $volid_hash = {};
6313     foreach my $storeid (keys %$info) {
6314         foreach my $item (@{$info->{$storeid}}) {
6315             next if !($item->{volid} && $item->{size});
6316             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6317             $volid_hash->{$item->{volid}} = $item;
6318         }
6319     }
6320
6321     return $volid_hash;
6322 }
6323
6324 sub update_disk_config {
6325     my ($vmid, $conf, $volid_hash) = @_;
6326
6327     my $changes;
6328     my $prefix = "VM $vmid";
6329
6330     # used and unused disks
6331     my $referenced = {};
6332
6333     # Note: it is allowed to define multiple storages with same path (alias), so
6334     # we need to check both 'volid' and real 'path' (two different volid can point
6335     # to the same path).
6336
6337     my $referencedpath = {};
6338
6339     # update size info
6340     PVE::QemuConfig->foreach_volume($conf, sub {
6341         my ($opt, $drive) = @_;
6342
6343         my $volid = $drive->{file};
6344         return if !$volid;
6345         my $volume = $volid_hash->{$volid};
6346
6347         # mark volid as "in-use" for next step
6348         $referenced->{$volid} = 1;
6349         if ($volume && (my $path = $volume->{path})) {
6350             $referencedpath->{$path} = 1;
6351         }
6352
6353         return if drive_is_cdrom($drive);
6354         return if !$volume;
6355
6356         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6357         if (defined($updated)) {
6358             $changes = 1;
6359             $conf->{$opt} = print_drive($updated);
6360             print "$prefix ($opt): $msg\n";
6361         }
6362     });
6363
6364     # remove 'unusedX' entry if volume is used
6365     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6366         my ($opt, $drive) = @_;
6367
6368         my $volid = $drive->{file};
6369         return if !$volid;
6370
6371         my $path;
6372         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6373         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6374             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6375             $changes = 1;
6376             delete $conf->{$opt};
6377         }
6378
6379         $referenced->{$volid} = 1;
6380         $referencedpath->{$path} = 1 if $path;
6381     });
6382
6383     foreach my $volid (sort keys %$volid_hash) {
6384         next if $volid =~ m/vm-$vmid-state-/;
6385         next if $referenced->{$volid};
6386         my $path = $volid_hash->{$volid}->{path};
6387         next if !$path; # just to be sure
6388         next if $referencedpath->{$path};
6389         $changes = 1;
6390         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6391         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6392         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6393     }
6394
6395     return $changes;
6396 }
6397
6398 sub rescan {
6399     my ($vmid, $nolock, $dryrun) = @_;
6400
6401     my $cfg = PVE::Storage::config();
6402
6403     print "rescan volumes...\n";
6404     my $volid_hash = scan_volids($cfg, $vmid);
6405
6406     my $updatefn =  sub {
6407         my ($vmid) = @_;
6408
6409         my $conf = PVE::QemuConfig->load_config($vmid);
6410
6411         PVE::QemuConfig->check_lock($conf);
6412
6413         my $vm_volids = {};
6414         foreach my $volid (keys %$volid_hash) {
6415             my $info = $volid_hash->{$volid};
6416             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6417         }
6418
6419         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6420
6421         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6422     };
6423
6424     if (defined($vmid)) {
6425         if ($nolock) {
6426             &$updatefn($vmid);
6427         } else {
6428             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6429         }
6430     } else {
6431         my $vmlist = config_list();
6432         foreach my $vmid (keys %$vmlist) {
6433             if ($nolock) {
6434                 &$updatefn($vmid);
6435             } else {
6436                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6437             }
6438         }
6439     }
6440 }
6441
6442 sub restore_proxmox_backup_archive {
6443     my ($archive, $vmid, $user, $options) = @_;
6444
6445     my $storecfg = PVE::Storage::config();
6446
6447     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6448     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6449
6450     my $fingerprint = $scfg->{fingerprint};
6451     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6452
6453     my $repo = PVE::PBSClient::get_repository($scfg);
6454
6455     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6456     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6457     local $ENV{PBS_PASSWORD} = $password;
6458     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6459
6460     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6461         PVE::Storage::parse_volname($storecfg, $archive);
6462
6463     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6464
6465     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6466
6467     my $tmpdir = "/var/tmp/vzdumptmp$$";
6468     rmtree $tmpdir;
6469     mkpath $tmpdir;
6470
6471     my $conffile = PVE::QemuConfig->config_file($vmid);
6472      # disable interrupts (always do cleanups)
6473     local $SIG{INT} =
6474         local $SIG{TERM} =
6475         local $SIG{QUIT} =
6476         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6477
6478     # Note: $oldconf is undef if VM does not exists
6479     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6480     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6481     my $new_conf_raw = '';
6482
6483     my $rpcenv = PVE::RPCEnvironment::get();
6484     my $devinfo = {};
6485
6486     eval {
6487         # enable interrupts
6488         local $SIG{INT} =
6489             local $SIG{TERM} =
6490             local $SIG{QUIT} =
6491             local $SIG{HUP} =
6492             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6493
6494         my $cfgfn = "$tmpdir/qemu-server.conf";
6495         my $firewall_config_fn = "$tmpdir/fw.conf";
6496         my $index_fn = "$tmpdir/index.json";
6497
6498         my $cmd = "restore";
6499
6500         my $param = [$pbs_backup_name, "index.json", $index_fn];
6501         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6502         my $index = PVE::Tools::file_get_contents($index_fn);
6503         $index = decode_json($index);
6504
6505         # print Dumper($index);
6506         foreach my $info (@{$index->{files}}) {
6507             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6508                 my $devname = $1;
6509                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6510                     $devinfo->{$devname}->{size} = $1;
6511                 } else {
6512                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6513                 }
6514             }
6515         }
6516
6517         my $is_qemu_server_backup = scalar(
6518             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6519         );
6520         if (!$is_qemu_server_backup) {
6521             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6522         }
6523         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6524
6525         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6526         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6527
6528         if ($has_firewall_config) {
6529             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6530             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6531
6532             my $pve_firewall_dir = '/etc/pve/firewall';
6533             mkdir $pve_firewall_dir; # make sure the dir exists
6534             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6535         }
6536
6537         my $fh = IO::File->new($cfgfn, "r") ||
6538             die "unable to read qemu-server.conf - $!\n";
6539
6540         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6541
6542         # fixme: rate limit?
6543
6544         # create empty/temp config
6545         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6546
6547         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6548
6549         # allocate volumes
6550         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6551
6552         foreach my $virtdev (sort keys %$virtdev_hash) {
6553             my $d = $virtdev_hash->{$virtdev};
6554             next if $d->{is_cloudinit}; # no need to restore cloudinit
6555
6556             # this fails if storage is unavailable
6557             my $volid = $d->{volid};
6558             my $path = PVE::Storage::path($storecfg, $volid);
6559
6560             # for live-restore we only want to preload the efidisk and TPM state
6561             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6562
6563             my $pbs_restore_cmd = [
6564                 '/usr/bin/pbs-restore',
6565                 '--repository', $repo,
6566                 $pbs_backup_name,
6567                 "$d->{devname}.img.fidx",
6568                 $path,
6569                 '--verbose',
6570                 ];
6571
6572             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6573             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6574
6575             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6576                 push @$pbs_restore_cmd, '--skip-zero';
6577             }
6578
6579             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6580             print "restore proxmox backup image: $dbg_cmdstring\n";
6581             run_command($pbs_restore_cmd);
6582         }
6583
6584         $fh->seek(0, 0) || die "seek failed - $!\n";
6585
6586         my $cookie = { netcount => 0 };
6587         while (defined(my $line = <$fh>)) {
6588             $new_conf_raw .= restore_update_config_line(
6589                 $cookie,
6590                 $map,
6591                 $line,
6592                 $options->{unique},
6593             );
6594         }
6595
6596         $fh->close();
6597     };
6598     my $err = $@;
6599
6600     if ($err || !$options->{live}) {
6601         $restore_deactivate_volumes->($storecfg, $devinfo);
6602     }
6603
6604     rmtree $tmpdir;
6605
6606     if ($err) {
6607         $restore_destroy_volumes->($storecfg, $devinfo);
6608         die $err;
6609     }
6610
6611     if ($options->{live}) {
6612         # keep lock during live-restore
6613         $new_conf_raw .= "\nlock: create";
6614     }
6615
6616     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6617
6618     PVE::Cluster::cfs_update(); # make sure we read new file
6619
6620     eval { rescan($vmid, 1); };
6621     warn $@ if $@;
6622
6623     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6624
6625     if ($options->{live}) {
6626         # enable interrupts
6627         local $SIG{INT} =
6628             local $SIG{TERM} =
6629             local $SIG{QUIT} =
6630             local $SIG{HUP} =
6631             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6632
6633         my $conf = PVE::QemuConfig->load_config($vmid);
6634         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6635
6636         # these special drives are already restored before start
6637         delete $devinfo->{'drive-efidisk0'};
6638         delete $devinfo->{'drive-tpmstate0-backup'};
6639         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6640
6641         PVE::QemuConfig->remove_lock($vmid, "create");
6642     }
6643 }
6644
6645 sub pbs_live_restore {
6646     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6647
6648     print "starting VM for live-restore\n";
6649     print "repository: '$repo', snapshot: '$snap'\n";
6650
6651     my $pbs_backing = {};
6652     for my $ds (keys %$restored_disks) {
6653         $ds =~ m/^drive-(.*)$/;
6654         my $confname = $1;
6655         $pbs_backing->{$confname} = {
6656             repository => $repo,
6657             snapshot => $snap,
6658             archive => "$ds.img.fidx",
6659         };
6660         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6661
6662         my $drive = parse_drive($confname, $conf->{$confname});
6663         print "restoring '$ds' to '$drive->{file}'\n";
6664     }
6665
6666     my $drives_streamed = 0;
6667     eval {
6668         # make sure HA doesn't interrupt our restore by stopping the VM
6669         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6670             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6671         }
6672
6673         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6674         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6675         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6676
6677         my $qmeventd_fd = register_qmeventd_handle($vmid);
6678
6679         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6680         # this will effectively collapse the backing image chain consisting of
6681         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6682         # removes itself once all backing images vanish with 'auto-remove=on')
6683         my $jobs = {};
6684         for my $ds (sort keys %$restored_disks) {
6685             my $job_id = "restore-$ds";
6686             mon_cmd($vmid, 'block-stream',
6687                 'job-id' => $job_id,
6688                 device => "$ds",
6689             );
6690             $jobs->{$job_id} = {};
6691         }
6692
6693         mon_cmd($vmid, 'cont');
6694         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6695
6696         print "restore-drive jobs finished successfully, removing all tracking block devices"
6697             ." to disconnect from Proxmox Backup Server\n";
6698
6699         for my $ds (sort keys %$restored_disks) {
6700             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6701         }
6702
6703         close($qmeventd_fd);
6704     };
6705
6706     my $err = $@;
6707
6708     if ($err) {
6709         warn "An error occured during live-restore: $err\n";
6710         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6711         die "live-restore failed\n";
6712     }
6713 }
6714
6715 sub restore_vma_archive {
6716     my ($archive, $vmid, $user, $opts, $comp) = @_;
6717
6718     my $readfrom = $archive;
6719
6720     my $cfg = PVE::Storage::config();
6721     my $commands = [];
6722     my $bwlimit = $opts->{bwlimit};
6723
6724     my $dbg_cmdstring = '';
6725     my $add_pipe = sub {
6726         my ($cmd) = @_;
6727         push @$commands, $cmd;
6728         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6729         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6730         $readfrom = '-';
6731     };
6732
6733     my $input = undef;
6734     if ($archive eq '-') {
6735         $input = '<&STDIN';
6736     } else {
6737         # If we use a backup from a PVE defined storage we also consider that
6738         # storage's rate limit:
6739         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6740         if (defined($volid)) {
6741             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6742             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6743             if ($readlimit) {
6744                 print STDERR "applying read rate limit: $readlimit\n";
6745                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6746                 $add_pipe->($cstream);
6747             }
6748         }
6749     }
6750
6751     if ($comp) {
6752         my $info = PVE::Storage::decompressor_info('vma', $comp);
6753         my $cmd = $info->{decompressor};
6754         push @$cmd, $readfrom;
6755         $add_pipe->($cmd);
6756     }
6757
6758     my $tmpdir = "/var/tmp/vzdumptmp$$";
6759     rmtree $tmpdir;
6760
6761     # disable interrupts (always do cleanups)
6762     local $SIG{INT} =
6763         local $SIG{TERM} =
6764         local $SIG{QUIT} =
6765         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6766
6767     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6768     POSIX::mkfifo($mapfifo, 0600);
6769     my $fifofh;
6770     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6771
6772     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6773
6774     my $oldtimeout;
6775     my $timeout = 5;
6776
6777     my $devinfo = {};
6778
6779     my $rpcenv = PVE::RPCEnvironment::get();
6780
6781     my $conffile = PVE::QemuConfig->config_file($vmid);
6782
6783     # Note: $oldconf is undef if VM does not exist
6784     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6785     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6786     my $new_conf_raw = '';
6787
6788     my %storage_limits;
6789
6790     my $print_devmap = sub {
6791         my $cfgfn = "$tmpdir/qemu-server.conf";
6792
6793         # we can read the config - that is already extracted
6794         my $fh = IO::File->new($cfgfn, "r") ||
6795             die "unable to read qemu-server.conf - $!\n";
6796
6797         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6798         if (-f $fwcfgfn) {
6799             my $pve_firewall_dir = '/etc/pve/firewall';
6800             mkdir $pve_firewall_dir; # make sure the dir exists
6801             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6802         }
6803
6804         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6805
6806         foreach my $info (values %{$virtdev_hash}) {
6807             my $storeid = $info->{storeid};
6808             next if defined($storage_limits{$storeid});
6809
6810             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6811             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6812             $storage_limits{$storeid} = $limit * 1024;
6813         }
6814
6815         foreach my $devname (keys %$devinfo) {
6816             die "found no device mapping information for device '$devname'\n"
6817                 if !$devinfo->{$devname}->{virtdev};
6818         }
6819
6820         # create empty/temp config
6821         if ($oldconf) {
6822             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6823             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6824         }
6825
6826         # allocate volumes
6827         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6828
6829         # print restore information to $fifofh
6830         foreach my $virtdev (sort keys %$virtdev_hash) {
6831             my $d = $virtdev_hash->{$virtdev};
6832             next if $d->{is_cloudinit}; # no need to restore cloudinit
6833
6834             my $storeid = $d->{storeid};
6835             my $volid = $d->{volid};
6836
6837             my $map_opts = '';
6838             if (my $limit = $storage_limits{$storeid}) {
6839                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6840             }
6841
6842             my $write_zeros = 1;
6843             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6844                 $write_zeros = 0;
6845             }
6846
6847             my $path = PVE::Storage::path($cfg, $volid);
6848
6849             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6850
6851             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6852         }
6853
6854         $fh->seek(0, 0) || die "seek failed - $!\n";
6855
6856         my $cookie = { netcount => 0 };
6857         while (defined(my $line = <$fh>)) {
6858             $new_conf_raw .= restore_update_config_line(
6859                 $cookie,
6860                 $map,
6861                 $line,
6862                 $opts->{unique},
6863             );
6864         }
6865
6866         $fh->close();
6867     };
6868
6869     eval {
6870         # enable interrupts
6871         local $SIG{INT} =
6872             local $SIG{TERM} =
6873             local $SIG{QUIT} =
6874             local $SIG{HUP} =
6875             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6876         local $SIG{ALRM} = sub { die "got timeout\n"; };
6877
6878         $oldtimeout = alarm($timeout);
6879
6880         my $parser = sub {
6881             my $line = shift;
6882
6883             print "$line\n";
6884
6885             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6886                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6887                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6888             } elsif ($line =~ m/^CTIME: /) {
6889                 # we correctly received the vma config, so we can disable
6890                 # the timeout now for disk allocation (set to 10 minutes, so
6891                 # that we always timeout if something goes wrong)
6892                 alarm(600);
6893                 &$print_devmap();
6894                 print $fifofh "done\n";
6895                 my $tmp = $oldtimeout || 0;
6896                 $oldtimeout = undef;
6897                 alarm($tmp);
6898                 close($fifofh);
6899                 $fifofh = undef;
6900             }
6901         };
6902
6903         print "restore vma archive: $dbg_cmdstring\n";
6904         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6905     };
6906     my $err = $@;
6907
6908     alarm($oldtimeout) if $oldtimeout;
6909
6910     $restore_deactivate_volumes->($cfg, $devinfo);
6911
6912     close($fifofh) if $fifofh;
6913     unlink $mapfifo;
6914     rmtree $tmpdir;
6915
6916     if ($err) {
6917         $restore_destroy_volumes->($cfg, $devinfo);
6918         die $err;
6919     }
6920
6921     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6922
6923     PVE::Cluster::cfs_update(); # make sure we read new file
6924
6925     eval { rescan($vmid, 1); };
6926     warn $@ if $@;
6927
6928     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6929 }
6930
6931 sub restore_tar_archive {
6932     my ($archive, $vmid, $user, $opts) = @_;
6933
6934     if ($archive ne '-') {
6935         my $firstfile = tar_archive_read_firstfile($archive);
6936         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6937             if $firstfile ne 'qemu-server.conf';
6938     }
6939
6940     my $storecfg = PVE::Storage::config();
6941
6942     # avoid zombie disks when restoring over an existing VM -> cleanup first
6943     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6944     # skiplock=1 because qmrestore has set the 'create' lock itself already
6945     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6946     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6947
6948     my $tocmd = "/usr/lib/qemu-server/qmextract";
6949
6950     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6951     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6952     $tocmd .= ' --prealloc' if $opts->{prealloc};
6953     $tocmd .= ' --info' if $opts->{info};
6954
6955     # tar option "xf" does not autodetect compression when read from STDIN,
6956     # so we pipe to zcat
6957     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6958         PVE::Tools::shellquote("--to-command=$tocmd");
6959
6960     my $tmpdir = "/var/tmp/vzdumptmp$$";
6961     mkpath $tmpdir;
6962
6963     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6964     local $ENV{VZDUMP_VMID} = $vmid;
6965     local $ENV{VZDUMP_USER} = $user;
6966
6967     my $conffile = PVE::QemuConfig->config_file($vmid);
6968     my $new_conf_raw = '';
6969
6970     # disable interrupts (always do cleanups)
6971     local $SIG{INT} =
6972         local $SIG{TERM} =
6973         local $SIG{QUIT} =
6974         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6975
6976     eval {
6977         # enable interrupts
6978         local $SIG{INT} =
6979             local $SIG{TERM} =
6980             local $SIG{QUIT} =
6981             local $SIG{HUP} =
6982             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6983
6984         if ($archive eq '-') {
6985             print "extracting archive from STDIN\n";
6986             run_command($cmd, input => "<&STDIN");
6987         } else {
6988             print "extracting archive '$archive'\n";
6989             run_command($cmd);
6990         }
6991
6992         return if $opts->{info};
6993
6994         # read new mapping
6995         my $map = {};
6996         my $statfile = "$tmpdir/qmrestore.stat";
6997         if (my $fd = IO::File->new($statfile, "r")) {
6998             while (defined (my $line = <$fd>)) {
6999                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7000                     $map->{$1} = $2 if $1;
7001                 } else {
7002                     print STDERR "unable to parse line in statfile - $line\n";
7003                 }
7004             }
7005             $fd->close();
7006         }
7007
7008         my $confsrc = "$tmpdir/qemu-server.conf";
7009
7010         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7011
7012         my $cookie = { netcount => 0 };
7013         while (defined (my $line = <$srcfd>)) {
7014             $new_conf_raw .= restore_update_config_line(
7015                 $cookie,
7016                 $map,
7017                 $line,
7018                 $opts->{unique},
7019             );
7020         }
7021
7022         $srcfd->close();
7023     };
7024     if (my $err = $@) {
7025         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7026         die $err;
7027     }
7028
7029     rmtree $tmpdir;
7030
7031     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7032
7033     PVE::Cluster::cfs_update(); # make sure we read new file
7034
7035     eval { rescan($vmid, 1); };
7036     warn $@ if $@;
7037 };
7038
7039 sub foreach_storage_used_by_vm {
7040     my ($conf, $func) = @_;
7041
7042     my $sidhash = {};
7043
7044     PVE::QemuConfig->foreach_volume($conf, sub {
7045         my ($ds, $drive) = @_;
7046         return if drive_is_cdrom($drive);
7047
7048         my $volid = $drive->{file};
7049
7050         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7051         $sidhash->{$sid} = $sid if $sid;
7052     });
7053
7054     foreach my $sid (sort keys %$sidhash) {
7055         &$func($sid);
7056     }
7057 }
7058
7059 my $qemu_snap_storage = {
7060     rbd => 1,
7061 };
7062 sub do_snapshots_with_qemu {
7063     my ($storecfg, $volid, $deviceid) = @_;
7064
7065     return if $deviceid =~ m/tpmstate0/;
7066
7067     my $storage_name = PVE::Storage::parse_volume_id($volid);
7068     my $scfg = $storecfg->{ids}->{$storage_name};
7069     die "could not find storage '$storage_name'\n" if !defined($scfg);
7070
7071     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7072         return 1;
7073     }
7074
7075     if ($volid =~ m/\.(qcow2|qed)$/){
7076         return 1;
7077     }
7078
7079     return;
7080 }
7081
7082 sub qga_check_running {
7083     my ($vmid, $nowarn) = @_;
7084
7085     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7086     if ($@) {
7087         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7088         return 0;
7089     }
7090     return 1;
7091 }
7092
7093 sub template_create {
7094     my ($vmid, $conf, $disk) = @_;
7095
7096     my $storecfg = PVE::Storage::config();
7097
7098     PVE::QemuConfig->foreach_volume($conf, sub {
7099         my ($ds, $drive) = @_;
7100
7101         return if drive_is_cdrom($drive);
7102         return if $disk && $ds ne $disk;
7103
7104         my $volid = $drive->{file};
7105         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7106
7107         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7108         $drive->{file} = $voliddst;
7109         $conf->{$ds} = print_drive($drive);
7110         PVE::QemuConfig->write_config($vmid, $conf);
7111     });
7112 }
7113
7114 sub convert_iscsi_path {
7115     my ($path) = @_;
7116
7117     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7118         my $portal = $1;
7119         my $target = $2;
7120         my $lun = $3;
7121
7122         my $initiator_name = get_initiator_name();
7123
7124         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7125                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7126     }
7127
7128     die "cannot convert iscsi path '$path', unkown format\n";
7129 }
7130
7131 sub qemu_img_convert {
7132     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7133
7134     my $storecfg = PVE::Storage::config();
7135     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7136     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7137
7138     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7139
7140     my $cachemode;
7141     my $src_path;
7142     my $src_is_iscsi = 0;
7143     my $src_format;
7144
7145     if ($src_storeid) {
7146         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7147         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7148         $src_format = qemu_img_format($src_scfg, $src_volname);
7149         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7150         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7151         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7152     } elsif (-f $src_volid) {
7153         $src_path = $src_volid;
7154         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7155             $src_format = $1;
7156         }
7157     }
7158
7159     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7160
7161     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7162     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7163     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7164     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7165
7166     my $cmd = [];
7167     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7168     push @$cmd, '-l', "snapshot.name=$snapname"
7169         if $snapname && $src_format && $src_format eq "qcow2";
7170     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7171     push @$cmd, '-T', $cachemode if defined($cachemode);
7172
7173     if ($src_is_iscsi) {
7174         push @$cmd, '--image-opts';
7175         $src_path = convert_iscsi_path($src_path);
7176     } elsif ($src_format) {
7177         push @$cmd, '-f', $src_format;
7178     }
7179
7180     if ($dst_is_iscsi) {
7181         push @$cmd, '--target-image-opts';
7182         $dst_path = convert_iscsi_path($dst_path);
7183     } else {
7184         push @$cmd, '-O', $dst_format;
7185     }
7186
7187     push @$cmd, $src_path;
7188
7189     if (!$dst_is_iscsi && $is_zero_initialized) {
7190         push @$cmd, "zeroinit:$dst_path";
7191     } else {
7192         push @$cmd, $dst_path;
7193     }
7194
7195     my $parser = sub {
7196         my $line = shift;
7197         if($line =~ m/\((\S+)\/100\%\)/){
7198             my $percent = $1;
7199             my $transferred = int($size * $percent / 100);
7200             my $total_h = render_bytes($size, 1);
7201             my $transferred_h = render_bytes($transferred, 1);
7202
7203             print "transferred $transferred_h of $total_h ($percent%)\n";
7204         }
7205
7206     };
7207
7208     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7209     my $err = $@;
7210     die "copy failed: $err" if $err;
7211 }
7212
7213 sub qemu_img_format {
7214     my ($scfg, $volname) = @_;
7215
7216     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7217         return $1;
7218     } else {
7219         return "raw";
7220     }
7221 }
7222
7223 sub qemu_drive_mirror {
7224     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7225
7226     $jobs = {} if !$jobs;
7227
7228     my $qemu_target;
7229     my $format;
7230     $jobs->{"drive-$drive"} = {};
7231
7232     if ($dst_volid =~ /^nbd:/) {
7233         $qemu_target = $dst_volid;
7234         $format = "nbd";
7235     } else {
7236         my $storecfg = PVE::Storage::config();
7237         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7238
7239         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7240
7241         $format = qemu_img_format($dst_scfg, $dst_volname);
7242
7243         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7244
7245         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7246     }
7247
7248     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7249     $opts->{format} = $format if $format;
7250
7251     if (defined($src_bitmap)) {
7252         $opts->{sync} = 'incremental';
7253         $opts->{bitmap} = $src_bitmap;
7254         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7255     }
7256
7257     if (defined($bwlimit)) {
7258         $opts->{speed} = $bwlimit * 1024;
7259         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7260     } else {
7261         print "drive mirror is starting for drive-$drive\n";
7262     }
7263
7264     # if a job already runs for this device we get an error, catch it for cleanup
7265     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7266     if (my $err = $@) {
7267         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7268         warn "$@\n" if $@;
7269         die "mirroring error: $err\n";
7270     }
7271
7272     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7273 }
7274
7275 # $completion can be either
7276 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7277 # 'cancel': wait until all jobs are ready, block-job-cancel them
7278 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7279 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7280 sub qemu_drive_mirror_monitor {
7281     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7282
7283     $completion //= 'complete';
7284     $op //= "mirror";
7285
7286     eval {
7287         my $err_complete = 0;
7288
7289         my $starttime = time ();
7290         while (1) {
7291             die "block job ('$op') timed out\n" if $err_complete > 300;
7292
7293             my $stats = mon_cmd($vmid, "query-block-jobs");
7294             my $ctime = time();
7295
7296             my $running_jobs = {};
7297             for my $stat (@$stats) {
7298                 next if $stat->{type} ne $op;
7299                 $running_jobs->{$stat->{device}} = $stat;
7300             }
7301
7302             my $readycounter = 0;
7303
7304             for my $job_id (sort keys %$jobs) {
7305                 my $job = $running_jobs->{$job_id};
7306
7307                 my $vanished = !defined($job);
7308                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7309                 if($complete || ($vanished && $completion eq 'auto')) {
7310                     print "$job_id: $op-job finished\n";
7311                     delete $jobs->{$job_id};
7312                     next;
7313                 }
7314
7315                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7316
7317                 my $busy = $job->{busy};
7318                 my $ready = $job->{ready};
7319                 if (my $total = $job->{len}) {
7320                     my $transferred = $job->{offset} || 0;
7321                     my $remaining = $total - $transferred;
7322                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7323
7324                     my $duration = $ctime - $starttime;
7325                     my $total_h = render_bytes($total, 1);
7326                     my $transferred_h = render_bytes($transferred, 1);
7327
7328                     my $status = sprintf(
7329                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7330                         render_duration($duration),
7331                     );
7332
7333                     if ($ready) {
7334                         if ($busy) {
7335                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7336                         } else {
7337                             $status .= ", ready";
7338                         }
7339                     }
7340                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7341                     $jobs->{$job_id}->{ready} = $ready;
7342                 }
7343
7344                 $readycounter++ if $job->{ready};
7345             }
7346
7347             last if scalar(keys %$jobs) == 0;
7348
7349             if ($readycounter == scalar(keys %$jobs)) {
7350                 print "all '$op' jobs are ready\n";
7351
7352                 # do the complete later (or has already been done)
7353                 last if $completion eq 'skip' || $completion eq 'auto';
7354
7355                 if ($vmiddst && $vmiddst != $vmid) {
7356                     my $agent_running = $qga && qga_check_running($vmid);
7357                     if ($agent_running) {
7358                         print "freeze filesystem\n";
7359                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7360                     } else {
7361                         print "suspend vm\n";
7362                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7363                     }
7364
7365                     # if we clone a disk for a new target vm, we don't switch the disk
7366                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7367
7368                     if ($agent_running) {
7369                         print "unfreeze filesystem\n";
7370                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7371                     } else {
7372                         print "resume vm\n";
7373                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7374                     }
7375
7376                     last;
7377                 } else {
7378
7379                     for my $job_id (sort keys %$jobs) {
7380                         # try to switch the disk if source and destination are on the same guest
7381                         print "$job_id: Completing block job_id...\n";
7382
7383                         my $op;
7384                         if ($completion eq 'complete') {
7385                             $op = 'block-job-complete';
7386                         } elsif ($completion eq 'cancel') {
7387                             $op = 'block-job-cancel';
7388                         } else {
7389                             die "invalid completion value: $completion\n";
7390                         }
7391                         eval { mon_cmd($vmid, $op, device => $job_id) };
7392                         if ($@ =~ m/cannot be completed/) {
7393                             print "$job_id: block job cannot be completed, trying again.\n";
7394                             $err_complete++;
7395                         }else {
7396                             print "$job_id: Completed successfully.\n";
7397                             $jobs->{$job_id}->{complete} = 1;
7398                         }
7399                     }
7400                 }
7401             }
7402             sleep 1;
7403         }
7404     };
7405     my $err = $@;
7406
7407     if ($err) {
7408         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7409         die "block job ($op) error: $err";
7410     }
7411 }
7412
7413 sub qemu_blockjobs_cancel {
7414     my ($vmid, $jobs) = @_;
7415
7416     foreach my $job (keys %$jobs) {
7417         print "$job: Cancelling block job\n";
7418         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7419         $jobs->{$job}->{cancel} = 1;
7420     }
7421
7422     while (1) {
7423         my $stats = mon_cmd($vmid, "query-block-jobs");
7424
7425         my $running_jobs = {};
7426         foreach my $stat (@$stats) {
7427             $running_jobs->{$stat->{device}} = $stat;
7428         }
7429
7430         foreach my $job (keys %$jobs) {
7431
7432             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7433                 print "$job: Done.\n";
7434                 delete $jobs->{$job};
7435             }
7436         }
7437
7438         last if scalar(keys %$jobs) == 0;
7439
7440         sleep 1;
7441     }
7442 }
7443
7444 sub clone_disk {
7445     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7446         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7447
7448     my $newvolid;
7449
7450     if (!$full) {
7451         print "create linked clone of drive $drivename ($drive->{file})\n";
7452         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7453         push @$newvollist, $newvolid;
7454     } else {
7455
7456         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7457         $storeid = $storage if $storage;
7458
7459         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7460
7461         print "create full clone of drive $drivename ($drive->{file})\n";
7462         my $name = undef;
7463         my $size = undef;
7464         if (drive_is_cloudinit($drive)) {
7465             $name = "vm-$newvmid-cloudinit";
7466             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7467             if ($scfg->{path}) {
7468                 $name .= ".$dst_format";
7469             }
7470             $snapname = undef;
7471             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7472         } elsif ($drivename eq 'efidisk0') {
7473             $size = get_efivars_size($conf);
7474         } elsif ($drivename eq 'tpmstate0') {
7475             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7476         } else {
7477             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7478         }
7479         $newvolid = PVE::Storage::vdisk_alloc(
7480             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7481         );
7482         push @$newvollist, $newvolid;
7483
7484         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7485
7486         if (drive_is_cloudinit($drive)) {
7487             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7488             # if this is the case, we have to complete any block-jobs still there from
7489             # previous drive-mirrors
7490             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7491                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7492             }
7493             goto no_data_clone;
7494         }
7495
7496         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7497         if (!$running || $snapname) {
7498             # TODO: handle bwlimits
7499             if ($drivename eq 'efidisk0') {
7500                 # the relevant data on the efidisk may be smaller than the source
7501                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7502                 # that is given by the OVMF_VARS.fd
7503                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7504                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7505
7506                 # better for Ceph if block size is not too small, see bug #3324
7507                 my $bs = 1024*1024;
7508
7509                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7510                     "if=$src_path", "of=$dst_path"]);
7511             } else {
7512                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7513             }
7514         } else {
7515
7516             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7517
7518             my $kvmver = get_running_qemu_version ($vmid);
7519             if (!min_version($kvmver, 2, 7)) {
7520                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7521                     if $drive->{iothread};
7522             }
7523
7524             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7525                 $completion, $qga, $bwlimit);
7526         }
7527     }
7528
7529 no_data_clone:
7530     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7531
7532     my $disk = $drive;
7533     $disk->{format} = undef;
7534     $disk->{file} = $newvolid;
7535     $disk->{size} = $size if defined($size);
7536
7537     return $disk;
7538 }
7539
7540 sub get_running_qemu_version {
7541     my ($vmid) = @_;
7542     my $res = mon_cmd($vmid, "query-version");
7543     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7544 }
7545
7546 sub qemu_use_old_bios_files {
7547     my ($machine_type) = @_;
7548
7549     return if !$machine_type;
7550
7551     my $use_old_bios_files = undef;
7552
7553     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7554         $machine_type = $1;
7555         $use_old_bios_files = 1;
7556     } else {
7557         my $version = extract_version($machine_type, kvm_user_version());
7558         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7559         # load new efi bios files on migration. So this hack is required to allow
7560         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7561         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7562         $use_old_bios_files = !min_version($version, 2, 4);
7563     }
7564
7565     return ($use_old_bios_files, $machine_type);
7566 }
7567
7568 sub get_efivars_size {
7569     my ($conf) = @_;
7570     my $arch = get_vm_arch($conf);
7571     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7572     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7573     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7574     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7575     return -s $ovmf_vars;
7576 }
7577
7578 sub update_efidisk_size {
7579     my ($conf) = @_;
7580
7581     return if !defined($conf->{efidisk0});
7582
7583     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7584     $disk->{size} = get_efivars_size($conf);
7585     $conf->{efidisk0} = print_drive($disk);
7586
7587     return;
7588 }
7589
7590 sub update_tpmstate_size {
7591     my ($conf) = @_;
7592
7593     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7594     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7595     $conf->{tpmstate0} = print_drive($disk);
7596 }
7597
7598 sub create_efidisk($$$$$$$) {
7599     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7600
7601     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7602     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7603
7604     my $vars_size_b = -s $ovmf_vars;
7605     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7606     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7607     PVE::Storage::activate_volumes($storecfg, [$volid]);
7608
7609     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7610     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7611
7612     return ($volid, $size/1024);
7613 }
7614
7615 sub vm_iothreads_list {
7616     my ($vmid) = @_;
7617
7618     my $res = mon_cmd($vmid, 'query-iothreads');
7619
7620     my $iothreads = {};
7621     foreach my $iothread (@$res) {
7622         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7623     }
7624
7625     return $iothreads;
7626 }
7627
7628 sub scsihw_infos {
7629     my ($conf, $drive) = @_;
7630
7631     my $maxdev = 0;
7632
7633     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7634         $maxdev = 7;
7635     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7636         $maxdev = 1;
7637     } else {
7638         $maxdev = 256;
7639     }
7640
7641     my $controller = int($drive->{index} / $maxdev);
7642     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7643         ? "virtioscsi"
7644         : "scsihw";
7645
7646     return ($maxdev, $controller, $controller_prefix);
7647 }
7648
7649 sub windows_version {
7650     my ($ostype) = @_;
7651
7652     return 0 if !$ostype;
7653
7654     my $winversion = 0;
7655
7656     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7657         $winversion = 5;
7658     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7659         $winversion = 6;
7660     } elsif ($ostype =~ m/^win(\d+)$/) {
7661         $winversion = $1;
7662     }
7663
7664     return $winversion;
7665 }
7666
7667 sub resolve_dst_disk_format {
7668         my ($storecfg, $storeid, $src_volname, $format) = @_;
7669         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7670
7671         if (!$format) {
7672             # if no target format is specified, use the source disk format as hint
7673             if ($src_volname) {
7674                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7675                 $format = qemu_img_format($scfg, $src_volname);
7676             } else {
7677                 return $defFormat;
7678             }
7679         }
7680
7681         # test if requested format is supported - else use default
7682         my $supported = grep { $_ eq $format } @$validFormats;
7683         $format = $defFormat if !$supported;
7684         return $format;
7685 }
7686
7687 # NOTE: if this logic changes, please update docs & possibly gui logic
7688 sub find_vmstate_storage {
7689     my ($conf, $storecfg) = @_;
7690
7691     # first, return storage from conf if set
7692     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7693
7694     my ($target, $shared, $local);
7695
7696     foreach_storage_used_by_vm($conf, sub {
7697         my ($sid) = @_;
7698         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7699         my $dst = $scfg->{shared} ? \$shared : \$local;
7700         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7701     });
7702
7703     # second, use shared storage where VM has at least one disk
7704     # third, use local storage where VM has at least one disk
7705     # fall back to local storage
7706     $target = $shared // $local // 'local';
7707
7708     return $target;
7709 }
7710
7711 sub generate_uuid {
7712     my ($uuid, $uuid_str);
7713     UUID::generate($uuid);
7714     UUID::unparse($uuid, $uuid_str);
7715     return $uuid_str;
7716 }
7717
7718 sub generate_smbios1_uuid {
7719     return "uuid=".generate_uuid();
7720 }
7721
7722 sub nbd_stop {
7723     my ($vmid) = @_;
7724
7725     mon_cmd($vmid, 'nbd-server-stop');
7726 }
7727
7728 sub create_reboot_request {
7729     my ($vmid) = @_;
7730     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7731         or die "failed to create reboot trigger file: $!\n";
7732     close($fh);
7733 }
7734
7735 sub clear_reboot_request {
7736     my ($vmid) = @_;
7737     my $path = "/run/qemu-server/$vmid.reboot";
7738     my $res = 0;
7739
7740     $res = unlink($path);
7741     die "could not remove reboot request for $vmid: $!"
7742         if !$res && $! != POSIX::ENOENT;
7743
7744     return $res;
7745 }
7746
7747 sub bootorder_from_legacy {
7748     my ($conf, $bootcfg) = @_;
7749
7750     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7751     my $bootindex_hash = {};
7752     my $i = 1;
7753     foreach my $o (split(//, $boot)) {
7754         $bootindex_hash->{$o} = $i*100;
7755         $i++;
7756     }
7757
7758     my $bootorder = {};
7759
7760     PVE::QemuConfig->foreach_volume($conf, sub {
7761         my ($ds, $drive) = @_;
7762
7763         if (drive_is_cdrom ($drive, 1)) {
7764             if ($bootindex_hash->{d}) {
7765                 $bootorder->{$ds} = $bootindex_hash->{d};
7766                 $bootindex_hash->{d} += 1;
7767             }
7768         } elsif ($bootindex_hash->{c}) {
7769             $bootorder->{$ds} = $bootindex_hash->{c}
7770                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7771             $bootindex_hash->{c} += 1;
7772         }
7773     });
7774
7775     if ($bootindex_hash->{n}) {
7776         for (my $i = 0; $i < $MAX_NETS; $i++) {
7777             my $netname = "net$i";
7778             next if !$conf->{$netname};
7779             $bootorder->{$netname} = $bootindex_hash->{n};
7780             $bootindex_hash->{n} += 1;
7781         }
7782     }
7783
7784     return $bootorder;
7785 }
7786
7787 # Generate default device list for 'boot: order=' property. Matches legacy
7788 # default boot order, but with explicit device names. This is important, since
7789 # the fallback for when neither 'order' nor the old format is specified relies
7790 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7791 sub get_default_bootdevices {
7792     my ($conf) = @_;
7793
7794     my @ret = ();
7795
7796     # harddisk
7797     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7798     push @ret, $first if $first;
7799
7800     # cdrom
7801     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7802     push @ret, $first if $first;
7803
7804     # network
7805     for (my $i = 0; $i < $MAX_NETS; $i++) {
7806         my $netname = "net$i";
7807         next if !$conf->{$netname};
7808         push @ret, $netname;
7809         last;
7810     }
7811
7812     return \@ret;
7813 }
7814
7815 sub device_bootorder {
7816     my ($conf) = @_;
7817
7818     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7819
7820     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7821
7822     my $bootorder = {};
7823     if (!defined($boot) || $boot->{legacy}) {
7824         $bootorder = bootorder_from_legacy($conf, $boot);
7825     } elsif ($boot->{order}) {
7826         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7827         for my $dev (PVE::Tools::split_list($boot->{order})) {
7828             $bootorder->{$dev} = $i++;
7829         }
7830     }
7831
7832     return $bootorder;
7833 }
7834
7835 sub register_qmeventd_handle {
7836     my ($vmid) = @_;
7837
7838     my $fh;
7839     my $peer = "/var/run/qmeventd.sock";
7840     my $count = 0;
7841
7842     for (;;) {
7843         $count++;
7844         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7845         last if $fh;
7846         if ($! != EINTR && $! != EAGAIN) {
7847             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7848         }
7849         if ($count > 4) {
7850             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7851               . "after $count retries\n";
7852         }
7853         usleep(25000);
7854     }
7855
7856     # send handshake to mark VM as backing up
7857     print $fh to_json({vzdump => {vmid => "$vmid"}});
7858
7859     # return handle to be closed later when inhibit is no longer required
7860     return $fh;
7861 }
7862
7863 # bash completion helper
7864
7865 sub complete_backup_archives {
7866     my ($cmdname, $pname, $cvalue) = @_;
7867
7868     my $cfg = PVE::Storage::config();
7869
7870     my $storeid;
7871
7872     if ($cvalue =~ m/^([^:]+):/) {
7873         $storeid = $1;
7874     }
7875
7876     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7877
7878     my $res = [];
7879     foreach my $id (keys %$data) {
7880         foreach my $item (@{$data->{$id}}) {
7881             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7882             push @$res, $item->{volid} if defined($item->{volid});
7883         }
7884     }
7885
7886     return $res;
7887 }
7888
7889 my $complete_vmid_full = sub {
7890     my ($running) = @_;
7891
7892     my $idlist = vmstatus();
7893
7894     my $res = [];
7895
7896     foreach my $id (keys %$idlist) {
7897         my $d = $idlist->{$id};
7898         if (defined($running)) {
7899             next if $d->{template};
7900             next if $running && $d->{status} ne 'running';
7901             next if !$running && $d->{status} eq 'running';
7902         }
7903         push @$res, $id;
7904
7905     }
7906     return $res;
7907 };
7908
7909 sub complete_vmid {
7910     return &$complete_vmid_full();
7911 }
7912
7913 sub complete_vmid_stopped {
7914     return &$complete_vmid_full(0);
7915 }
7916
7917 sub complete_vmid_running {
7918     return &$complete_vmid_full(1);
7919 }
7920
7921 sub complete_storage {
7922
7923     my $cfg = PVE::Storage::config();
7924     my $ids = $cfg->{ids};
7925
7926     my $res = [];
7927     foreach my $sid (keys %$ids) {
7928         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7929         next if !$ids->{$sid}->{content}->{images};
7930         push @$res, $sid;
7931     }
7932
7933     return $res;
7934 }
7935
7936 sub complete_migration_storage {
7937     my ($cmd, $param, $current_value, $all_args) = @_;
7938
7939     my $targetnode = @$all_args[1];
7940
7941     my $cfg = PVE::Storage::config();
7942     my $ids = $cfg->{ids};
7943
7944     my $res = [];
7945     foreach my $sid (keys %$ids) {
7946         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7947         next if !$ids->{$sid}->{content}->{images};
7948         push @$res, $sid;
7949     }
7950
7951     return $res;
7952 }
7953
7954 sub vm_is_paused {
7955     my ($vmid) = @_;
7956     my $qmpstatus = eval {
7957         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7958         mon_cmd($vmid, "query-status");
7959     };
7960     warn "$@\n" if $@;
7961     return $qmpstatus && $qmpstatus->{status} eq "paused";
7962 }
7963
7964 sub check_volume_storage_type {
7965     my ($storecfg, $vol) = @_;
7966
7967     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7968     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7969     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7970
7971     die "storage '$storeid' does not support content-type '$vtype'\n"
7972         if !$scfg->{content}->{$vtype};
7973
7974     return 1;
7975 }
7976
7977 1;