PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122
 123 sub map_storage {
 124     my ($map, $source) = @_;
 125
 126     return $source if !defined($map);
 127
 128     return $map->{entries}->{$source}
 129         if $map->{entries} && defined($map->{entries}->{$source});
 130
 131     return $map->{default} if $map->{default};
 132
 133     # identity (fallback)
 134     return $source;
 135 }
 136
 137 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 138     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 139     type => 'string',
 140     format => 'storage-pair-list',
 141     optional => 1,
 142 });
 143
 144 #no warnings 'redefine';
 145
 146 my $nodename_cache;
 147 sub nodename {
 148     $nodename_cache //= PVE::INotify::nodename();
 149     return $nodename_cache;
 150 }
 151
 152 my $watchdog_fmt = {
 153     model => {
 154         default_key => 1,
 155         type => 'string',
 156         enum => [qw(i6300esb ib700)],
 157         description => "Watchdog type to emulate.",
 158         default => 'i6300esb',
 159         optional => 1,
 160     },
 161     action => {
 162         type => 'string',
 163         enum => [qw(reset shutdown poweroff pause debug none)],
 164         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 165         optional => 1,
 166     },
 167 };
 168 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 169
 170 my $agent_fmt = {
 171     enabled => {
 172         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 173         type => 'boolean',
 174         default => 0,
 175         default_key => 1,
 176     },
 177     fstrim_cloned_disks => {
 178         description => "Run fstrim after moving a disk or migrating the VM.",
 179         type => 'boolean',
 180         optional => 1,
 181         default => 0
 182     },
 183     type => {
 184         description => "Select the agent type",
 185         type => 'string',
 186         default => 'virtio',
 187         optional => 1,
 188         enum => [qw(virtio isa)],
 189     },
 190 };
 191
 192 my $vga_fmt = {
 193     type => {
 194         description => "Select the VGA type.",
 195         type => 'string',
 196         default => 'std',
 197         optional => 1,
 198         default_key => 1,
 199         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 200     },
 201     memory => {
 202         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 203         type => 'integer',
 204         optional => 1,
 205         minimum => 4,
 206         maximum => 512,
 207     },
 208 };
 209
 210 my $ivshmem_fmt = {
 211     size => {
 212         type => 'integer',
 213         minimum => 1,
 214         description => "The size of the file in MB.",
 215     },
 216     name => {
 217         type => 'string',
 218         pattern => '[a-zA-Z0-9\-]+',
 219         optional => 1,
 220         format_description => 'string',
 221         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 222     },
 223 };
 224
 225 my $audio_fmt = {
 226     device => {
 227         type => 'string',
 228         enum => [qw(ich9-intel-hda intel-hda AC97)],
 229         description =>  "Configure an audio device."
 230     },
 231     driver =>  {
 232         type => 'string',
 233         enum => ['spice', 'none'],
 234         default => 'spice',
 235         optional => 1,
 236         description => "Driver backend for the audio device."
 237     },
 238 };
 239
 240 my $spice_enhancements_fmt = {
 241     foldersharing => {
 242         type => 'boolean',
 243         optional => 1,
 244         default => '0',
 245         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 246     },
 247     videostreaming =>  {
 248         type => 'string',
 249         enum => ['off', 'all', 'filter'],
 250         default => 'off',
 251         optional => 1,
 252         description => "Enable video streaming. Uses compression for detected video streams."
 253     },
 254 };
 255
 256 my $rng_fmt = {
 257     source => {
 258         type => 'string',
 259         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 260         default_key => 1,
 261         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 262             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 263             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 264             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 265             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 266             ." a hardware RNG from the host.",
 267     },
 268     max_bytes => {
 269         type => 'integer',
 270         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 271             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 272             ." `0` to disable limiting (potentially dangerous!).",
 273         optional => 1,
 274
 275         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 276         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 277         # reading from /dev/urandom
 278         default => 1024,
 279     },
 280     period => {
 281         type => 'integer',
 282         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 283             ." the guest to retrieve another 'max_bytes' of entropy.",
 284         optional => 1,
 285         default => 1000,
 286     },
 287 };
 288
 289 my $meta_info_fmt = {
 290     'ctime' => {
 291         type => 'integer',
 292         description => "The guest creation timestamp as UNIX epoch time",
 293         minimum => 0,
 294         optional => 1,
 295     },
 296     'creation-qemu' => {
 297         type => 'string',
 298         description => "The QEMU (machine) version from the time this VM was created.",
 299         pattern => '\d+(\.\d+)+',
 300         optional => 1,
 301     },
 302 };
 303
 304 my $confdesc = {
 305     onboot => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Specifies whether a VM will be started during system bootup.",
 309         default => 0,
 310     },
 311     autostart => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Automatic restart after crash (currently ignored).",
 315         default => 0,
 316     },
 317     hotplug => {
 318         optional => 1,
 319         type => 'string', format => 'pve-hotplug-features',
 320         description => "Selectively enable hotplug features. This is a comma separated list of"
 321             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 322             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 323         default => 'network,disk,usb',
 324     },
 325     reboot => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 329         default => 1,
 330     },
 331     lock => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Lock/unlock the VM.",
 335         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 336     },
 337     cpulimit => {
 338         optional => 1,
 339         type => 'number',
 340         description => "Limit of CPU usage.",
 341         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 342             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 343         minimum => 0,
 344         maximum => 128,
 345         default => 0,
 346     },
 347     cpuunits => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 351         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 352             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 353             ." weights of all the other running VMs.",
 354         minimum => 2,
 355         maximum => 262144,
 356         default => 'cgroup v1: 1024, cgroup v2: 100',
 357     },
 358     memory => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 362             ." you use the balloon device.",
 363         minimum => 16,
 364         default => 512,
 365     },
 366     balloon => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 370         minimum => 0,
 371     },
 372     shares => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 376             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 377             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 378         minimum => 0,
 379         maximum => 50000,
 380         default => 1000,
 381     },
 382     keyboard => {
 383         optional => 1,
 384         type => 'string',
 385         description => "Keyboard layout for VNC server. The default is read from the"
 386             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 387         enum => PVE::Tools::kvmkeymaplist(),
 388         default => undef,
 389     },
 390     name => {
 391         optional => 1,
 392         type => 'string', format => 'dns-name',
 393         description => "Set a name for the VM. Only used on the configuration web interface.",
 394     },
 395     scsihw => {
 396         optional => 1,
 397         type => 'string',
 398         description => "SCSI controller model",
 399         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 400         default => 'lsi',
 401     },
 402     description => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Description for the VM. Shown in the web-interface VM's summary."
 406             ." This is saved as comment inside the configuration file.",
 407         maxLength => 1024 * 8,
 408     },
 409     ostype => {
 410         optional => 1,
 411         type => 'string',
 412         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 413         description => "Specify guest operating system.",
 414         verbose_description => <<EODESC,
 415 Specify guest operating system. This is used to enable special
 416 optimization/features for specific operating systems:
 417
 418 [horizontal]
 419 other;; unspecified OS
 420 wxp;; Microsoft Windows XP
 421 w2k;; Microsoft Windows 2000
 422 w2k3;; Microsoft Windows 2003
 423 w2k8;; Microsoft Windows 2008
 424 wvista;; Microsoft Windows Vista
 425 win7;; Microsoft Windows 7
 426 win8;; Microsoft Windows 8/2012/2012r2
 427 win10;; Microsoft Windows 10/2016/2019
 428 win11;; Microsoft Windows 11/2022
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6 - 5.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string', format => 'pve-qm-boot',
 437         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 438             ." key or 'legacy=' is deprecated.",
 439     },
 440     bootdisk => {
 441         optional => 1,
 442         type => 'string', format => 'pve-qm-bootdisk',
 443         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 444         pattern => '(ide|sata|scsi|virtio)\d+',
 445     },
 446     smp => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "The number of CPUs. Please use option -sockets instead.",
 450         minimum => 1,
 451         default => 1,
 452     },
 453     sockets => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPU sockets.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     cores => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of cores per socket.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     numa => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable NUMA.",
 471         default => 0,
 472     },
 473     hugepages => {
 474         optional => 1,
 475         type => 'string',
 476         description => "Enable/disable hugepages memory.",
 477         enum => [qw(any 2 1024)],
 478     },
 479     keephugepages => {
 480         optional => 1,
 481         type => 'boolean',
 482         default => 0,
 483         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 484             ." after VM shutdown and can be used for subsequent starts.",
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 521             ." the `ostype` indicates a Microsoft Windows OS.",
 522     },
 523     freeze => {
 524         optional => 1,
 525         type => 'boolean',
 526         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 527     },
 528     vga => {
 529         optional => 1,
 530         type => 'string', format => $vga_fmt,
 531         description => "Configure the VGA hardware.",
 532         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 533             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 534             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 535             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 536             ." display server. For win* OS you can select how many independent displays you want,"
 537             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 538             ." using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 545             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 546             ." else the watchdog will reset the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are:"
 553             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 582             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 583             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 584             ." may consider disabling this to save some context switches. This is turned off by"
 585             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => 'pve-vm-cpu-conf',
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally"
 627             ." for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 635             ." for snapshots.",
 636     }),
 637     runningcpu => {
 638         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 639             ." internally for snapshots.",
 640         optional => 1,
 641         type => 'string',
 642         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 643         format_description => 'QEMU -cpu parameter'
 644     },
 645     machine => get_standard_option('pve-qemu-machine'),
 646     arch => {
 647         description => "Virtual processor architecture. Defaults to the host.",
 648         optional => 1,
 649         type => 'string',
 650         enum => [qw(x86_64 aarch64)],
 651     },
 652     smbios1 => {
 653         description => "Specify SMBIOS type 1 fields.",
 654         type => 'string', format => 'pve-qm-smbios1',
 655         maxLength => 512,
 656         optional => 1,
 657     },
 658     protection => {
 659         optional => 1,
 660         type => 'boolean',
 661         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 662             ." remove disk operations.",
 663         default => 0,
 664     },
 665     bios => {
 666         optional => 1,
 667         type => 'string',
 668         enum => [ qw(seabios ovmf) ],
 669         description => "Select BIOS implementation.",
 670         default => 'seabios',
 671     },
 672     vmgenid => {
 673         type => 'string',
 674         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 675         format_description => 'UUID',
 676         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 677             ." to disable explicitly.",
 678         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 679             ." value identifier to the guest OS. This allows to notify the guest operating system"
 680             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 681             ." execution or creation from a template). The guest operating system notices the"
 682             ." change, and is then able to react as appropriate by marking its copies of"
 683             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 684             ."Note that auto-creation only works when done through API/CLI create or update methods"
 685             .", but not when manually editing the config file.",
 686         default => "1 (autogenerated)",
 687         optional => 1,
 688     },
 689     hookscript => {
 690         type => 'string',
 691         format => 'pve-volume-id',
 692         optional => 1,
 693         description => "Script that will be executed during various steps in the vms lifetime.",
 694     },
 695     ivshmem => {
 696         type => 'string',
 697         format => $ivshmem_fmt,
 698         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 699             ." the host.",
 700         optional => 1,
 701     },
 702     audio0 => {
 703         type => 'string',
 704         format => $audio_fmt,
 705         description => "Configure a audio device, useful in combination with QXL/Spice.",
 706         optional => 1
 707     },
 708     spice_enhancements => {
 709         type => 'string',
 710         format => $spice_enhancements_fmt,
 711         description => "Configure additional enhancements for SPICE.",
 712         optional => 1
 713     },
 714     tags => {
 715         type => 'string', format => 'pve-tag-list',
 716         description => 'Tags of the VM. This is only meta information.',
 717         optional => 1,
 718     },
 719     rng0 => {
 720         type => 'string',
 721         format => $rng_fmt,
 722         description => "Configure a VirtIO-based Random Number Generator.",
 723         optional => 1,
 724     },
 725     meta => {
 726         type => 'string',
 727         format => $meta_info_fmt,
 728         description => "Some (read-only) meta-information about this guest.",
 729         optional => 1,
 730     },
 731 };
 732
 733 my $cicustom_fmt = {
 734     meta => {
 735         type => 'string',
 736         optional => 1,
 737         description => 'Specify a custom file containing all meta data passed to the VM via"
 738             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 739         format => 'pve-volume-id',
 740         format_description => 'volume',
 741     },
 742     network => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all network data passed to the VM via'
 746             .' cloud-init.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     user => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'Specify a custom file containing all user data passed to the VM via'
 754             .' cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758     vendor => {
 759     type => 'string',
 760     optional => 1,
 761     description => 'Specify a custom file containing all vendor data passed to the VM via'
 762      .' cloud-init.',
 763     format => 'pve-volume-id',
 764     format_description => 'volume',
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 768
 769 my $confdesc_cloudinit = {
 770     citype => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'Specifies the cloud-init configuration format. The default depends on the'
 774             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 775             .' and `configdrive2` for windows.',
 776         enum => ['configdrive2', 'nocloud', 'opennebula'],
 777     },
 778     ciuser => {
 779         optional => 1,
 780         type => 'string',
 781         description => "cloud-init: User name to change ssh keys and password for instead of the"
 782             ." image's configured default user.",
 783     },
 784     cipassword => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Password to assign the user. Using this is generally not'
 788             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 789             .' support hashed passwords.',
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => "cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.",
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.",
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_USB_DEVICES = 5;
 842 my $MAX_NETS = 32;
 843 my $MAX_SERIAL_PORTS = 4;
 844 my $MAX_PARALLEL_PORTS = 3;
 845 my $MAX_NUMA = 8;
 846
 847 my $numa_fmt = {
 848     cpus => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "CPUs accessing this NUMA node.",
 852         format_description => "id[-id];...",
 853     },
 854     memory => {
 855         type => "number",
 856         description => "Amount of memory this NUMA node provides.",
 857         optional => 1,
 858     },
 859     hostnodes => {
 860         type => "string",
 861         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 862         description => "Host NUMA nodes to use.",
 863         format_description => "id[-id];...",
 864         optional => 1,
 865     },
 866     policy => {
 867         type => 'string',
 868         enum => [qw(preferred bind interleave)],
 869         description => "NUMA allocation policy.",
 870         optional => 1,
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 874 my $numadesc = {
 875     optional => 1,
 876     type => 'string', format => $numa_fmt,
 877     description => "NUMA topology.",
 878 };
 879 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 880
 881 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 882     $confdesc->{"numa$i"} = $numadesc;
 883 }
 884
 885 my $nic_model_list = [
 886     'e1000',
 887     'e1000-82540em',
 888     'e1000-82544gc',
 889     'e1000-82545em',
 890     'e1000e',
 891     'i82551',
 892     'i82557b',
 893     'i82559er',
 894     'ne2k_isa',
 895     'ne2k_pci',
 896     'pcnet',
 897     'rtl8139',
 898     'virtio',
 899     'vmxnet3',
 900 ];
 901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 902
 903 my $net_fmt_bridge_descr = <<__EOD__;
 904 Bridge to attach the network device to. The Proxmox VE standard bridge
 905 is called 'vmbr0'.
 906
 907 If you do not specify a bridge, we create a kvm user (NATed) network
 908 device, which provides DHCP and DNS services. The following addresses
 909 are used:
 910
 911  10.0.2.2   Gateway
 912  10.0.2.3   DNS Server
 913  10.0.2.4   SMB Server
 914
 915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 916 __EOD__
 917
 918 my $net_fmt = {
 919     macaddr  => get_standard_option('mac-addr', {
 920         description => "MAC address. That address must be unique withing your network. This is"
 921             ." automatically generated if not specified.",
 922     }),
 923     model => {
 924         type => 'string',
 925         description => "Network Card Model. The 'virtio' model provides the best performance with"
 926             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 927             ." best to use 'e1000'.",
 928         enum => $nic_model_list,
 929         default_key => 1,
 930     },
 931     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 932     bridge => {
 933         type => 'string',
 934         description => $net_fmt_bridge_descr,
 935         format_description => 'bridge',
 936         pattern => '[-_.\w\d]+',
 937         optional => 1,
 938     },
 939     queues => {
 940         type => 'integer',
 941         minimum => 0, maximum => 16,
 942         description => 'Number of packet queues to be used on the device.',
 943         optional => 1,
 944     },
 945     rate => {
 946         type => 'number',
 947         minimum => 0,
 948         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 949         optional => 1,
 950     },
 951     tag => {
 952         type => 'integer',
 953         minimum => 1, maximum => 4094,
 954         description => 'VLAN tag to apply to packets on this interface.',
 955         optional => 1,
 956     },
 957     trunks => {
 958         type => 'string',
 959         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 960         description => 'VLAN trunks to pass through this interface.',
 961         format_description => 'vlanid[;vlanid...]',
 962         optional => 1,
 963     },
 964     firewall => {
 965         type => 'boolean',
 966         description => 'Whether this interface should be protected by the firewall.',
 967         optional => 1,
 968     },
 969     link_down => {
 970         type => 'boolean',
 971         description => 'Whether this interface should be disconnected (like pulling the plug).',
 972         optional => 1,
 973     },
 974     mtu => {
 975         type => 'integer',
 976         minimum => 1, maximum => 65520,
 977         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 978         optional => 1,
 979     },
 980 };
 981
 982 my $netdesc = {
 983     optional => 1,
 984     type => 'string', format => $net_fmt,
 985     description => "Specify network devices.",
 986 };
 987
 988 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 989
 990 my $ipconfig_fmt = {
 991     ip => {
 992         type => 'string',
 993         format => 'pve-ipv4-config',
 994         format_description => 'IPv4Format/CIDR',
 995         description => 'IPv4 address in CIDR format.',
 996         optional => 1,
 997         default => 'dhcp',
 998     },
 999     gw => {
1000         type => 'string',
1001         format => 'ipv4',
1002         format_description => 'GatewayIPv4',
1003         description => 'Default gateway for IPv4 traffic.',
1004         optional => 1,
1005         requires => 'ip',
1006     },
1007     ip6 => {
1008         type => 'string',
1009         format => 'pve-ipv6-config',
1010         format_description => 'IPv6Format/CIDR',
1011         description => 'IPv6 address in CIDR format.',
1012         optional => 1,
1013         default => 'dhcp',
1014     },
1015     gw6 => {
1016         type => 'string',
1017         format => 'ipv6',
1018         format_description => 'GatewayIPv6',
1019         description => 'Default gateway for IPv6 traffic.',
1020         optional => 1,
1021         requires => 'ip6',
1022     },
1023 };
1024 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1025 my $ipconfigdesc = {
1026     optional => 1,
1027     type => 'string', format => 'pve-qm-ipconfig',
1028     description => <<'EODESCR',
1029 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1030
1031 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1032
1033 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1034 gateway should be provided.
1035 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1036 cloud-init 19.4 or newer.
1037
1038 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1039 dhcp on IPv4.
1040 EODESCR
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1043
1044 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1045     $confdesc->{"net$i"} = $netdesc;
1046     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1047 }
1048
1049 foreach my $key (keys %$confdesc_cloudinit) {
1050     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1051 }
1052
1053 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1054 sub verify_volume_id_or_qm_path {
1055     my ($volid, $noerr) = @_;
1056
1057     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1058         return $volid;
1059     }
1060
1061     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1062     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1063     if ($@) {
1064         return if $noerr;
1065         die $@;
1066     }
1067     return $volid;
1068 }
1069
1070 my $usb_fmt = {
1071     host => {
1072         default_key => 1,
1073         type => 'string', format => 'pve-qm-usb-device',
1074         format_description => 'HOSTUSBDEVICE|spice',
1075         description => <<EODESCR,
1076 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1077
1078  'bus-port(.port)*' (decimal numbers) or
1079  'vendor_id:product_id' (hexadeciaml numbers) or
1080  'spice'
1081
1082 You can use the 'lsusb -t' command to list existing usb devices.
1083
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1086
1087 The value 'spice' can be used to add a usb redirection devices for spice.
1088 EODESCR
1089     },
1090     usb3 => {
1091         optional => 1,
1092         type => 'boolean',
1093         description => "Specifies whether if given host option is a USB3 device or port.",
1094         default => 0,
1095     },
1096 };
1097
1098 my $usbdesc = {
1099     optional => 1,
1100     type => 'string', format => $usb_fmt,
1101     description => "Configure an USB device (n is 0 to 4).",
1102 };
1103 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1104
1105 my $serialdesc = {
1106         optional => 1,
1107         type => 'string',
1108         pattern => '(/dev/.+|socket)',
1109         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1110         verbose_description =>  <<EODESCR,
1111 Create a serial device inside the VM (n is 0 to 3), and pass through a
1112 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1113 host side (use 'qm terminal' to open a terminal connection).
1114
1115 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1116 use with special care.
1117
1118 CAUTION: Experimental! User reported problems with this option.
1119 EODESCR
1120 };
1121
1122 my $paralleldesc= {
1123         optional => 1,
1124         type => 'string',
1125         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1126         description =>  "Map host parallel devices (n is 0 to 2).",
1127         verbose_description =>  <<EODESCR,
1128 Map host parallel devices (n is 0 to 2).
1129
1130 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1131 machines - use with special care.
1132
1133 CAUTION: Experimental! User reported problems with this option.
1134 EODESCR
1135 };
1136
1137 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1138     $confdesc->{"parallel$i"} = $paralleldesc;
1139 }
1140
1141 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1142     $confdesc->{"serial$i"} = $serialdesc;
1143 }
1144
1145 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1146     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1147 }
1148
1149 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1150     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1151 }
1152
1153 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1154     $confdesc->{"usb$i"} = $usbdesc;
1155 }
1156
1157 my $boot_fmt = {
1158     legacy => {
1159         optional => 1,
1160         default_key => 1,
1161         type => 'string',
1162         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1163                      . " Deprecated, use 'order=' instead.",
1164         pattern => '[acdn]{1,4}',
1165         format_description => "[acdn]{1,4}",
1166
1167         # note: this is also the fallback if boot: is not given at all
1168         default => 'cdn',
1169     },
1170     order => {
1171         optional => 1,
1172         type => 'string',
1173         format => 'pve-qm-bootdev-list',
1174         format_description => "device[;device...]",
1175         description => <<EODESC,
1176 The guest will attempt to boot from devices in the order they appear here.
1177
1178 Disks, optical drives and passed-through storage USB devices will be directly
1179 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1180 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1181
1182 Note that only devices in this list will be marked as bootable and thus loaded
1183 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1184 (e.g. software-raid), you need to specify all of them here.
1185
1186 Overrides the deprecated 'legacy=[acdn]*' value when given.
1187 EODESC
1188     },
1189 };
1190 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1191
1192 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1193 sub verify_bootdev {
1194     my ($dev, $noerr) = @_;
1195
1196     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1197     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1198
1199     my $check = sub {
1200         my ($base) = @_;
1201         return 0 if $dev !~ m/^$base\d+$/;
1202         return 0 if !$confdesc->{$dev};
1203         return 1;
1204     };
1205
1206     return $dev if $check->("net");
1207     return $dev if $check->("usb");
1208     return $dev if $check->("hostpci");
1209
1210     return if $noerr;
1211     die "invalid boot device '$dev'\n";
1212 }
1213
1214 sub print_bootorder {
1215     my ($devs) = @_;
1216     return "" if !@$devs;
1217     my $data = { order => join(';', @$devs) };
1218     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1219 }
1220
1221 my $kvm_api_version = 0;
1222
1223 sub kvm_version {
1224     return $kvm_api_version if $kvm_api_version;
1225
1226     open my $fh, '<', '/dev/kvm' or return;
1227
1228     # 0xae00 => KVM_GET_API_VERSION
1229     $kvm_api_version = ioctl($fh, 0xae00, 0);
1230     close($fh);
1231
1232     return $kvm_api_version;
1233 }
1234
1235 my $kvm_user_version = {};
1236 my $kvm_mtime = {};
1237
1238 sub kvm_user_version {
1239     my ($binary) = @_;
1240
1241     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1242     my $st = stat($binary);
1243
1244     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1245     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1246         $cachedmtime == $st->mtime;
1247
1248     $kvm_user_version->{$binary} = 'unknown';
1249     $kvm_mtime->{$binary} = $st->mtime;
1250
1251     my $code = sub {
1252         my $line = shift;
1253         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1254             $kvm_user_version->{$binary} = $2;
1255         }
1256     };
1257
1258     eval { run_command([$binary, '--version'], outfunc => $code); };
1259     warn $@ if $@;
1260
1261     return $kvm_user_version->{$binary};
1262
1263 }
1264 my sub extract_version {
1265     my ($machine_type, $version) = @_;
1266     $version = kvm_user_version() if !defined($version);
1267     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1268 }
1269
1270 sub kernel_has_vhost_net {
1271     return -c '/dev/vhost-net';
1272 }
1273
1274 sub option_exists {
1275     my $key = shift;
1276     return defined($confdesc->{$key});
1277 }
1278
1279 my $cdrom_path;
1280 sub get_cdrom_path {
1281
1282     return  $cdrom_path if $cdrom_path;
1283
1284     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1285     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1286     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1287 }
1288
1289 sub get_iso_path {
1290     my ($storecfg, $vmid, $cdrom) = @_;
1291
1292     if ($cdrom eq 'cdrom') {
1293         return get_cdrom_path();
1294     } elsif ($cdrom eq 'none') {
1295         return '';
1296     } elsif ($cdrom =~ m|^/|) {
1297         return $cdrom;
1298     } else {
1299         return PVE::Storage::path($storecfg, $cdrom);
1300     }
1301 }
1302
1303 # try to convert old style file names to volume IDs
1304 sub filename_to_volume_id {
1305     my ($vmid, $file, $media) = @_;
1306
1307      if (!($file eq 'none' || $file eq 'cdrom' ||
1308           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1309
1310         return if $file =~ m|/|;
1311
1312         if ($media && $media eq 'cdrom') {
1313             $file = "local:iso/$file";
1314         } else {
1315             $file = "local:$vmid/$file";
1316         }
1317     }
1318
1319     return $file;
1320 }
1321
1322 sub verify_media_type {
1323     my ($opt, $vtype, $media) = @_;
1324
1325     return if !$media;
1326
1327     my $etype;
1328     if ($media eq 'disk') {
1329         $etype = 'images';
1330     } elsif ($media eq 'cdrom') {
1331         $etype = 'iso';
1332     } else {
1333         die "internal error";
1334     }
1335
1336     return if ($vtype eq $etype);
1337
1338     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1339 }
1340
1341 sub cleanup_drive_path {
1342     my ($opt, $storecfg, $drive) = @_;
1343
1344     # try to convert filesystem paths to volume IDs
1345
1346     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1347         ($drive->{file} !~ m|^/dev/.+|) &&
1348         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1349         ($drive->{file} !~ m/^\d+$/)) {
1350         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1351         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1352             if !$vtype;
1353         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1354         verify_media_type($opt, $vtype, $drive->{media});
1355         $drive->{file} = $volid;
1356     }
1357
1358     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1359 }
1360
1361 sub parse_hotplug_features {
1362     my ($data) = @_;
1363
1364     my $res = {};
1365
1366     return $res if $data eq '0';
1367
1368     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1369
1370     foreach my $feature (PVE::Tools::split_list($data)) {
1371         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1372             $res->{$1} = 1;
1373         } else {
1374             die "invalid hotplug feature '$feature'\n";
1375         }
1376     }
1377     return $res;
1378 }
1379
1380 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1381 sub pve_verify_hotplug_features {
1382     my ($value, $noerr) = @_;
1383
1384     return $value if parse_hotplug_features($value);
1385
1386     return if $noerr;
1387
1388     die "unable to parse hotplug option\n";
1389 }
1390
1391 sub scsi_inquiry {
1392     my($fh, $noerr) = @_;
1393
1394     my $SG_IO = 0x2285;
1395     my $SG_GET_VERSION_NUM = 0x2282;
1396
1397     my $versionbuf = "\x00" x 8;
1398     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1399     if (!$ret) {
1400         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1401         return;
1402     }
1403     my $version = unpack("I", $versionbuf);
1404     if ($version < 30000) {
1405         die "scsi generic interface too old\n"  if !$noerr;
1406         return;
1407     }
1408
1409     my $buf = "\x00" x 36;
1410     my $sensebuf = "\x00" x 8;
1411     my $cmd = pack("C x3 C x1", 0x12, 36);
1412
1413     # see /usr/include/scsi/sg.h
1414     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1415
1416     my $packet = pack(
1417         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1418     );
1419
1420     $ret = ioctl($fh, $SG_IO, $packet);
1421     if (!$ret) {
1422         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1423         return;
1424     }
1425
1426     my @res = unpack($sg_io_hdr_t, $packet);
1427     if ($res[17] || $res[18]) {
1428         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1429         return;
1430     }
1431
1432     my $res = {};
1433     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1434
1435     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1436     $res->{type} &= 0x1F;
1437
1438     return $res;
1439 }
1440
1441 sub path_is_scsi {
1442     my ($path) = @_;
1443
1444     my $fh = IO::File->new("+<$path") || return;
1445     my $res = scsi_inquiry($fh, 1);
1446     close($fh);
1447
1448     return $res;
1449 }
1450
1451 sub print_tabletdevice_full {
1452     my ($conf, $arch) = @_;
1453
1454     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1455
1456     # we use uhci for old VMs because tablet driver was buggy in older qemu
1457     my $usbbus;
1458     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1459         $usbbus = 'ehci';
1460     } else {
1461         $usbbus = 'uhci';
1462     }
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_keyboarddevice_full {
1468     my ($conf, $arch) = @_;
1469
1470     return if $arch ne 'aarch64';
1471
1472     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1473 }
1474
1475 my sub get_drive_id {
1476     my ($drive) = @_;
1477     return "$drive->{interface}$drive->{index}";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     my $drive_id = get_drive_id($drive);
1487     if ($drive->{interface} eq 'virtio') {
1488         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1489         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1490         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1491     } elsif ($drive->{interface} eq 'scsi') {
1492
1493         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1494         my $unit = $drive->{index} % $maxdev;
1495         my $devicetype = 'hd';
1496         my $path = '';
1497         if (drive_is_cdrom($drive)) {
1498             $devicetype = 'cd';
1499         } else {
1500             if ($drive->{file} =~ m|^/|) {
1501                 $path = $drive->{file};
1502                 if (my $info = path_is_scsi($path)) {
1503                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1504                         $devicetype = 'block';
1505                     } elsif ($info->{type} == 1) { # tape
1506                         $devicetype = 'generic';
1507                     }
1508                 }
1509             } else {
1510                  $path = PVE::Storage::path($storecfg, $drive->{file});
1511             }
1512
1513             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1514             my $version = extract_version($machine_type, kvm_user_version());
1515             if ($path =~ m/^iscsi\:\/\// &&
1516                !min_version($version, 4, 1)) {
1517                 $devicetype = 'generic';
1518             }
1519         }
1520
1521         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1522             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1523         } else {
1524             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1525                 .",lun=$drive->{index}";
1526         }
1527         $device .= ",drive=drive-$drive_id,id=$drive_id";
1528
1529         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1530             $device .= ",rotation_rate=1";
1531         }
1532         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1533
1534     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1535         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype";
1541         if ($drive->{interface} eq 'ide') {
1542             $device .= ",bus=ide.$controller,unit=$unit";
1543         } else {
1544             $device .= ",bus=ahci$controller.$unit";
1545         }
1546         $device .= ",drive=drive-$drive_id,id=$drive_id";
1547
1548         if ($devicetype eq 'hd') {
1549             if (my $model = $drive->{model}) {
1550                 $model = URI::Escape::uri_unescape($model);
1551                 $device .= ",model=$model";
1552             }
1553             if ($drive->{ssd}) {
1554                 $device .= ",rotation_rate=1";
1555             }
1556         }
1557         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1558     } elsif ($drive->{interface} eq 'usb') {
1559         die "implement me";
1560         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1561     } else {
1562         die "unsupported interface type";
1563     }
1564
1565     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1566
1567     if (my $serial = $drive->{serial}) {
1568         $serial = URI::Escape::uri_unescape($serial);
1569         $device .= ",serial=$serial";
1570     }
1571
1572
1573     return $device;
1574 }
1575
1576 sub get_initiator_name {
1577     my $initiator;
1578
1579     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1580     while (defined(my $line = <$fh>)) {
1581         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1582         $initiator = $1;
1583         last;
1584     }
1585     $fh->close();
1586
1587     return $initiator;
1588 }
1589
1590 sub print_drive_commandline_full {
1591     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1592
1593     my $path;
1594     my $volid = $drive->{file};
1595     my $format = $drive->{format};
1596     my $drive_id = get_drive_id($drive);
1597
1598     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1599     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1600
1601     if (drive_is_cdrom($drive)) {
1602         $path = get_iso_path($storecfg, $vmid, $volid);
1603         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1604     } else {
1605         if ($storeid) {
1606             $path = PVE::Storage::path($storecfg, $volid);
1607             $format //= qemu_img_format($scfg, $volname);
1608         } else {
1609             $path = $volid;
1610             $format //= "raw";
1611         }
1612    }
1613
1614    my $is_rbd = $path =~ m/^rbd:/;
1615
1616     my $opts = '';
1617     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1618     foreach my $o (@qemu_drive_options) {
1619         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1620     }
1621
1622     # snapshot only accepts on|off
1623     if (defined($drive->{snapshot})) {
1624         my $v = $drive->{snapshot} ? 'on' : 'off';
1625         $opts .= ",snapshot=$v";
1626     }
1627
1628     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1629         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1630     }
1631
1632     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1633         my ($dir, $qmpname) = @$type;
1634         if (my $v = $drive->{"mbps$dir"}) {
1635             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1636         }
1637         if (my $v = $drive->{"mbps${dir}_max"}) {
1638             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1639         }
1640         if (my $v = $drive->{"bps${dir}_max_length"}) {
1641             $opts .= ",throttling.bps$qmpname-max-length=$v";
1642         }
1643         if (my $v = $drive->{"iops${dir}"}) {
1644             $opts .= ",throttling.iops$qmpname=$v";
1645         }
1646         if (my $v = $drive->{"iops${dir}_max"}) {
1647             $opts .= ",throttling.iops$qmpname-max=$v";
1648         }
1649         if (my $v = $drive->{"iops${dir}_max_length"}) {
1650             $opts .= ",throttling.iops$qmpname-max-length=$v";
1651         }
1652     }
1653
1654     if ($pbs_name) {
1655         $format = "rbd" if $is_rbd;
1656         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1657             if !$format;
1658         $opts .= ",format=alloc-track,file.driver=$format";
1659     } elsif ($format) {
1660         $opts .= ",format=$format";
1661     }
1662
1663     my $cache_direct = 0;
1664
1665     if (my $cache = $drive->{cache}) {
1666         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1667     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1668         $opts .= ",cache=none";
1669         $cache_direct = 1;
1670     }
1671
1672     # io_uring with cache mode writeback or writethrough on krbd will hang...
1673     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1674
1675     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1676     # sometimes, just plain disable...
1677     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1678
1679     if (!$drive->{aio}) {
1680         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1681             # io_uring supports all cache modes
1682             $opts .= ",aio=io_uring";
1683         } else {
1684             # aio native works only with O_DIRECT
1685             if($cache_direct) {
1686                 $opts .= ",aio=native";
1687             } else {
1688                 $opts .= ",aio=threads";
1689             }
1690         }
1691     }
1692
1693     if (!drive_is_cdrom($drive)) {
1694         my $detectzeroes;
1695         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1696             $detectzeroes = 'off';
1697         } elsif ($drive->{discard}) {
1698             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1699         } else {
1700             # This used to be our default with discard not being specified:
1701             $detectzeroes = 'on';
1702         }
1703
1704         # note: 'detect-zeroes' works per blockdev and we want it to persist
1705         # after the alloc-track is removed, so put it on 'file' directly
1706         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1707         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1708     }
1709
1710     if ($pbs_name) {
1711         $opts .= ",backing=$pbs_name";
1712         $opts .= ",auto-remove=on";
1713     }
1714
1715     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1716     my $file_param = "file";
1717     if ($pbs_name) {
1718         # non-rbd drivers require the underlying file to be a seperate block
1719         # node, so add a second .file indirection
1720         $file_param .= ".file" if !$is_rbd;
1721         $file_param .= ".filename";
1722     }
1723     my $pathinfo = $path ? "$file_param=$path," : '';
1724
1725     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1726 }
1727
1728 sub print_pbs_blockdev {
1729     my ($pbs_conf, $pbs_name) = @_;
1730     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1731     $blockdev .= ",repository=$pbs_conf->{repository}";
1732     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1733     $blockdev .= ",archive=$pbs_conf->{archive}";
1734     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1735     return $blockdev;
1736 }
1737
1738 sub print_netdevice_full {
1739     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1740
1741     my $device = $net->{model};
1742     if ($net->{model} eq 'virtio') {
1743          $device = 'virtio-net-pci';
1744      };
1745
1746     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1747     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1748     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1749         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1750         # and out of each queue plus one config interrupt and control vector queue
1751         my $vectors = $net->{queues} * 2 + 2;
1752         $tmpstr .= ",vectors=$vectors,mq=on";
1753     }
1754     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1755
1756     if (my $mtu = $net->{mtu}) {
1757         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1758             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1759             if ($mtu == 1) {
1760                  $mtu = $bridge_mtu;
1761             } elsif ($mtu < 576) {
1762                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1763             } elsif ($mtu > $bridge_mtu) {
1764                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1765             }
1766             $tmpstr .= ",host_mtu=$mtu";
1767         } else {
1768             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1769         }
1770     }
1771
1772     if ($use_old_bios_files) {
1773         my $romfile;
1774         if ($device eq 'virtio-net-pci') {
1775             $romfile = 'pxe-virtio.rom';
1776         } elsif ($device eq 'e1000') {
1777             $romfile = 'pxe-e1000.rom';
1778         } elsif ($device eq 'e1000e') {
1779             $romfile = 'pxe-e1000e.rom';
1780         } elsif ($device eq 'ne2k') {
1781             $romfile = 'pxe-ne2k_pci.rom';
1782         } elsif ($device eq 'pcnet') {
1783             $romfile = 'pxe-pcnet.rom';
1784         } elsif ($device eq 'rtl8139') {
1785             $romfile = 'pxe-rtl8139.rom';
1786         }
1787         $tmpstr .= ",romfile=$romfile" if $romfile;
1788     }
1789
1790     return $tmpstr;
1791 }
1792
1793 sub print_netdev_full {
1794     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1795
1796     my $i = '';
1797     if ($netid =~ m/^net(\d+)$/) {
1798         $i = int($1);
1799     }
1800
1801     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1802
1803     my $ifname = "tap${vmid}i$i";
1804
1805     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1806     die "interface name '$ifname' is too long (max 15 character)\n"
1807         if length($ifname) >= 16;
1808
1809     my $vhostparam = '';
1810     if (is_native($arch)) {
1811         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1812     }
1813
1814     my $vmname = $conf->{name} || "vm$vmid";
1815
1816     my $netdev = "";
1817     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1818
1819     if ($net->{bridge}) {
1820         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1821             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1822     } else {
1823         $netdev = "type=user,id=$netid,hostname=$vmname";
1824     }
1825
1826     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1827
1828     return $netdev;
1829 }
1830
1831 my $vga_map = {
1832     'cirrus' => 'cirrus-vga',
1833     'std' => 'VGA',
1834     'vmware' => 'vmware-svga',
1835     'virtio' => 'virtio-vga',
1836 };
1837
1838 sub print_vga_device {
1839     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1840
1841     my $type = $vga_map->{$vga->{type}};
1842     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1843         $type = 'virtio-gpu';
1844     }
1845     my $vgamem_mb = $vga->{memory};
1846
1847     my $max_outputs = '';
1848     if ($qxlnum) {
1849         $type = $id ? 'qxl' : 'qxl-vga';
1850
1851         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1852             # set max outputs so linux can have up to 4 qxl displays with one device
1853             if (min_version($machine_version, 4, 1)) {
1854                 $max_outputs = ",max_outputs=4";
1855             }
1856         }
1857     }
1858
1859     die "no devicetype for $vga->{type}\n" if !$type;
1860
1861     my $memory = "";
1862     if ($vgamem_mb) {
1863         if ($vga->{type} eq 'virtio') {
1864             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1865             $memory = ",max_hostmem=$bytes";
1866         } elsif ($qxlnum) {
1867             # from https://www.spice-space.org/multiple-monitors.html
1868             $memory = ",vgamem_mb=$vga->{memory}";
1869             my $ram = $vgamem_mb * 4;
1870             my $vram = $vgamem_mb * 2;
1871             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1872         } else {
1873             $memory = ",vgamem_mb=$vga->{memory}";
1874         }
1875     } elsif ($qxlnum && $id) {
1876         $memory = ",ram_size=67108864,vram_size=33554432";
1877     }
1878
1879     my $edidoff = "";
1880     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1881         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1882     }
1883
1884     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1885     my $vgaid = "vga" . ($id // '');
1886     my $pciaddr;
1887     if ($q35 && $vgaid eq 'vga') {
1888         # the first display uses pcie.0 bus on q35 machines
1889         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1890     } else {
1891         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 sub parse_number_sets {
1898     my ($set) = @_;
1899     my $res = [];
1900     foreach my $part (split(/;/, $set)) {
1901         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1902             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1903             push @$res, [ $1, $2 ];
1904         } else {
1905             die "invalid range: $part\n";
1906         }
1907     }
1908     return $res;
1909 }
1910
1911 sub parse_numa {
1912     my ($data) = @_;
1913
1914     my $res = parse_property_string($numa_fmt, $data);
1915     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1916     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1917     return $res;
1918 }
1919
1920 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1921 sub parse_net {
1922     my ($data) = @_;
1923
1924     my $res = eval { parse_property_string($net_fmt, $data) };
1925     if ($@) {
1926         warn $@;
1927         return;
1928     }
1929     if (!defined($res->{macaddr})) {
1930         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1931         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1932     }
1933     return $res;
1934 }
1935
1936 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1937 sub parse_ipconfig {
1938     my ($data) = @_;
1939
1940     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1941     if ($@) {
1942         warn $@;
1943         return;
1944     }
1945
1946     if ($res->{gw} && !$res->{ip}) {
1947         warn 'gateway specified without specifying an IP address';
1948         return;
1949     }
1950     if ($res->{gw6} && !$res->{ip6}) {
1951         warn 'IPv6 gateway specified without specifying an IPv6 address';
1952         return;
1953     }
1954     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1955         warn 'gateway specified together with DHCP';
1956         return;
1957     }
1958     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1959         # gw6 + auto/dhcp
1960         warn "IPv6 gateway specified together with $res->{ip6} address";
1961         return;
1962     }
1963
1964     if (!$res->{ip} && !$res->{ip6}) {
1965         return { ip => 'dhcp', ip6 => 'dhcp' };
1966     }
1967
1968     return $res;
1969 }
1970
1971 sub print_net {
1972     my $net = shift;
1973
1974     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1975 }
1976
1977 sub add_random_macs {
1978     my ($settings) = @_;
1979
1980     foreach my $opt (keys %$settings) {
1981         next if $opt !~ m/^net(\d+)$/;
1982         my $net = parse_net($settings->{$opt});
1983         next if !$net;
1984         $settings->{$opt} = print_net($net);
1985     }
1986 }
1987
1988 sub vm_is_volid_owner {
1989     my ($storecfg, $vmid, $volid) = @_;
1990
1991     if ($volid !~  m|^/|) {
1992         my ($path, $owner);
1993         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1994         if ($owner && ($owner == $vmid)) {
1995             return 1;
1996         }
1997     }
1998
1999     return;
2000 }
2001
2002 sub vmconfig_register_unused_drive {
2003     my ($storecfg, $vmid, $conf, $drive) = @_;
2004
2005     if (drive_is_cloudinit($drive)) {
2006         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2007         warn $@ if $@;
2008     } elsif (!drive_is_cdrom($drive)) {
2009         my $volid = $drive->{file};
2010         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2011             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2012         }
2013     }
2014 }
2015
2016 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2017 my $smbios1_fmt = {
2018     uuid => {
2019         type => 'string',
2020         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2021         format_description => 'UUID',
2022         description => "Set SMBIOS1 UUID.",
2023         optional => 1,
2024     },
2025     version => {
2026         type => 'string',
2027         pattern => '[A-Za-z0-9+\/]+={0,2}',
2028         format_description => 'Base64 encoded string',
2029         description => "Set SMBIOS1 version.",
2030         optional => 1,
2031     },
2032     serial => {
2033         type => 'string',
2034         pattern => '[A-Za-z0-9+\/]+={0,2}',
2035         format_description => 'Base64 encoded string',
2036         description => "Set SMBIOS1 serial number.",
2037         optional => 1,
2038     },
2039     manufacturer => {
2040         type => 'string',
2041         pattern => '[A-Za-z0-9+\/]+={0,2}',
2042         format_description => 'Base64 encoded string',
2043         description => "Set SMBIOS1 manufacturer.",
2044         optional => 1,
2045     },
2046     product => {
2047         type => 'string',
2048         pattern => '[A-Za-z0-9+\/]+={0,2}',
2049         format_description => 'Base64 encoded string',
2050         description => "Set SMBIOS1 product ID.",
2051         optional => 1,
2052     },
2053     sku => {
2054         type => 'string',
2055         pattern => '[A-Za-z0-9+\/]+={0,2}',
2056         format_description => 'Base64 encoded string',
2057         description => "Set SMBIOS1 SKU string.",
2058         optional => 1,
2059     },
2060     family => {
2061         type => 'string',
2062         pattern => '[A-Za-z0-9+\/]+={0,2}',
2063         format_description => 'Base64 encoded string',
2064         description => "Set SMBIOS1 family string.",
2065         optional => 1,
2066     },
2067     base64 => {
2068         type => 'boolean',
2069         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2070         optional => 1,
2071     },
2072 };
2073
2074 sub parse_smbios1 {
2075     my ($data) = @_;
2076
2077     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub print_smbios1 {
2083     my ($smbios1) = @_;
2084     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2085 }
2086
2087 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2088
2089 sub parse_watchdog {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 sub parse_guest_agent {
2100     my ($conf) = @_;
2101
2102     return {} if !defined($conf->{agent});
2103
2104     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2105     warn $@ if $@;
2106
2107     # if the agent is disabled ignore the other potentially set properties
2108     return {} if !$res->{enabled};
2109     return $res;
2110 }
2111
2112 sub get_qga_key {
2113     my ($conf, $key) = @_;
2114     return undef if !defined($conf->{agent});
2115
2116     my $agent = parse_guest_agent($conf);
2117     return $agent->{$key};
2118 }
2119
2120 sub parse_vga {
2121     my ($value) = @_;
2122
2123     return {} if !$value;
2124     my $res = eval { parse_property_string($vga_fmt, $value) };
2125     warn $@ if $@;
2126     return $res;
2127 }
2128
2129 sub parse_rng {
2130     my ($value) = @_;
2131
2132     return if !$value;
2133
2134     my $res = eval { parse_property_string($rng_fmt, $value) };
2135     warn $@ if $@;
2136     return $res;
2137 }
2138
2139 sub parse_meta_info {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub new_meta_info_string {
2150     my () = @_; # for now do not allow to override any value
2151
2152     return PVE::JSONSchema::print_property_string(
2153         {
2154             'creation-qemu' => kvm_user_version(),
2155             ctime => "". int(time()),
2156         },
2157         $meta_info_fmt
2158     );
2159 }
2160
2161 sub qemu_created_version_fixups {
2162     my ($conf, $forcemachine, $kvmver) = @_;
2163
2164     my $meta = parse_meta_info($conf->{meta}) // {};
2165     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2166
2167     # check if we need to apply some handling for VMs that always use the latest machine version but
2168     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2169     # would be required (we do not want to pin machine version for non-windows OS type)
2170     if (
2171         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2172         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2173         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2174         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2175     ) {
2176         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2177         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2178             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2179             # and thus with the predictable interface naming of systemd
2180             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2181         }
2182     }
2183     return;
2184 }
2185
2186 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2187 sub verify_usb_device {
2188     my ($value, $noerr) = @_;
2189
2190     return $value if parse_usb_device($value);
2191
2192     return if $noerr;
2193
2194     die "unable to parse usb device\n";
2195 }
2196
2197 # add JSON properties for create and set function
2198 sub json_config_properties {
2199     my $prop = shift;
2200
2201     my $skip_json_config_opts = {
2202         parent => 1,
2203         snaptime => 1,
2204         vmstate => 1,
2205         runningmachine => 1,
2206         runningcpu => 1,
2207         meta => 1,
2208     };
2209
2210     foreach my $opt (keys %$confdesc) {
2211         next if $skip_json_config_opts->{$opt};
2212         $prop->{$opt} = $confdesc->{$opt};
2213     }
2214
2215     return $prop;
2216 }
2217
2218 # return copy of $confdesc_cloudinit to generate documentation
2219 sub cloudinit_config_properties {
2220
2221     return dclone($confdesc_cloudinit);
2222 }
2223
2224 sub check_type {
2225     my ($key, $value) = @_;
2226
2227     die "unknown setting '$key'\n" if !$confdesc->{$key};
2228
2229     my $type = $confdesc->{$key}->{type};
2230
2231     if (!defined($value)) {
2232         die "got undefined value\n";
2233     }
2234
2235     if ($value =~ m/[\n\r]/) {
2236         die "property contains a line feed\n";
2237     }
2238
2239     if ($type eq 'boolean') {
2240         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2241         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2242         die "type check ('boolean') failed - got '$value'\n";
2243     } elsif ($type eq 'integer') {
2244         return int($1) if $value =~ m/^(\d+)$/;
2245         die "type check ('integer') failed - got '$value'\n";
2246     } elsif ($type eq 'number') {
2247         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2248         die "type check ('number') failed - got '$value'\n";
2249     } elsif ($type eq 'string') {
2250         if (my $fmt = $confdesc->{$key}->{format}) {
2251             PVE::JSONSchema::check_format($fmt, $value);
2252             return $value;
2253         }
2254         $value =~ s/^\"(.*)\"$/$1/;
2255         return $value;
2256     } else {
2257         die "internal error"
2258     }
2259 }
2260
2261 sub destroy_vm {
2262     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2263
2264     my $conf = PVE::QemuConfig->load_config($vmid);
2265
2266     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2267
2268     if ($conf->{template}) {
2269         # check if any base image is still used by a linked clone
2270         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2271                 my ($ds, $drive) = @_;
2272                 return if drive_is_cdrom($drive);
2273
2274                 my $volid = $drive->{file};
2275                 return if !$volid || $volid =~ m|^/|;
2276
2277                 die "base volume '$volid' is still in use by linked cloned\n"
2278                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2279
2280         });
2281     }
2282
2283     my $volids = {};
2284     my $remove_owned_drive = sub {
2285         my ($ds, $drive) = @_;
2286         return if drive_is_cdrom($drive, 1);
2287
2288         my $volid = $drive->{file};
2289         return if !$volid || $volid =~ m|^/|;
2290         return if $volids->{$volid};
2291
2292         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2293         return if !$path || !$owner || ($owner != $vmid);
2294
2295         $volids->{$volid} = 1;
2296         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2297         warn "Could not remove disk '$volid', check manually: $@" if $@;
2298     };
2299
2300     # only remove disks owned by this VM (referenced in the config)
2301     my $include_opts = {
2302         include_unused => 1,
2303         extra_keys => ['vmstate'],
2304     };
2305     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2306
2307     for my $snap (values %{$conf->{snapshots}}) {
2308         next if !defined($snap->{vmstate});
2309         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2310         next if !defined($drive);
2311         $remove_owned_drive->('vmstate', $drive);
2312     }
2313
2314     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2315
2316     if ($purge_unreferenced) { # also remove unreferenced disk
2317         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2318         PVE::Storage::foreach_volid($vmdisks, sub {
2319             my ($volid, $sid, $volname, $d) = @_;
2320             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2321             warn $@ if $@;
2322         });
2323     }
2324
2325     if (defined $replacement_conf) {
2326         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2327     } else {
2328         PVE::QemuConfig->destroy_config($vmid);
2329     }
2330 }
2331
2332 sub parse_vm_config {
2333     my ($filename, $raw) = @_;
2334
2335     return if !defined($raw);
2336
2337     my $res = {
2338         digest => Digest::SHA::sha1_hex($raw),
2339         snapshots => {},
2340         pending => {},
2341     };
2342
2343     $filename =~ m|/qemu-server/(\d+)\.conf$|
2344         || die "got strange filename '$filename'";
2345
2346     my $vmid = $1;
2347
2348     my $conf = $res;
2349     my $descr;
2350     my $section = '';
2351
2352     my @lines = split(/\n/, $raw);
2353     foreach my $line (@lines) {
2354         next if $line =~ m/^\s*$/;
2355
2356         if ($line =~ m/^\[PENDING\]\s*$/i) {
2357             $section = 'pending';
2358             if (defined($descr)) {
2359                 $descr =~ s/\s+$//;
2360                 $conf->{description} = $descr;
2361             }
2362             $descr = undef;
2363             $conf = $res->{$section} = {};
2364             next;
2365
2366         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2367             $section = $1;
2368             if (defined($descr)) {
2369                 $descr =~ s/\s+$//;
2370                 $conf->{description} = $descr;
2371             }
2372             $descr = undef;
2373             $conf = $res->{snapshots}->{$section} = {};
2374             next;
2375         }
2376
2377         if ($line =~ m/^\#(.*)\s*$/) {
2378             $descr = '' if !defined($descr);
2379             $descr .= PVE::Tools::decode_text($1) . "\n";
2380             next;
2381         }
2382
2383         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2384             $descr = '' if !defined($descr);
2385             $descr .= PVE::Tools::decode_text($2);
2386         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2387             $conf->{snapstate} = $1;
2388         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             $conf->{$key} = $value;
2392         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2393             my $value = $1;
2394             if ($section eq 'pending') {
2395                 $conf->{delete} = $value; # we parse this later
2396             } else {
2397                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2398             }
2399         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2400             my $key = $1;
2401             my $value = $2;
2402             eval { $value = check_type($key, $value); };
2403             if ($@) {
2404                 warn "vm $vmid - unable to parse value of '$key' - $@";
2405             } else {
2406                 $key = 'ide2' if $key eq 'cdrom';
2407                 my $fmt = $confdesc->{$key}->{format};
2408                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2409                     my $v = parse_drive($key, $value);
2410                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2411                         $v->{file} = $volid;
2412                         $value = print_drive($v);
2413                     } else {
2414                         warn "vm $vmid - unable to parse value of '$key'\n";
2415                         next;
2416                     }
2417                 }
2418
2419                 $conf->{$key} = $value;
2420             }
2421         } else {
2422             warn "vm $vmid - unable to parse config: $line\n";
2423         }
2424     }
2425
2426     if (defined($descr)) {
2427         $descr =~ s/\s+$//;
2428         $conf->{description} = $descr;
2429     }
2430     delete $res->{snapstate}; # just to be sure
2431
2432     return $res;
2433 }
2434
2435 sub write_vm_config {
2436     my ($filename, $conf) = @_;
2437
2438     delete $conf->{snapstate}; # just to be sure
2439
2440     if ($conf->{cdrom}) {
2441         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2442         $conf->{ide2} = $conf->{cdrom};
2443         delete $conf->{cdrom};
2444     }
2445
2446     # we do not use 'smp' any longer
2447     if ($conf->{sockets}) {
2448         delete $conf->{smp};
2449     } elsif ($conf->{smp}) {
2450         $conf->{sockets} = $conf->{smp};
2451         delete $conf->{cores};
2452         delete $conf->{smp};
2453     }
2454
2455     my $used_volids = {};
2456
2457     my $cleanup_config = sub {
2458         my ($cref, $pending, $snapname) = @_;
2459
2460         foreach my $key (keys %$cref) {
2461             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2462                 $key eq 'snapstate' || $key eq 'pending';
2463             my $value = $cref->{$key};
2464             if ($key eq 'delete') {
2465                 die "propertry 'delete' is only allowed in [PENDING]\n"
2466                     if !$pending;
2467                 # fixme: check syntax?
2468                 next;
2469             }
2470             eval { $value = check_type($key, $value); };
2471             die "unable to parse value of '$key' - $@" if $@;
2472
2473             $cref->{$key} = $value;
2474
2475             if (!$snapname && is_valid_drivename($key)) {
2476                 my $drive = parse_drive($key, $value);
2477                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2478             }
2479         }
2480     };
2481
2482     &$cleanup_config($conf);
2483
2484     &$cleanup_config($conf->{pending}, 1);
2485
2486     foreach my $snapname (keys %{$conf->{snapshots}}) {
2487         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2488         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2489     }
2490
2491     # remove 'unusedX' settings if we re-add a volume
2492     foreach my $key (keys %$conf) {
2493         my $value = $conf->{$key};
2494         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2495             delete $conf->{$key};
2496         }
2497     }
2498
2499     my $generate_raw_config = sub {
2500         my ($conf, $pending) = @_;
2501
2502         my $raw = '';
2503
2504         # add description as comment to top of file
2505         if (defined(my $descr = $conf->{description})) {
2506             if ($descr) {
2507                 foreach my $cl (split(/\n/, $descr)) {
2508                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2509                 }
2510             } else {
2511                 $raw .= "#\n" if $pending;
2512             }
2513         }
2514
2515         foreach my $key (sort keys %$conf) {
2516             next if $key =~ /^(digest|description|pending|snapshots)$/;
2517             $raw .= "$key: $conf->{$key}\n";
2518         }
2519         return $raw;
2520     };
2521
2522     my $raw = &$generate_raw_config($conf);
2523
2524     if (scalar(keys %{$conf->{pending}})){
2525         $raw .= "\n[PENDING]\n";
2526         $raw .= &$generate_raw_config($conf->{pending}, 1);
2527     }
2528
2529     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2530         $raw .= "\n[$snapname]\n";
2531         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2532     }
2533
2534     return $raw;
2535 }
2536
2537 sub load_defaults {
2538
2539     my $res = {};
2540
2541     # we use static defaults from our JSON schema configuration
2542     foreach my $key (keys %$confdesc) {
2543         if (defined(my $default = $confdesc->{$key}->{default})) {
2544             $res->{$key} = $default;
2545         }
2546     }
2547
2548     return $res;
2549 }
2550
2551 sub config_list {
2552     my $vmlist = PVE::Cluster::get_vmlist();
2553     my $res = {};
2554     return $res if !$vmlist || !$vmlist->{ids};
2555     my $ids = $vmlist->{ids};
2556     my $nodename = nodename();
2557
2558     foreach my $vmid (keys %$ids) {
2559         my $d = $ids->{$vmid};
2560         next if !$d->{node} || $d->{node} ne $nodename;
2561         next if !$d->{type} || $d->{type} ne 'qemu';
2562         $res->{$vmid}->{exists} = 1;
2563     }
2564     return $res;
2565 }
2566
2567 # test if VM uses local resources (to prevent migration)
2568 sub check_local_resources {
2569     my ($conf, $noerr) = @_;
2570
2571     my @loc_res = ();
2572
2573     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2574     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2575
2576     push @loc_res, "ivshmem" if $conf->{ivshmem};
2577
2578     foreach my $k (keys %$conf) {
2579         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2580         # sockets are safe: they will recreated be on the target side post-migrate
2581         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2582         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2583     }
2584
2585     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2586
2587     return \@loc_res;
2588 }
2589
2590 # check if used storages are available on all nodes (use by migrate)
2591 sub check_storage_availability {
2592     my ($storecfg, $conf, $node) = @_;
2593
2594     PVE::QemuConfig->foreach_volume($conf, sub {
2595         my ($ds, $drive) = @_;
2596
2597         my $volid = $drive->{file};
2598         return if !$volid;
2599
2600         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2601         return if !$sid;
2602
2603         # check if storage is available on both nodes
2604         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2605         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2606
2607         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2608
2609         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2610             if !$scfg->{content}->{$vtype};
2611    });
2612 }
2613
2614 # list nodes where all VM images are available (used by has_feature API)
2615 sub shared_nodes {
2616     my ($conf, $storecfg) = @_;
2617
2618     my $nodelist = PVE::Cluster::get_nodelist();
2619     my $nodehash = { map { $_ => 1 } @$nodelist };
2620     my $nodename = nodename();
2621
2622     PVE::QemuConfig->foreach_volume($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         if ($storeid) {
2630             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2631             if ($scfg->{disable}) {
2632                 $nodehash = {};
2633             } elsif (my $avail = $scfg->{nodes}) {
2634                 foreach my $node (keys %$nodehash) {
2635                     delete $nodehash->{$node} if !$avail->{$node};
2636                 }
2637             } elsif (!$scfg->{shared}) {
2638                 foreach my $node (keys %$nodehash) {
2639                     delete $nodehash->{$node} if $node ne $nodename
2640                 }
2641             }
2642         }
2643     });
2644
2645     return $nodehash
2646 }
2647
2648 sub check_local_storage_availability {
2649     my ($conf, $storecfg) = @_;
2650
2651     my $nodelist = PVE::Cluster::get_nodelist();
2652     my $nodehash = { map { $_ => {} } @$nodelist };
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663
2664             if ($scfg->{disable}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2667                 }
2668             } elsif (my $avail = $scfg->{nodes}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     if (!$avail->{$node}) {
2671                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2672                     }
2673                 }
2674             }
2675         }
2676     });
2677
2678     foreach my $node (values %$nodehash) {
2679         if (my $unavail = $node->{unavailable_storages}) {
2680             $node->{unavailable_storages} = [ sort keys %$unavail ];
2681         }
2682     }
2683
2684     return $nodehash
2685 }
2686
2687 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2688 sub check_running {
2689     my ($vmid, $nocheck, $node) = @_;
2690
2691     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2692     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2693 }
2694
2695 sub vzlist {
2696
2697     my $vzlist = config_list();
2698
2699     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2700
2701     while (defined(my $de = $fd->read)) {
2702         next if $de !~ m/^(\d+)\.pid$/;
2703         my $vmid = $1;
2704         next if !defined($vzlist->{$vmid});
2705         if (my $pid = check_running($vmid)) {
2706             $vzlist->{$vmid}->{pid} = $pid;
2707         }
2708     }
2709
2710     return $vzlist;
2711 }
2712
2713 our $vmstatus_return_properties = {
2714     vmid => get_standard_option('pve-vmid'),
2715     status => {
2716         description => "Qemu process status.",
2717         type => 'string',
2718         enum => ['stopped', 'running'],
2719     },
2720     maxmem => {
2721         description => "Maximum memory in bytes.",
2722         type => 'integer',
2723         optional => 1,
2724         renderer => 'bytes',
2725     },
2726     maxdisk => {
2727         description => "Root disk size in bytes.",
2728         type => 'integer',
2729         optional => 1,
2730         renderer => 'bytes',
2731     },
2732     name => {
2733         description => "VM name.",
2734         type => 'string',
2735         optional => 1,
2736     },
2737     qmpstatus => {
2738         description => "Qemu QMP agent status.",
2739         type => 'string',
2740         optional => 1,
2741     },
2742     pid => {
2743         description => "PID of running qemu process.",
2744         type => 'integer',
2745         optional => 1,
2746     },
2747     uptime => {
2748         description => "Uptime.",
2749         type => 'integer',
2750         optional => 1,
2751         renderer => 'duration',
2752     },
2753     cpus => {
2754         description => "Maximum usable CPUs.",
2755         type => 'number',
2756         optional => 1,
2757     },
2758     lock => {
2759         description => "The current config lock, if any.",
2760         type => 'string',
2761         optional => 1,
2762     },
2763     tags => {
2764         description  => "The current configured tags, if any",
2765         type => 'string',
2766         optional => 1,
2767     },
2768     'running-machine' => {
2769         description => "The currently running machine type (if running).",
2770         type => 'string',
2771         optional => 1,
2772     },
2773     'running-qemu' => {
2774         description => "The currently running QEMU version (if running).",
2775         type => 'string',
2776         optional => 1,
2777     },
2778 };
2779
2780 my $last_proc_pid_stat;
2781
2782 # get VM status information
2783 # This must be fast and should not block ($full == false)
2784 # We only query KVM using QMP if $full == true (this can be slow)
2785 sub vmstatus {
2786     my ($opt_vmid, $full) = @_;
2787
2788     my $res = {};
2789
2790     my $storecfg = PVE::Storage::config();
2791
2792     my $list = vzlist();
2793     my $defaults = load_defaults();
2794
2795     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2796
2797     my $cpucount = $cpuinfo->{cpus} || 1;
2798
2799     foreach my $vmid (keys %$list) {
2800         next if $opt_vmid && ($vmid ne $opt_vmid);
2801
2802         my $conf = PVE::QemuConfig->load_config($vmid);
2803
2804         my $d = { vmid => int($vmid) };
2805         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2806
2807         # fixme: better status?
2808         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2809
2810         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2811         if (defined($size)) {
2812             $d->{disk} = 0; # no info available
2813             $d->{maxdisk} = $size;
2814         } else {
2815             $d->{disk} = 0;
2816             $d->{maxdisk} = 0;
2817         }
2818
2819         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2820             * ($conf->{cores} || $defaults->{cores});
2821         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2822         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2823
2824         $d->{name} = $conf->{name} || "VM $vmid";
2825         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2826             : $defaults->{memory}*(1024*1024);
2827
2828         if ($conf->{balloon}) {
2829             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2830             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2831                 : $defaults->{shares};
2832         }
2833
2834         $d->{uptime} = 0;
2835         $d->{cpu} = 0;
2836         $d->{mem} = 0;
2837
2838         $d->{netout} = 0;
2839         $d->{netin} = 0;
2840
2841         $d->{diskread} = 0;
2842         $d->{diskwrite} = 0;
2843
2844         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2845
2846         $d->{serial} = 1 if conf_has_serial($conf);
2847         $d->{lock} = $conf->{lock} if $conf->{lock};
2848         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2849
2850         $res->{$vmid} = $d;
2851     }
2852
2853     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2854     foreach my $dev (keys %$netdev) {
2855         next if $dev !~ m/^tap([1-9]\d*)i/;
2856         my $vmid = $1;
2857         my $d = $res->{$vmid};
2858         next if !$d;
2859
2860         $d->{netout} += $netdev->{$dev}->{receive};
2861         $d->{netin} += $netdev->{$dev}->{transmit};
2862
2863         if ($full) {
2864             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2865             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2866         }
2867
2868     }
2869
2870     my $ctime = gettimeofday;
2871
2872     foreach my $vmid (keys %$list) {
2873
2874         my $d = $res->{$vmid};
2875         my $pid = $d->{pid};
2876         next if !$pid;
2877
2878         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2879         next if !$pstat; # not running
2880
2881         my $used = $pstat->{utime} + $pstat->{stime};
2882
2883         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2884
2885         if ($pstat->{vsize}) {
2886             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2887         }
2888
2889         my $old = $last_proc_pid_stat->{$pid};
2890         if (!$old) {
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => 0,
2895             };
2896             next;
2897         }
2898
2899         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2900
2901         if ($dtime > 1000) {
2902             my $dutime = $used -  $old->{used};
2903
2904             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2905             $last_proc_pid_stat->{$pid} = {
2906                 time => $ctime,
2907                 used => $used,
2908                 cpu => $d->{cpu},
2909             };
2910         } else {
2911             $d->{cpu} = $old->{cpu};
2912         }
2913     }
2914
2915     return $res if !$full;
2916
2917     my $qmpclient = PVE::QMPClient->new();
2918
2919     my $ballooncb = sub {
2920         my ($vmid, $resp) = @_;
2921
2922         my $info = $resp->{'return'};
2923         return if !$info->{max_mem};
2924
2925         my $d = $res->{$vmid};
2926
2927         # use memory assigned to VM
2928         $d->{maxmem} = $info->{max_mem};
2929         $d->{balloon} = $info->{actual};
2930
2931         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2932             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2933             $d->{freemem} = $info->{free_mem};
2934         }
2935
2936         $d->{ballooninfo} = $info;
2937     };
2938
2939     my $blockstatscb = sub {
2940         my ($vmid, $resp) = @_;
2941         my $data = $resp->{'return'} || [];
2942         my $totalrdbytes = 0;
2943         my $totalwrbytes = 0;
2944
2945         for my $blockstat (@$data) {
2946             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2947             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2948
2949             $blockstat->{device} =~ s/drive-//;
2950             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2951         }
2952         $res->{$vmid}->{diskread} = $totalrdbytes;
2953         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2954     };
2955
2956     my $machinecb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} || [];
2959
2960         $res->{$vmid}->{'running-machine'} =
2961             PVE::QemuServer::Machine::current_from_query_machines($data);
2962     };
2963
2964     my $versioncb = sub {
2965         my ($vmid, $resp) = @_;
2966         my $data = $resp->{'return'} // {};
2967         my $version = 'unknown';
2968
2969         if (my $v = $data->{qemu}) {
2970             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2971         }
2972
2973         $res->{$vmid}->{'running-qemu'} = $version;
2974     };
2975
2976     my $statuscb = sub {
2977         my ($vmid, $resp) = @_;
2978
2979         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2980         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2981         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2982         # this fails if ballon driver is not loaded, so this must be
2983         # the last commnand (following command are aborted if this fails).
2984         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2985
2986         my $status = 'unknown';
2987         if (!defined($status = $resp->{'return'}->{status})) {
2988             warn "unable to get VM status\n";
2989             return;
2990         }
2991
2992         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2993     };
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; # not running
2998         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2999     }
3000
3001     $qmpclient->queue_execute(undef, 2);
3002
3003     foreach my $vmid (keys %$list) {
3004         next if $opt_vmid && ($vmid ne $opt_vmid);
3005         next if !$res->{$vmid}->{pid}; #not running
3006
3007         # we can't use the $qmpclient since it might have already aborted on
3008         # 'query-balloon', but this might also fail for older versions...
3009         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3010         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3011     }
3012
3013     foreach my $vmid (keys %$list) {
3014         next if $opt_vmid && ($vmid ne $opt_vmid);
3015         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3016     }
3017
3018     return $res;
3019 }
3020
3021 sub conf_has_serial {
3022     my ($conf) = @_;
3023
3024     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3025         if ($conf->{"serial$i"}) {
3026             return 1;
3027         }
3028     }
3029
3030     return 0;
3031 }
3032
3033 sub conf_has_audio {
3034     my ($conf, $id) = @_;
3035
3036     $id //= 0;
3037     my $audio = $conf->{"audio$id"};
3038     return if !defined($audio);
3039
3040     my $audioproperties = parse_property_string($audio_fmt, $audio);
3041     my $audiodriver = $audioproperties->{driver} // 'spice';
3042
3043     return {
3044         dev => $audioproperties->{device},
3045         dev_id => "audiodev$id",
3046         backend => $audiodriver,
3047         backend_id => "$audiodriver-backend${id}",
3048     };
3049 }
3050
3051 sub audio_devs {
3052     my ($audio, $audiopciaddr, $machine_version) = @_;
3053
3054     my $devs = [];
3055
3056     my $id = $audio->{dev_id};
3057     my $audiodev = "";
3058     if (min_version($machine_version, 4, 2)) {
3059         $audiodev = ",audiodev=$audio->{backend_id}";
3060     }
3061
3062     if ($audio->{dev} eq 'AC97') {
3063         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3064     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3065         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3066         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3067         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3068     } else {
3069         die "unkown audio device '$audio->{dev}', implement me!";
3070     }
3071
3072     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3073
3074     return $devs;
3075 }
3076
3077 sub get_tpm_paths {
3078     my ($vmid) = @_;
3079     return {
3080         socket => "/var/run/qemu-server/$vmid.swtpm",
3081         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3082     };
3083 }
3084
3085 sub add_tpm_device {
3086     my ($vmid, $devices, $conf) = @_;
3087
3088     return if !$conf->{tpmstate0};
3089
3090     my $paths = get_tpm_paths($vmid);
3091
3092     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3093     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3094     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3095 }
3096
3097 sub start_swtpm {
3098     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3099
3100     return if !$tpmdrive;
3101
3102     my $state;
3103     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3104     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3105     if ($storeid) {
3106         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3107     } else {
3108         $state = $tpm->{file};
3109     }
3110
3111     my $paths = get_tpm_paths($vmid);
3112
3113     # during migration, we will get state from remote
3114     #
3115     if (!$migration) {
3116         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3117         my $setup_cmd = [
3118             "swtpm_setup",
3119             "--tpmstate",
3120             "file://$state",
3121             "--createek",
3122             "--create-ek-cert",
3123             "--create-platform-cert",
3124             "--lock-nvram",
3125             "--config",
3126             "/etc/swtpm_setup.conf", # do not use XDG configs
3127             "--runas",
3128             "0", # force creation as root, error if not possible
3129             "--not-overwrite", # ignore existing state, do not modify
3130         ];
3131
3132         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3133         # TPM 2.0 supports ECC crypto, use if possible
3134         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3135
3136         run_command($setup_cmd, outfunc => sub {
3137             print "swtpm_setup: $1\n";
3138         });
3139     }
3140
3141     my $emulator_cmd = [
3142         "swtpm",
3143         "socket",
3144         "--tpmstate",
3145         "backend-uri=file://$state,mode=0600",
3146         "--ctrl",
3147         "type=unixio,path=$paths->{socket},mode=0600",
3148         "--pid",
3149         "file=$paths->{pid}",
3150         "--terminate", # terminate on QEMU disconnect
3151         "--daemon",
3152     ];
3153     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3154     run_command($emulator_cmd, outfunc => sub { print $1; });
3155
3156     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3157     while (! -e $paths->{pid}) {
3158         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3159         usleep(50_000);
3160     }
3161
3162     # return untainted PID of swtpm daemon so it can be killed on error
3163     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3164     return $1;
3165 }
3166
3167 sub vga_conf_has_spice {
3168     my ($vga) = @_;
3169
3170     my $vgaconf = parse_vga($vga);
3171     my $vgatype = $vgaconf->{type};
3172     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3173
3174     return $1 || 1;
3175 }
3176
3177 sub is_native($) {
3178     my ($arch) = @_;
3179     return get_host_arch() eq $arch;
3180 }
3181
3182 sub get_vm_arch {
3183     my ($conf) = @_;
3184     return $conf->{arch} // get_host_arch();
3185 }
3186
3187 my $default_machines = {
3188     x86_64 => 'pc',
3189     aarch64 => 'virt',
3190 };
3191
3192 sub get_installed_machine_version {
3193     my ($kvmversion) = @_;
3194     $kvmversion = kvm_user_version() if !defined($kvmversion);
3195     $kvmversion =~ m/^(\d+\.\d+)/;
3196     return $1;
3197 }
3198
3199 sub windows_get_pinned_machine_version {
3200     my ($machine, $base_version, $kvmversion) = @_;
3201
3202     my $pin_version = $base_version;
3203     if (!defined($base_version) ||
3204         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3205     ) {
3206         $pin_version = get_installed_machine_version($kvmversion);
3207     }
3208     if (!$machine || $machine eq 'pc') {
3209         $machine = "pc-i440fx-$pin_version";
3210     } elsif ($machine eq 'q35') {
3211         $machine = "pc-q35-$pin_version";
3212     } elsif ($machine eq 'virt') {
3213         $machine = "virt-$pin_version";
3214     } else {
3215         warn "unknown machine type '$machine', not touching that!\n";
3216     }
3217
3218     return $machine;
3219 }
3220
3221 sub get_vm_machine {
3222     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3223
3224     my $machine = $forcemachine || $conf->{machine};
3225
3226     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3227         $kvmversion //= kvm_user_version();
3228         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3229         # layout which confuses windows quite a bit and may result in various regressions..
3230         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3231         if (windows_version($conf->{ostype})) {
3232             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3233         }
3234         $arch //= 'x86_64';
3235         $machine ||= $default_machines->{$arch};
3236         if ($add_pve_version) {
3237             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3238             $machine .= "+pve$pvever";
3239         }
3240     }
3241
3242     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3243         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3244         $machine = $1 if $is_pxe;
3245
3246         # for version-pinned machines that do not include a pve-version (e.g.
3247         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3248         $machine .= '+pve0';
3249
3250         $machine .= '.pxe' if $is_pxe;
3251     }
3252
3253     return $machine;
3254 }
3255
3256 sub get_ovmf_files($$$) {
3257     my ($arch, $efidisk, $smm) = @_;
3258
3259     my $types = $OVMF->{$arch}
3260         or die "no OVMF images known for architecture '$arch'\n";
3261
3262     my $type = 'default';
3263     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3264         $type = $smm ? "4m" : "4m-no-smm";
3265         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3266     }
3267
3268     return $types->{$type}->@*;
3269 }
3270
3271 my $Arch2Qemu = {
3272     aarch64 => '/usr/bin/qemu-system-aarch64',
3273     x86_64 => '/usr/bin/qemu-system-x86_64',
3274 };
3275 sub get_command_for_arch($) {
3276     my ($arch) = @_;
3277     return '/usr/bin/kvm' if is_native($arch);
3278
3279     my $cmd = $Arch2Qemu->{$arch}
3280         or die "don't know how to emulate architecture '$arch'\n";
3281     return $cmd;
3282 }
3283
3284 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3285 # to use in a QEMU command line (-cpu element), first array_intersect the result
3286 # of query_supported_ with query_understood_. This is necessary because:
3287 #
3288 # a) query_understood_ returns flags the host cannot use and
3289 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3290 #    flags, but CPU settings - with most of them being flags. Those settings
3291 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3292 #
3293 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3294 # expensive. If you need the value returned from this, you can get it much
3295 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3296 # $accel being 'kvm' or 'tcg'.
3297 #
3298 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3299 # changes, automatically populating pmxcfs.
3300 #
3301 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3302 # since kvm and tcg machines support different flags
3303 #
3304 sub query_supported_cpu_flags {
3305     my ($arch) = @_;
3306
3307     $arch //= get_host_arch();
3308     my $default_machine = $default_machines->{$arch};
3309
3310     my $flags = {};
3311
3312     # FIXME: Once this is merged, the code below should work for ARM as well:
3313     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3314     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3315         $arch eq "aarch64";
3316
3317     my $kvm_supported = defined(kvm_version());
3318     my $qemu_cmd = get_command_for_arch($arch);
3319     my $fakevmid = -1;
3320     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3321
3322     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3323     my $query_supported_run_qemu = sub {
3324         my ($kvm) = @_;
3325
3326         my $flags = {};
3327         my $cmd = [
3328             $qemu_cmd,
3329             '-machine', $default_machine,
3330             '-display', 'none',
3331             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3332             '-mon', 'chardev=qmp,mode=control',
3333             '-pidfile', $pidfile,
3334             '-S', '-daemonize'
3335         ];
3336
3337         if (!$kvm) {
3338             push @$cmd, '-accel', 'tcg';
3339         }
3340
3341         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3342         die "QEMU flag querying VM exited with code " . $rc if $rc;
3343
3344         eval {
3345             my $cmd_result = mon_cmd(
3346                 $fakevmid,
3347                 'query-cpu-model-expansion',
3348                 type => 'full',
3349                 model => { name => 'host' }
3350             );
3351
3352             my $props = $cmd_result->{model}->{props};
3353             foreach my $prop (keys %$props) {
3354                 next if $props->{$prop} ne '1';
3355                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3356                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3357                 # We only keep those with underscores, to match /proc/cpuinfo
3358                 $prop =~ s/\.|-/_/g;
3359                 $flags->{$prop} = 1;
3360             }
3361         };
3362         my $err = $@;
3363
3364         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3365         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3366
3367         die $err if $err;
3368
3369         return [ sort keys %$flags ];
3370     };
3371
3372     # We need to query QEMU twice, since KVM and TCG have different supported flags
3373     PVE::QemuConfig->lock_config($fakevmid, sub {
3374         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3375         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3376
3377         if ($kvm_supported) {
3378             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3379             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3380         }
3381     });
3382
3383     return $flags;
3384 }
3385
3386 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3387 my $understood_cpu_flag_dir = "/usr/share/kvm";
3388 sub query_understood_cpu_flags {
3389     my $arch = get_host_arch();
3390     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3391
3392     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3393         if ! -e $filepath;
3394
3395     my $raw = file_get_contents($filepath);
3396     $raw =~ s/^\s+|\s+$//g;
3397     my @flags = split(/\s+/, $raw);
3398
3399     return \@flags;
3400 }
3401
3402 my sub get_cpuunits {
3403     my ($conf) = @_;
3404     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3405 }
3406
3407 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3408 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3409 my sub should_disable_smm {
3410     my ($conf, $vga) = @_;
3411
3412     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3413         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3414 }
3415
3416 sub config_to_command {
3417     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3418         $pbs_backing) = @_;
3419
3420     my $cmd = [];
3421     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3422     my $devices = [];
3423     my $bridges = {};
3424     my $ostype = $conf->{ostype};
3425     my $winversion = windows_version($ostype);
3426     my $kvm = $conf->{kvm};
3427     my $nodename = nodename();
3428
3429     my $arch = get_vm_arch($conf);
3430     my $kvm_binary = get_command_for_arch($arch);
3431     my $kvmver = kvm_user_version($kvm_binary);
3432
3433     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3434         $kvmver //= "undefined";
3435         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3436     }
3437
3438     my $add_pve_version = min_version($kvmver, 4, 1);
3439
3440     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3441     my $machine_version = extract_version($machine_type, $kvmver);
3442     $kvm //= 1 if is_native($arch);
3443
3444     $machine_version =~ m/(\d+)\.(\d+)/;
3445     my ($machine_major, $machine_minor) = ($1, $2);
3446
3447     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3448         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3449     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3450         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3451             ." please upgrade node '$nodename'\n"
3452     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3453         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3454         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3455             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3456             ." node '$nodename'\n";
3457     }
3458
3459     # if a specific +pve version is required for a feature, use $version_guard
3460     # instead of min_version to allow machines to be run with the minimum
3461     # required version
3462     my $required_pve_version = 0;
3463     my $version_guard = sub {
3464         my ($major, $minor, $pve) = @_;
3465         return 0 if !min_version($machine_version, $major, $minor, $pve);
3466         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3467         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3468         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3469         return 1;
3470     };
3471
3472     if ($kvm && !defined kvm_version()) {
3473         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3474             ." or enable in BIOS.\n";
3475     }
3476
3477     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3478     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3479     my $use_old_bios_files = undef;
3480     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3481
3482     my $cpuunits = get_cpuunits($conf);
3483
3484     push @$cmd, $kvm_binary;
3485
3486     push @$cmd, '-id', $vmid;
3487
3488     my $vmname = $conf->{name} || "vm$vmid";
3489
3490     push @$cmd, '-name', $vmname;
3491
3492     push @$cmd, '-no-shutdown';
3493
3494     my $use_virtio = 0;
3495
3496     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3497     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3498     push @$cmd, '-mon', "chardev=qmp,mode=control";
3499
3500     if (min_version($machine_version, 2, 12)) {
3501         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3502         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3503     }
3504
3505     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3506
3507     push @$cmd, '-daemonize';
3508
3509     if ($conf->{smbios1}) {
3510         my $smbios_conf = parse_smbios1($conf->{smbios1});
3511         if ($smbios_conf->{base64}) {
3512             # Do not pass base64 flag to qemu
3513             delete $smbios_conf->{base64};
3514             my $smbios_string = "";
3515             foreach my $key (keys %$smbios_conf) {
3516                 my $value;
3517                 if ($key eq "uuid") {
3518                     $value = $smbios_conf->{uuid}
3519                 } else {
3520                     $value = decode_base64($smbios_conf->{$key});
3521                 }
3522                 # qemu accepts any binary data, only commas need escaping by double comma
3523                 $value =~ s/,/,,/g;
3524                 $smbios_string .= "," . $key . "=" . $value if $value;
3525             }
3526             push @$cmd, '-smbios', "type=1" . $smbios_string;
3527         } else {
3528             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3529         }
3530     }
3531
3532     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3533         my $d;
3534         if (my $efidisk = $conf->{efidisk0}) {
3535             $d = parse_drive('efidisk0', $efidisk);
3536         }
3537
3538         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3540
3541         my ($path, $format);
3542         my $read_only_str = '';
3543         if ($d) {
3544             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3545             $format = $d->{format};
3546             if ($storeid) {
3547                 $path = PVE::Storage::path($storecfg, $d->{file});
3548                 if (!defined($format)) {
3549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3550                     $format = qemu_img_format($scfg, $volname);
3551                 }
3552             } else {
3553                 $path = $d->{file};
3554                 die "efidisk format must be specified\n"
3555                     if !defined($format);
3556             }
3557
3558             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3559         } else {
3560             warn "no efidisk configured! Using temporary efivars disk.\n";
3561             $path = "/tmp/$vmid-ovmf.fd";
3562             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3563             $format = 'raw';
3564         }
3565
3566         my $size_str = "";
3567
3568         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3569             $size_str = ",size=" . (-s $ovmf_vars);
3570         }
3571
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         my $cache = "";
3574         if ($path =~ m/^rbd:/) {
3575                 $cache = ',cache=writeback';
3576                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3577         }
3578
3579         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3580         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3581     }
3582
3583     if ($q35) { # tell QEMU to load q35 config early
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3593         push @$cmd, $fixups->@*;
3594     }
3595
3596     if ($conf->{vmgenid}) {
3597         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3598     }
3599
3600     # add usb controllers
3601     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3602         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3603     push @$devices, @usbcontrollers if @usbcontrollers;
3604     my $vga = parse_vga($conf->{vga});
3605
3606     my $qxlnum = vga_conf_has_spice($conf->{vga});
3607     $vga->{type} = 'qxl' if $qxlnum;
3608
3609     if (!$vga->{type}) {
3610         if ($arch eq 'aarch64') {
3611             $vga->{type} = 'virtio';
3612         } elsif (min_version($machine_version, 2, 9)) {
3613             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3614         } else {
3615             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3616         }
3617     }
3618
3619     # enable absolute mouse coordinates (needed by vnc)
3620     my $tablet = $conf->{tablet};
3621     if (!defined($tablet)) {
3622         $tablet = $defaults->{tablet};
3623         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3624         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3625     }
3626
3627     if ($tablet) {
3628         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3629         my $kbd = print_keyboarddevice_full($conf, $arch);
3630         push @$devices, '-device', $kbd if defined($kbd);
3631     }
3632
3633     my $bootorder = device_bootorder($conf);
3634
3635     # host pci device passthrough
3636     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3637         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3638
3639     # usb devices
3640     my $usb_dev_features = {};
3641     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3642
3643     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3644         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3645     push @$devices, @usbdevices if @usbdevices;
3646
3647     # serial devices
3648     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3649         my $path = $conf->{"serial$i"} or next;
3650         if ($path eq 'socket') {
3651             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3652             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3653             # On aarch64, serial0 is the UART device. Qemu only allows
3654             # connecting UART devices via the '-serial' command line, as
3655             # the device has a fixed slot on the hardware...
3656             if ($arch eq 'aarch64' && $i == 0) {
3657                 push @$devices, '-serial', "chardev:serial$i";
3658             } else {
3659                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3660             }
3661         } else {
3662             die "no such serial device\n" if ! -c $path;
3663             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3664             push @$devices, '-device', "isa-serial,chardev=serial$i";
3665         }
3666     }
3667
3668     # parallel devices
3669     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3670         if (my $path = $conf->{"parallel$i"}) {
3671             die "no such parallel device\n" if ! -c $path;
3672             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3673             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3674             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3675         }
3676     }
3677
3678     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3679         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3680         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3681         push @$devices, @$audio_devs;
3682     }
3683
3684     add_tpm_device($vmid, $devices, $conf);
3685
3686     my $sockets = 1;
3687     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3688     $sockets = $conf->{sockets} if  $conf->{sockets};
3689
3690     my $cores = $conf->{cores} || 1;
3691
3692     my $maxcpus = $sockets * $cores;
3693
3694     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3695
3696     my $allowed_vcpus = $cpuinfo->{cpus};
3697
3698     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3699
3700     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3701         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702         for (my $i = 2; $i <= $vcpus; $i++)  {
3703             my $cpustr = print_cpu_device($conf,$i);
3704             push @$cmd, '-device', $cpustr;
3705         }
3706
3707     } else {
3708
3709         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3710     }
3711     push @$cmd, '-nodefaults';
3712
3713     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3714
3715     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3716
3717     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3718
3719     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3720         push @$devices, '-device', print_vga_device(
3721             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3722         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3723         push @$cmd,  '-vnc', "unix:$socket,password=on";
3724     } else {
3725         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3726         push @$cmd, '-nographic';
3727     }
3728
3729     # time drift fix
3730     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3731     my $useLocaltime = $conf->{localtime};
3732
3733     if ($winversion >= 5) { # windows
3734         $useLocaltime = 1 if !defined($conf->{localtime});
3735
3736         # use time drift fix when acpi is enabled
3737         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3738             $tdf = 1 if !defined($conf->{tdf});
3739         }
3740     }
3741
3742     if ($winversion >= 6) {
3743         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3744         push @$cmd, '-no-hpet';
3745     }
3746
3747     push @$rtcFlags, 'driftfix=slew' if $tdf;
3748
3749     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3750         push @$rtcFlags, "base=$conf->{startdate}";
3751     } elsif ($useLocaltime) {
3752         push @$rtcFlags, 'base=localtime';
3753     }
3754
3755     if ($forcecpu) {
3756         push @$cmd, '-cpu', $forcecpu;
3757     } else {
3758         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3759     }
3760
3761     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3762
3763     push @$cmd, '-S' if $conf->{freeze};
3764
3765     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3766
3767     my $guest_agent = parse_guest_agent($conf);
3768
3769     if ($guest_agent->{enabled}) {
3770         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3771         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3772
3773         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3774             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3775             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3776             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3777         } elsif ($guest_agent->{type} eq 'isa') {
3778             push @$devices, '-device', "isa-serial,chardev=qga0";
3779         }
3780     }
3781
3782     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3783     if ($rng && $version_guard->(4, 1, 2)) {
3784         check_rng_source($rng->{source});
3785
3786         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3787         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3788         my $limiter_str = "";
3789         if ($max_bytes) {
3790             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3791         }
3792
3793         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3794         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3795         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3796     }
3797
3798     my $spice_port;
3799
3800     if ($qxlnum) {
3801         if ($qxlnum > 1) {
3802             if ($winversion){
3803                 for (my $i = 1; $i < $qxlnum; $i++){
3804                     push @$devices, '-device', print_vga_device(
3805                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3806                 }
3807             } else {
3808                 # assume other OS works like Linux
3809                 my ($ram, $vram) = ("134217728", "67108864");
3810                 if ($vga->{memory}) {
3811                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3812                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3813                 }
3814                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3815                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3816             }
3817         }
3818
3819         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3820
3821         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3822         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3823         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3824
3825         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3826         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3827         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3828
3829         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3830         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3831
3832         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3833         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3834         if ($spice_enhancement->{foldersharing}) {
3835             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3836             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3837         }
3838
3839         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3840         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3841             if $spice_enhancement->{videostreaming};
3842
3843         push @$devices, '-spice', "$spice_opts";
3844     }
3845
3846     # enable balloon by default, unless explicitly disabled
3847     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3848         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3849         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3850     }
3851
3852     if ($conf->{watchdog}) {
3853         my $wdopts = parse_watchdog($conf->{watchdog});
3854         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3855         my $watchdog = $wdopts->{model} || 'i6300esb';
3856         push @$devices, '-device', "$watchdog$pciaddr";
3857         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3858     }
3859
3860     my $vollist = [];
3861     my $scsicontroller = {};
3862     my $ahcicontroller = {};
3863     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3864
3865     # Add iscsi initiator name if available
3866     if (my $initiator = get_initiator_name()) {
3867         push @$devices, '-iscsi', "initiator-name=$initiator";
3868     }
3869
3870     PVE::QemuConfig->foreach_volume($conf, sub {
3871         my ($ds, $drive) = @_;
3872
3873         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3874             check_volume_storage_type($storecfg, $drive->{file});
3875             push @$vollist, $drive->{file};
3876         }
3877
3878         # ignore efidisk here, already added in bios/fw handling code above
3879         return if $drive->{interface} eq 'efidisk';
3880         # similar for TPM
3881         return if $drive->{interface} eq 'tpmstate';
3882
3883         $use_virtio = 1 if $ds =~ m/^virtio/;
3884
3885         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3886
3887         if ($drive->{interface} eq 'virtio'){
3888            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3889         }
3890
3891         if ($drive->{interface} eq 'scsi') {
3892
3893             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3894
3895             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3896                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3897
3898             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3899             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3900
3901             my $iothread = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3903                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3904                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3905             } elsif ($drive->{iothread}) {
3906                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3907             }
3908
3909             my $queues = '';
3910             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3911                 $queues = ",num_queues=$drive->{queues}";
3912             }
3913
3914             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3915                 if !$scsicontroller->{$controller};
3916             $scsicontroller->{$controller}=1;
3917         }
3918
3919         if ($drive->{interface} eq 'sata') {
3920             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3921             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3922             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3923                 if !$ahcicontroller->{$controller};
3924             $ahcicontroller->{$controller}=1;
3925         }
3926
3927         my $pbs_conf = $pbs_backing->{$ds};
3928         my $pbs_name = undef;
3929         if ($pbs_conf) {
3930             $pbs_name = "drive-$ds-pbs";
3931             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3932         }
3933
3934         my $drive_cmd = print_drive_commandline_full(
3935             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3936
3937         # extra protection for templates, but SATA and IDE don't support it..
3938         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3939
3940         push @$devices, '-drive',$drive_cmd;
3941         push @$devices, '-device', print_drivedevice_full(
3942             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3943     });
3944
3945     for (my $i = 0; $i < $MAX_NETS; $i++) {
3946         my $netname = "net$i";
3947
3948         next if !$conf->{$netname};
3949         my $d = parse_net($conf->{$netname});
3950         next if !$d;
3951
3952         $use_virtio = 1 if $d->{model} eq 'virtio';
3953
3954         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3955
3956         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3957         push @$devices, '-netdev', $netdevfull;
3958
3959         my $netdevicefull = print_netdevice_full(
3960             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3961
3962         push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3980             .",size=$ivshmem->{size}M";
3981     }
3982
3983     # pci.4 is nested in pci.1
3984     $bridges->{1} = 1 if $bridges->{4};
3985
3986     if (!$q35) { # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992     }
3993
3994     for my $k (sort {$b cmp $a} keys %$bridges) {
3995         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3996
3997         my $k_name = $k;
3998         if ($k == 2 && $legacy_igd) {
3999             $k_name = "$k-igd";
4000         }
4001         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4002         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4003
4004         if ($q35) { # add after -readconfig pve-q35.cfg
4005             splice @$devices, 2, 0, '-device', $devstr;
4006         } else {
4007             unshift @$devices, '-device', $devstr if $k > 0;
4008         }
4009     }
4010
4011     if (!$kvm) {
4012         push @$machineFlags, 'accel=tcg';
4013     }
4014
4015     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4016
4017     my $machine_type_min = $machine_type;
4018     if ($add_pve_version) {
4019         $machine_type_min =~ s/\+pve\d+$//;
4020         $machine_type_min .= "+pve$required_pve_version";
4021     }
4022     push @$machineFlags, "type=${machine_type_min}";
4023
4024     push @$cmd, @$devices;
4025     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4027     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4028
4029     if (my $vmstate = $conf->{vmstate}) {
4030         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4031         push @$vollist, $vmstate;
4032         push @$cmd, '-loadstate', $statepath;
4033         print "activating and using '$vmstate' as vmstate\n";
4034     }
4035
4036     if (PVE::QemuConfig->is_template($conf)) {
4037         # needed to workaround base volumes being read-only
4038         push @$cmd, '-snapshot';
4039     }
4040
4041     # add custom args
4042     if ($conf->{args}) {
4043         my $aa = PVE::Tools::split_args($conf->{args});
4044         push @$cmd, @$aa;
4045     }
4046
4047     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4048 }
4049
4050 sub check_rng_source {
4051     my ($source) = @_;
4052
4053     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4054     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4055         if ! -e $source;
4056
4057     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4058     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4059         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4060         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4061         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4062             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4063             ." to the host.\n";
4064     }
4065 }
4066
4067 sub spice_port {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-spice');
4071
4072     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4073 }
4074
4075 sub vm_devices_list {
4076     my ($vmid) = @_;
4077
4078     my $res = mon_cmd($vmid, 'query-pci');
4079     my $devices_to_check = [];
4080     my $devices = {};
4081     foreach my $pcibus (@$res) {
4082         push @$devices_to_check, @{$pcibus->{devices}},
4083     }
4084
4085     while (@$devices_to_check) {
4086         my $to_check = [];
4087         for my $d (@$devices_to_check) {
4088             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4089             next if !$d->{'pci_bridge'};
4090
4091             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4092             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4093         }
4094         $devices_to_check = $to_check;
4095     }
4096
4097     my $resblock = mon_cmd($vmid, 'query-block');
4098     foreach my $block (@$resblock) {
4099         if($block->{device} =~ m/^drive-(\S+)/){
4100                 $devices->{$1} = 1;
4101         }
4102     }
4103
4104     my $resmice = mon_cmd($vmid, 'query-mice');
4105     foreach my $mice (@$resmice) {
4106         if ($mice->{name} eq 'QEMU HID Tablet') {
4107             $devices->{tablet} = 1;
4108             last;
4109         }
4110     }
4111
4112     # for usb devices there is no query-usb
4113     # but we can iterate over the entries in
4114     # qom-list path=/machine/peripheral
4115     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4116     foreach my $per (@$resperipheral) {
4117         if ($per->{name} =~ m/^usb\d+$/) {
4118             $devices->{$per->{name}} = 1;
4119         }
4120     }
4121
4122     return $devices;
4123 }
4124
4125 sub vm_deviceplug {
4126     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4127
4128     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if defined($devices_list->{$deviceid});
4132
4133     # add PCI bridge if we need it for the device
4134     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4135
4136     if ($deviceid eq 'tablet') {
4137         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4138     } elsif ($deviceid eq 'keyboard') {
4139         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4140     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4141         die "usb hotplug currently not reliable\n";
4142         # since we can't reliably hot unplug all added usb devices and usb
4143         # passthrough breaks live migration we disable usb hotplugging for now
4144         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         qemu_iothread_add($vmid, $deviceid, $device);
4147
4148         qemu_driveadd($storecfg, $vmid, $device);
4149         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         eval { qemu_deviceaddverify($vmid, $deviceid); };
4153         if (my $err = $@) {
4154             eval { qemu_drivedel($vmid, $deviceid); };
4155             warn $@ if $@;
4156             die $err;
4157         }
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4177         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4178         qemu_driveadd($storecfg, $vmid, $device);
4179
4180         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4181         eval { qemu_deviceadd($vmid, $devicefull); };
4182         if (my $err = $@) {
4183             eval { qemu_drivedel($vmid, $deviceid); };
4184             warn $@ if $@;
4185             die $err;
4186         }
4187     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4188         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4189
4190         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4191         my $use_old_bios_files = undef;
4192         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4193
4194         my $netdevicefull = print_netdevice_full(
4195             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4196         qemu_deviceadd($vmid, $netdevicefull);
4197         eval {
4198             qemu_deviceaddverify($vmid, $deviceid);
4199             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4200         };
4201         if (my $err = $@) {
4202             eval { qemu_netdevdel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4207         my $bridgeid = $2;
4208         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4209         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         qemu_deviceaddverify($vmid, $deviceid);
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4228     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4229
4230     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4231         qemu_devicedel($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^usb\d+$/) {
4233         die "usb hotplug currently not reliable\n";
4234         # when unplugging usb devices this way, there may be remaining usb
4235         # controllers/hubs so we disable it for now
4236         #qemu_devicedel($vmid, $deviceid);
4237         #qemu_devicedelverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239         my $device = parse_drive($deviceid, $conf->{$deviceid});
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($vmid, $deviceid, $device);
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4249         my $device = parse_drive($deviceid, $conf->{$deviceid});
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_drivedel($vmid, $deviceid);
4253         qemu_deletescsihw($conf, $vmid, $deviceid);
4254
4255         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4256             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4257     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4258         qemu_devicedel($vmid, $deviceid);
4259         qemu_devicedelverify($vmid, $deviceid);
4260         qemu_netdevdel($vmid, $deviceid);
4261     } else {
4262         die "can't unplug device '$deviceid'\n";
4263     }
4264
4265     return 1;
4266 }
4267
4268 sub qemu_deviceadd {
4269     my ($vmid, $devicefull) = @_;
4270
4271     $devicefull = "driver=".$devicefull;
4272     my %options =  split(/[=,]/, $devicefull);
4273
4274     mon_cmd($vmid, "device_add" , %options);
4275 }
4276
4277 sub qemu_devicedel {
4278     my ($vmid, $deviceid) = @_;
4279
4280     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4281 }
4282
4283 sub qemu_iothread_add {
4284     my ($vmid, $deviceid, $device) = @_;
4285
4286     if ($device->{iothread}) {
4287         my $iothreads = vm_iothreads_list($vmid);
4288         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4289     }
4290 }
4291
4292 sub qemu_iothread_del {
4293     my ($vmid, $deviceid, $device) = @_;
4294
4295     if ($device->{iothread}) {
4296         my $iothreads = vm_iothreads_list($vmid);
4297         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4298     }
4299 }
4300
4301 sub qemu_objectadd {
4302     my ($vmid, $objectid, $qomtype) = @_;
4303
4304     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_objectdel {
4310     my ($vmid, $objectid) = @_;
4311
4312     mon_cmd($vmid, "object-del", id => $objectid);
4313
4314     return 1;
4315 }
4316
4317 sub qemu_driveadd {
4318     my ($storecfg, $vmid, $device) = @_;
4319
4320     my $kvmver = get_running_qemu_version($vmid);
4321     my $io_uring = min_version($kvmver, 6, 0);
4322     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4323     $drive =~ s/\\/\\\\/g;
4324     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4325
4326     # If the command succeeds qemu prints: "OK"
4327     return 1 if $ret =~ m/OK/s;
4328
4329     die "adding drive failed: $ret\n";
4330 }
4331
4332 sub qemu_drivedel {
4333     my ($vmid, $deviceid) = @_;
4334
4335     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4336     $ret =~ s/^\s+//;
4337
4338     return 1 if $ret eq "";
4339
4340     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4341     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4342
4343     die "deleting drive $deviceid failed : $ret\n";
4344 }
4345
4346 sub qemu_deviceaddverify {
4347     my ($vmid, $deviceid) = @_;
4348
4349     for (my $i = 0; $i <= 5; $i++) {
4350          my $devices_list = vm_devices_list($vmid);
4351          return 1 if defined($devices_list->{$deviceid});
4352          sleep 1;
4353     }
4354
4355     die "error on hotplug device '$deviceid'\n";
4356 }
4357
4358
4359 sub qemu_devicedelverify {
4360     my ($vmid, $deviceid) = @_;
4361
4362     # need to verify that the device is correctly removed as device_del
4363     # is async and empty return is not reliable
4364
4365     for (my $i = 0; $i <= 5; $i++) {
4366          my $devices_list = vm_devices_list($vmid);
4367          return 1 if !defined($devices_list->{$deviceid});
4368          sleep 1;
4369     }
4370
4371     die "error on hot-unplugging device '$deviceid'\n";
4372 }
4373
4374 sub qemu_findorcreatescsihw {
4375     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4376
4377     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4378
4379     my $scsihwid="$controller_prefix$controller";
4380     my $devices_list = vm_devices_list($vmid);
4381
4382     if (!defined($devices_list->{$scsihwid})) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4384     }
4385
4386     return 1;
4387 }
4388
4389 sub qemu_deletescsihw {
4390     my ($conf, $vmid, $opt) = @_;
4391
4392     my $device = parse_drive($opt, $conf->{$opt});
4393
4394     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4395         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4396         return 1;
4397     }
4398
4399     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4400
4401     my $devices_list = vm_devices_list($vmid);
4402     foreach my $opt (keys %{$devices_list}) {
4403         if (is_valid_drivename($opt)) {
4404             my $drive = parse_drive($opt, $conf->{$opt});
4405             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4406                 return 1;
4407             }
4408         }
4409     }
4410
4411     my $scsihwid="scsihw$controller";
4412
4413     vm_deviceunplug($vmid, $conf, $scsihwid);
4414
4415     return 1;
4416 }
4417
4418 sub qemu_add_pci_bridge {
4419     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4420
4421     my $bridges = {};
4422
4423     my $bridgeid;
4424
4425     print_pci_addr($device, $bridges, $arch, $machine_type);
4426
4427     while (my ($k, $v) = each %$bridges) {
4428         $bridgeid = $k;
4429     }
4430     return 1 if !defined($bridgeid) || $bridgeid < 1;
4431
4432     my $bridge = "pci.$bridgeid";
4433     my $devices_list = vm_devices_list($vmid);
4434
4435     if (!defined($devices_list->{$bridge})) {
4436         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4437     }
4438
4439     return 1;
4440 }
4441
4442 sub qemu_set_link_status {
4443     my ($vmid, $device, $up) = @_;
4444
4445     mon_cmd($vmid, "set_link", name => $device,
4446                up => $up ? JSON::true : JSON::false);
4447 }
4448
4449 sub qemu_netdevadd {
4450     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4451
4452     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4453     my %options =  split(/[=,]/, $netdev);
4454
4455     if (defined(my $vhost = $options{vhost})) {
4456         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4457     }
4458
4459     if (defined(my $queues = $options{queues})) {
4460         $options{queues} = $queues + 0;
4461     }
4462
4463     mon_cmd($vmid, "netdev_add",  %options);
4464     return 1;
4465 }
4466
4467 sub qemu_netdevdel {
4468     my ($vmid, $deviceid) = @_;
4469
4470     mon_cmd($vmid, "netdev_del", id => $deviceid);
4471 }
4472
4473 sub qemu_usb_hotplug {
4474     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4475
4476     return if !$device;
4477
4478     # remove the old one first
4479     vm_deviceunplug($vmid, $conf, $deviceid);
4480
4481     # check if xhci controller is necessary and available
4482     if ($device->{usb3}) {
4483
4484         my $devicelist = vm_devices_list($vmid);
4485
4486         if (!$devicelist->{xhci}) {
4487             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4488             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4489         }
4490     }
4491     my $d = parse_usb_device($device->{host});
4492     $d->{usb3} = $device->{usb3};
4493
4494     # add the new one
4495     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4496 }
4497
4498 sub qemu_cpu_hotplug {
4499     my ($vmid, $conf, $vcpus) = @_;
4500
4501     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4502
4503     my $sockets = 1;
4504     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4505     $sockets = $conf->{sockets} if  $conf->{sockets};
4506     my $cores = $conf->{cores} || 1;
4507     my $maxcpus = $sockets * $cores;
4508
4509     $vcpus = $maxcpus if !$vcpus;
4510
4511     die "you can't add more vcpus than maxcpus\n"
4512         if $vcpus > $maxcpus;
4513
4514     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4515
4516     if ($vcpus < $currentvcpus) {
4517
4518         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4519
4520             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4521                 qemu_devicedel($vmid, "cpu$i");
4522                 my $retry = 0;
4523                 my $currentrunningvcpus = undef;
4524                 while (1) {
4525                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4526                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4527                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4528                     $retry++;
4529                     sleep 1;
4530                 }
4531                 #update conf after each succesfull cpu unplug
4532                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4533                 PVE::QemuConfig->write_config($vmid, $conf);
4534             }
4535         } else {
4536             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4537         }
4538
4539         return;
4540     }
4541
4542     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4543     die "vcpus in running vm does not match its configuration\n"
4544         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4545
4546     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4549             my $cpustr = print_cpu_device($conf, $i);
4550             qemu_deviceadd($vmid, $cpustr);
4551
4552             my $retry = 0;
4553             my $currentrunningvcpus = undef;
4554             while (1) {
4555                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4556                 last if scalar(@{$currentrunningvcpus}) == $i;
4557                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4558                 sleep 1;
4559                 $retry++;
4560             }
4561             #update conf after each succesfull cpu hotplug
4562             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         }
4565     } else {
4566
4567         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4568             mon_cmd($vmid, "cpu-add", id => int($i));
4569         }
4570     }
4571 }
4572
4573 sub qemu_block_set_io_throttle {
4574     my ($vmid, $deviceid,
4575         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4576         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4577         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4578         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4579
4580     return if !check_running($vmid) ;
4581
4582     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4583         bps => int($bps),
4584         bps_rd => int($bps_rd),
4585         bps_wr => int($bps_wr),
4586         iops => int($iops),
4587         iops_rd => int($iops_rd),
4588         iops_wr => int($iops_wr),
4589         bps_max => int($bps_max),
4590         bps_rd_max => int($bps_rd_max),
4591         bps_wr_max => int($bps_wr_max),
4592         iops_max => int($iops_max),
4593         iops_rd_max => int($iops_rd_max),
4594         iops_wr_max => int($iops_wr_max),
4595         bps_max_length => int($bps_max_length),
4596         bps_rd_max_length => int($bps_rd_max_length),
4597         bps_wr_max_length => int($bps_wr_max_length),
4598         iops_max_length => int($iops_max_length),
4599         iops_rd_max_length => int($iops_rd_max_length),
4600         iops_wr_max_length => int($iops_wr_max_length),
4601     );
4602
4603 }
4604
4605 sub qemu_block_resize {
4606     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4607
4608     my $running = check_running($vmid);
4609
4610     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4611
4612     return if !$running;
4613
4614     my $padding = (1024 - $size % 1024) % 1024;
4615     $size = $size + $padding;
4616
4617     mon_cmd(
4618         $vmid,
4619         "block_resize",
4620         device => $deviceid,
4621         size => int($size),
4622         timeout => 60,
4623     );
4624 }
4625
4626 sub qemu_volume_snapshot {
4627     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4628
4629     my $running = check_running($vmid);
4630
4631     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4632         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4633     } else {
4634         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4635     }
4636 }
4637
4638 sub qemu_volume_snapshot_delete {
4639     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4640
4641     my $running = check_running($vmid);
4642
4643     if($running) {
4644
4645         $running = undef;
4646         my $conf = PVE::QemuConfig->load_config($vmid);
4647         PVE::QemuConfig->foreach_volume($conf, sub {
4648             my ($ds, $drive) = @_;
4649             $running = 1 if $drive->{file} eq $volid;
4650         });
4651     }
4652
4653     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4654         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4655     } else {
4656         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4657     }
4658 }
4659
4660 sub set_migration_caps {
4661     my ($vmid, $savevm) = @_;
4662
4663     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4664
4665     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4666     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4667
4668     my $cap_ref = [];
4669
4670     my $enabled_cap = {
4671         "auto-converge" => 1,
4672         "xbzrle" => 1,
4673         "x-rdma-pin-all" => 0,
4674         "zero-blocks" => 0,
4675         "compress" => 0,
4676         "dirty-bitmaps" => $dirty_bitmaps,
4677     };
4678
4679     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4680
4681     for my $supported_capability (@$supported_capabilities) {
4682         push @$cap_ref, {
4683             capability => $supported_capability->{capability},
4684             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4685         };
4686     }
4687
4688     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4689 }
4690
4691 sub foreach_volid {
4692     my ($conf, $func, @param) = @_;
4693
4694     my $volhash = {};
4695
4696     my $test_volid = sub {
4697         my ($key, $drive, $snapname) = @_;
4698
4699         my $volid = $drive->{file};
4700         return if !$volid;
4701
4702         $volhash->{$volid}->{cdrom} //= 1;
4703         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4704
4705         my $replicate = $drive->{replicate} // 1;
4706         $volhash->{$volid}->{replicate} //= 0;
4707         $volhash->{$volid}->{replicate} = 1 if $replicate;
4708
4709         $volhash->{$volid}->{shared} //= 0;
4710         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4711
4712         $volhash->{$volid}->{referenced_in_config} //= 0;
4713         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4714
4715         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4716             if defined($snapname);
4717
4718         my $size = $drive->{size};
4719         $volhash->{$volid}->{size} //= $size if $size;
4720
4721         $volhash->{$volid}->{is_vmstate} //= 0;
4722         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4723
4724         $volhash->{$volid}->{is_tpmstate} //= 0;
4725         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4726
4727         $volhash->{$volid}->{is_unused} //= 0;
4728         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4729
4730         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4731     };
4732
4733     my $include_opts = {
4734         extra_keys => ['vmstate'],
4735         include_unused => 1,
4736     };
4737
4738     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4739     foreach my $snapname (keys %{$conf->{snapshots}}) {
4740         my $snap = $conf->{snapshots}->{$snapname};
4741         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4742     }
4743
4744     foreach my $volid (keys %$volhash) {
4745         &$func($volid, $volhash->{$volid}, @param);
4746     }
4747 }
4748
4749 my $fast_plug_option = {
4750     'lock' => 1,
4751     'name' => 1,
4752     'onboot' => 1,
4753     'shares' => 1,
4754     'startup' => 1,
4755     'description' => 1,
4756     'protection' => 1,
4757     'vmstatestorage' => 1,
4758     'hookscript' => 1,
4759     'tags' => 1,
4760 };
4761
4762 # hotplug changes in [PENDING]
4763 # $selection hash can be used to only apply specified options, for
4764 # example: { cores => 1 } (only apply changed 'cores')
4765 # $errors ref is used to return error messages
4766 sub vmconfig_hotplug_pending {
4767     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4768
4769     my $defaults = load_defaults();
4770     my $arch = get_vm_arch($conf);
4771     my $machine_type = get_vm_machine($conf, undef, $arch);
4772
4773     # commit values which do not have any impact on running VM first
4774     # Note: those option cannot raise errors, we we do not care about
4775     # $selection and always apply them.
4776
4777     my $add_error = sub {
4778         my ($opt, $msg) = @_;
4779         $errors->{$opt} = "hotplug problem - $msg";
4780     };
4781
4782     my $changes = 0;
4783     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4784         if ($fast_plug_option->{$opt}) {
4785             $conf->{$opt} = $conf->{pending}->{$opt};
4786             delete $conf->{pending}->{$opt};
4787             $changes = 1;
4788         }
4789     }
4790
4791     if ($changes) {
4792         PVE::QemuConfig->write_config($vmid, $conf);
4793     }
4794
4795     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4796
4797     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4798     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4799     foreach my $opt (sort keys %$pending_delete_hash) {
4800         next if $selection && !$selection->{$opt};
4801         my $force = $pending_delete_hash->{$opt}->{force};
4802         eval {
4803             if ($opt eq 'hotplug') {
4804                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4805             } elsif ($opt eq 'tablet') {
4806                 die "skip\n" if !$hotplug_features->{usb};
4807                 if ($defaults->{tablet}) {
4808                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4810                         if $arch eq 'aarch64';
4811                 } else {
4812                     vm_deviceunplug($vmid, $conf, 'tablet');
4813                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4814                 }
4815             } elsif ($opt =~ m/^usb\d+/) {
4816                 die "skip\n";
4817                 # since we cannot reliably hot unplug usb devices we are disabling it
4818                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4819                 #vm_deviceunplug($vmid, $conf, $opt);
4820             } elsif ($opt eq 'vcpus') {
4821                 die "skip\n" if !$hotplug_features->{cpu};
4822                 qemu_cpu_hotplug($vmid, $conf, undef);
4823             } elsif ($opt eq 'balloon') {
4824                 # enable balloon device is not hotpluggable
4825                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4826                 # here we reset the ballooning value to memory
4827                 my $balloon = $conf->{memory} || $defaults->{memory};
4828                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4829             } elsif ($fast_plug_option->{$opt}) {
4830                 # do nothing
4831             } elsif ($opt =~ m/^net(\d+)$/) {
4832                 die "skip\n" if !$hotplug_features->{network};
4833                 vm_deviceunplug($vmid, $conf, $opt);
4834             } elsif (is_valid_drivename($opt)) {
4835                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4836                 vm_deviceunplug($vmid, $conf, $opt);
4837                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4838             } elsif ($opt =~ m/^memory$/) {
4839                 die "skip\n" if !$hotplug_features->{memory};
4840                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4841             } elsif ($opt eq 'cpuunits') {
4842                 $cgroup->change_cpu_shares(undef, 1024);
4843             } elsif ($opt eq 'cpulimit') {
4844                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4845             } else {
4846                 die "skip\n";
4847             }
4848         };
4849         if (my $err = $@) {
4850             &$add_error($opt, $err) if $err ne "skip\n";
4851         } else {
4852             delete $conf->{$opt};
4853             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4854         }
4855     }
4856
4857     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4858     $apply_pending_cloudinit = sub {
4859         return if $apply_pending_cloudinit_done; # once is enough
4860         $apply_pending_cloudinit_done = 1; # once is enough
4861
4862         my ($key, $value) = @_;
4863
4864         my @cloudinit_opts = keys %$confdesc_cloudinit;
4865         foreach my $opt (keys %{$conf->{pending}}) {
4866             next if !grep { $_ eq $opt } @cloudinit_opts;
4867             $conf->{$opt} = delete $conf->{pending}->{$opt};
4868         }
4869
4870         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4871         foreach my $opt (sort keys %$pending_delete_hash) {
4872             next if !grep { $_ eq $opt } @cloudinit_opts;
4873             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4874             delete $conf->{$opt};
4875         }
4876
4877         my $new_conf = { %$conf };
4878         $new_conf->{$key} = $value;
4879         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4880     };
4881
4882     foreach my $opt (keys %{$conf->{pending}}) {
4883         next if $selection && !$selection->{$opt};
4884         my $value = $conf->{pending}->{$opt};
4885         eval {
4886             if ($opt eq 'hotplug') {
4887                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4888             } elsif ($opt eq 'tablet') {
4889                 die "skip\n" if !$hotplug_features->{usb};
4890                 if ($value == 1) {
4891                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4892                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4893                         if $arch eq 'aarch64';
4894                 } elsif ($value == 0) {
4895                     vm_deviceunplug($vmid, $conf, 'tablet');
4896                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4897                 }
4898             } elsif ($opt =~ m/^usb\d+$/) {
4899                 die "skip\n";
4900                 # since we cannot reliably hot unplug usb devices we disable it for now
4901                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4902                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4903                 #die "skip\n" if !$d;
4904                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4905             } elsif ($opt eq 'vcpus') {
4906                 die "skip\n" if !$hotplug_features->{cpu};
4907                 qemu_cpu_hotplug($vmid, $conf, $value);
4908             } elsif ($opt eq 'balloon') {
4909                 # enable/disable balloning device is not hotpluggable
4910                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4911                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4912                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4913
4914                 # allow manual ballooning if shares is set to zero
4915                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4916                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4917                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4918                 }
4919             } elsif ($opt =~ m/^net(\d+)$/) {
4920                 # some changes can be done without hotplug
4921                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4922                                     $vmid, $opt, $value, $arch, $machine_type);
4923             } elsif (is_valid_drivename($opt)) {
4924                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4925                 # some changes can be done without hotplug
4926                 my $drive = parse_drive($opt, $value);
4927                 if (drive_is_cloudinit($drive)) {
4928                     &$apply_pending_cloudinit($opt, $value);
4929                 }
4930                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4931                                      $vmid, $opt, $value, $arch, $machine_type);
4932             } elsif ($opt =~ m/^memory$/) { #dimms
4933                 die "skip\n" if !$hotplug_features->{memory};
4934                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4935             } elsif ($opt eq 'cpuunits') {
4936                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4937             } elsif ($opt eq 'cpulimit') {
4938                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4939                 $cgroup->change_cpu_quota($cpulimit, 100000);
4940             } elsif ($opt eq 'agent') {
4941                 vmconfig_update_agent($conf, $opt, $value);
4942             } else {
4943                 die "skip\n";  # skip non-hot-pluggable options
4944             }
4945         };
4946         if (my $err = $@) {
4947             &$add_error($opt, $err) if $err ne "skip\n";
4948         } else {
4949             $conf->{$opt} = $value;
4950             delete $conf->{pending}->{$opt};
4951         }
4952     }
4953
4954     PVE::QemuConfig->write_config($vmid, $conf);
4955 }
4956
4957 sub try_deallocate_drive {
4958     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4959
4960     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4961         my $volid = $drive->{file};
4962         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4963             my $sid = PVE::Storage::parse_volume_id($volid);
4964             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4965
4966             # check if the disk is really unused
4967             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4968                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4969             PVE::Storage::vdisk_free($storecfg, $volid);
4970             return 1;
4971         } else {
4972             # If vm is not owner of this disk remove from config
4973             return 1;
4974         }
4975     }
4976
4977     return;
4978 }
4979
4980 sub vmconfig_delete_or_detach_drive {
4981     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4982
4983     my $drive = parse_drive($opt, $conf->{$opt});
4984
4985     my $rpcenv = PVE::RPCEnvironment::get();
4986     my $authuser = $rpcenv->get_user();
4987
4988     if ($force) {
4989         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4990         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4991     } else {
4992         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4993     }
4994 }
4995
4996
4997
4998 sub vmconfig_apply_pending {
4999     my ($vmid, $conf, $storecfg, $errors) = @_;
5000
5001     my $add_apply_error = sub {
5002         my ($opt, $msg) = @_;
5003         my $err_msg = "unable to apply pending change $opt : $msg";
5004         $errors->{$opt} = $err_msg;
5005         warn $err_msg;
5006     };
5007
5008     # cold plug
5009
5010     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5011     foreach my $opt (sort keys %$pending_delete_hash) {
5012         my $force = $pending_delete_hash->{$opt}->{force};
5013         eval {
5014             if ($opt =~ m/^unused/) {
5015                 die "internal error";
5016             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5017                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5018             }
5019         };
5020         if (my $err = $@) {
5021             $add_apply_error->($opt, $err);
5022         } else {
5023             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5024             delete $conf->{$opt};
5025         }
5026     }
5027
5028     PVE::QemuConfig->cleanup_pending($conf);
5029
5030     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5031         next if $opt eq 'delete'; # just to be sure
5032         eval {
5033             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5034                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5035             }
5036         };
5037         if (my $err = $@) {
5038             $add_apply_error->($opt, $err);
5039         } else {
5040             $conf->{$opt} = delete $conf->{pending}->{$opt};
5041         }
5042     }
5043
5044     # write all changes at once to avoid unnecessary i/o
5045     PVE::QemuConfig->write_config($vmid, $conf);
5046 }
5047
5048 sub vmconfig_update_net {
5049     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5050
5051     my $newnet = parse_net($value);
5052
5053     if ($conf->{$opt}) {
5054         my $oldnet = parse_net($conf->{$opt});
5055
5056         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5057             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5058             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5059             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5060
5061             # for non online change, we try to hot-unplug
5062             die "skip\n" if !$hotplug;
5063             vm_deviceunplug($vmid, $conf, $opt);
5064         } else {
5065
5066             die "internal error" if $opt !~ m/net(\d+)/;
5067             my $iface = "tap${vmid}i$1";
5068
5069             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5070                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5071                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5072                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5073                 PVE::Network::tap_unplug($iface);
5074
5075                 if ($have_sdn) {
5076                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5077                 } else {
5078                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5079                 }
5080             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5081                 # Rate can be applied on its own but any change above needs to
5082                 # include the rate in tap_plug since OVS resets everything.
5083                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5084             }
5085
5086             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5087                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5088             }
5089
5090             return 1;
5091         }
5092     }
5093
5094     if ($hotplug) {
5095         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5096     } else {
5097         die "skip\n";
5098     }
5099 }
5100
5101 sub vmconfig_update_agent {
5102     my ($conf, $opt, $value) = @_;
5103
5104     die "skip\n" if !$conf->{$opt};
5105
5106     my $hotplug_options = { fstrim_cloned_disks => 1 };
5107
5108     my $old_agent = parse_guest_agent($conf);
5109     my $agent = parse_guest_agent({$opt => $value});
5110
5111     for my $option (keys %$agent) { # added/changed options
5112         next if defined($hotplug_options->{$option});
5113         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5114     }
5115
5116     for my $option (keys %$old_agent) { # removed options
5117         next if defined($hotplug_options->{$option});
5118         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5119     }
5120
5121     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5122 }
5123
5124 sub vmconfig_update_disk {
5125     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5126
5127     my $drive = parse_drive($opt, $value);
5128
5129     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5130         my $media = $drive->{media} || 'disk';
5131         my $oldmedia = $old_drive->{media} || 'disk';
5132         die "unable to change media type\n" if $media ne $oldmedia;
5133
5134         if (!drive_is_cdrom($old_drive)) {
5135
5136             if ($drive->{file} ne $old_drive->{file}) {
5137
5138                 die "skip\n" if !$hotplug;
5139
5140                 # unplug and register as unused
5141                 vm_deviceunplug($vmid, $conf, $opt);
5142                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5143
5144             } else {
5145                 # update existing disk
5146
5147                 # skip non hotpluggable value
5148                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5149                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5150                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5151                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5152                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5153                     die "skip\n";
5154                 }
5155
5156                 # apply throttle
5157                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5158                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5159                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5160                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5161                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5162                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5163                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5164                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5165                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5166                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5167                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5168                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5169                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5170                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5171                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5172                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5173                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5174                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5175
5176                     qemu_block_set_io_throttle(
5177                         $vmid,"drive-$opt",
5178                         ($drive->{mbps} || 0)*1024*1024,
5179                         ($drive->{mbps_rd} || 0)*1024*1024,
5180                         ($drive->{mbps_wr} || 0)*1024*1024,
5181                          $drive->{iops} || 0,
5182                          $drive->{iops_rd} || 0,
5183                          $drive->{iops_wr} || 0,
5184                         ($drive->{mbps_max} || 0)*1024*1024,
5185                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5186                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5187                          $drive->{iops_max} || 0,
5188                          $drive->{iops_rd_max} || 0,
5189                          $drive->{iops_wr_max} || 0,
5190                          $drive->{bps_max_length} || 1,
5191                          $drive->{bps_rd_max_length} || 1,
5192                          $drive->{bps_wr_max_length} || 1,
5193                          $drive->{iops_max_length} || 1,
5194                          $drive->{iops_rd_max_length} || 1,
5195                          $drive->{iops_wr_max_length} || 1,
5196                     );
5197
5198                 }
5199
5200                 return 1;
5201             }
5202
5203         } else { # cdrom
5204
5205             if ($drive->{file} eq 'none') {
5206                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5207                 if (drive_is_cloudinit($old_drive)) {
5208                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5209                 }
5210             } else {
5211                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5212
5213                 # force eject if locked
5214                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5215
5216                 if ($path) {
5217                     mon_cmd($vmid, "blockdev-change-medium",
5218                         id => "$opt", filename => "$path");
5219                 }
5220             }
5221
5222             return 1;
5223         }
5224     }
5225
5226     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5227     # hotplug new disks
5228     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5229     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5230 }
5231
5232 # called in locked context by incoming migration
5233 sub vm_migrate_get_nbd_disks {
5234     my ($storecfg, $conf, $replicated_volumes) = @_;
5235
5236     my $local_volumes = {};
5237     PVE::QemuConfig->foreach_volume($conf, sub {
5238         my ($ds, $drive) = @_;
5239
5240         return if drive_is_cdrom($drive);
5241
5242         my $volid = $drive->{file};
5243
5244         return if !$volid;
5245
5246         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5247
5248         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5249         return if $scfg->{shared};
5250
5251         # replicated disks re-use existing state via bitmap
5252         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5253         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5254     });
5255     return $local_volumes;
5256 }
5257
5258 # called in locked context by incoming migration
5259 sub vm_migrate_alloc_nbd_disks {
5260     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5261
5262     my $format = undef;
5263
5264     my $nbd = {};
5265     foreach my $opt (sort keys %$source_volumes) {
5266         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5267
5268         if ($use_existing) {
5269             $nbd->{$opt}->{drivestr} = print_drive($drive);
5270             $nbd->{$opt}->{volid} = $volid;
5271             $nbd->{$opt}->{replicated} = 1;
5272             next;
5273         }
5274
5275         # If a remote storage is specified and the format of the original
5276         # volume is not available there, fall back to the default format.
5277         # Otherwise use the same format as the original.
5278         if (!$storagemap->{identity}) {
5279             $storeid = map_storage($storagemap, $storeid);
5280             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5281             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5282             my $fileFormat = qemu_img_format($scfg, $volname);
5283             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5284         } else {
5285             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5286             $format = qemu_img_format($scfg, $volname);
5287         }
5288
5289         my $size = $drive->{size} / 1024;
5290         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5291         my $newdrive = $drive;
5292         $newdrive->{format} = $format;
5293         $newdrive->{file} = $newvolid;
5294         my $drivestr = print_drive($newdrive);
5295         $nbd->{$opt}->{drivestr} = $drivestr;
5296         $nbd->{$opt}->{volid} = $newvolid;
5297     }
5298
5299     return $nbd;
5300 }
5301
5302 # see vm_start_nolock for parameters, additionally:
5303 # migrate_opts:
5304 #   storagemap = parsed storage map for allocating NBD disks
5305 sub vm_start {
5306     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5307
5308     return PVE::QemuConfig->lock_config($vmid, sub {
5309         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5310
5311         die "you can't start a vm if it's a template\n"
5312             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5313
5314         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5315         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5316
5317         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5318
5319         if ($has_backup_lock && $running) {
5320             # a backup is currently running, attempt to start the guest in the
5321             # existing QEMU instance
5322             return vm_resume($vmid);
5323         }
5324
5325         PVE::QemuConfig->check_lock($conf)
5326             if !($params->{skiplock} || $has_suspended_lock);
5327
5328         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5329
5330         die "VM $vmid already running\n" if $running;
5331
5332         if (my $storagemap = $migrate_opts->{storagemap}) {
5333             my $replicated = $migrate_opts->{replicated_volumes};
5334             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5335             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5336
5337             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5338                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5339             }
5340         }
5341
5342         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5343     });
5344 }
5345
5346
5347 # params:
5348 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5349 #   skiplock => 0/1, skip checking for config lock
5350 #   skiptemplate => 0/1, skip checking whether VM is template
5351 #   forcemachine => to force Qemu machine (rollback/migration)
5352 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5353 #   timeout => in seconds
5354 #   paused => start VM in paused state (backup)
5355 #   resume => resume from hibernation
5356 #   pbs-backing => {
5357 #      sata0 => {
5358 #         repository
5359 #         snapshot
5360 #         keyfile
5361 #         archive
5362 #      },
5363 #      virtio2 => ...
5364 #   }
5365 # migrate_opts:
5366 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5367 #   migratedfrom => source node
5368 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5369 #   network => CIDR of migration network
5370 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5371 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5372 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5373 sub vm_start_nolock {
5374     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5375
5376     my $statefile = $params->{statefile};
5377     my $resume = $params->{resume};
5378
5379     my $migratedfrom = $migrate_opts->{migratedfrom};
5380     my $migration_type = $migrate_opts->{type};
5381
5382     my $res = {};
5383
5384     # clean up leftover reboot request files
5385     eval { clear_reboot_request($vmid); };
5386     warn $@ if $@;
5387
5388     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5389         vmconfig_apply_pending($vmid, $conf, $storecfg);
5390         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5391     }
5392
5393     # don't regenerate the ISO if the VM is started as part of a live migration
5394     # this way we can reuse the old ISO with the correct config
5395     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5396
5397     my $defaults = load_defaults();
5398
5399     # set environment variable useful inside network script
5400     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5401
5402     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5403
5404     my $forcemachine = $params->{forcemachine};
5405     my $forcecpu = $params->{forcecpu};
5406     if ($resume) {
5407         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5408         $forcemachine = $conf->{runningmachine};
5409         $forcecpu = $conf->{runningcpu};
5410         print "Resuming suspended VM\n";
5411     }
5412
5413     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5414         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5415
5416     my $migration_ip;
5417     my $get_migration_ip = sub {
5418         my ($nodename) = @_;
5419
5420         return $migration_ip if defined($migration_ip);
5421
5422         my $cidr = $migrate_opts->{network};
5423
5424         if (!defined($cidr)) {
5425             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426             $cidr = $dc_conf->{migration}->{network};
5427         }
5428
5429         if (defined($cidr)) {
5430             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5431
5432             die "could not get IP: no address configured on local " .
5433                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5434
5435             die "could not get IP: multiple addresses configured on local " .
5436                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5437
5438             $migration_ip = @$ips[0];
5439         }
5440
5441         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5442             if !defined($migration_ip);
5443
5444         return $migration_ip;
5445     };
5446
5447     my $migrate_uri;
5448     if ($statefile) {
5449         if ($statefile eq 'tcp') {
5450             my $localip = "localhost";
5451             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5452             my $nodename = nodename();
5453
5454             if (!defined($migration_type)) {
5455                 if (defined($datacenterconf->{migration}->{type})) {
5456                     $migration_type = $datacenterconf->{migration}->{type};
5457                 } else {
5458                     $migration_type = 'secure';
5459                 }
5460             }
5461
5462             if ($migration_type eq 'insecure') {
5463                 $localip = $get_migration_ip->($nodename);
5464                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5465             }
5466
5467             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5468             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5469             $migrate_uri = "tcp:${localip}:${migrate_port}";
5470             push @$cmd, '-incoming', $migrate_uri;
5471             push @$cmd, '-S';
5472
5473         } elsif ($statefile eq 'unix') {
5474             # should be default for secure migrations as a ssh TCP forward
5475             # tunnel is not deterministic reliable ready and fails regurarly
5476             # to set up in time, so use UNIX socket forwards
5477             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5478             unlink $socket_addr;
5479
5480             $migrate_uri = "unix:$socket_addr";
5481
5482             push @$cmd, '-incoming', $migrate_uri;
5483             push @$cmd, '-S';
5484
5485         } elsif (-e $statefile) {
5486             push @$cmd, '-loadstate', $statefile;
5487         } else {
5488             my $statepath = PVE::Storage::path($storecfg, $statefile);
5489             push @$vollist, $statefile;
5490             push @$cmd, '-loadstate', $statepath;
5491         }
5492     } elsif ($params->{paused}) {
5493         push @$cmd, '-S';
5494     }
5495
5496     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5497
5498     my $pci_devices = {}; # host pci devices
5499     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5500         my $dev = $conf->{"hostpci$i"} or next;
5501         $pci_devices->{$i} = parse_hostpci($dev);
5502     }
5503
5504     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5505     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5506
5507     # map to a flat list of pci ids
5508     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5509
5510     # reserve all PCI IDs before actually doing anything with them
5511     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5512
5513     eval {
5514         for my $id (sort keys %$pci_devices) {
5515             my $d = $pci_devices->{$id};
5516             for my $dev ($d->{pciid}->@*) {
5517                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5518             }
5519         }
5520     };
5521     if (my $err = $@) {
5522         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5523         warn $@ if $@;
5524         die $err;
5525     }
5526
5527     PVE::Storage::activate_volumes($storecfg, $vollist);
5528
5529     eval {
5530         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5531     };
5532     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5533     # timeout should be more than enough here...
5534     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5535
5536     my $cpuunits = get_cpuunits($conf);
5537
5538     my %run_params = (
5539         timeout => $statefile ? undef : $start_timeout,
5540         umask => 0077,
5541         noerr => 1,
5542     );
5543
5544     # when migrating, prefix QEMU output so other side can pick up any
5545     # errors that might occur and show the user
5546     if ($migratedfrom) {
5547         $run_params{quiet} = 1;
5548         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5549     }
5550
5551     my %systemd_properties = (
5552         Slice => 'qemu.slice',
5553         KillMode => 'process',
5554         SendSIGKILL => 0,
5555         TimeoutStopUSec => ULONG_MAX, # infinity
5556     );
5557
5558     if (PVE::CGroup::cgroup_mode() == 2) {
5559         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5560         $systemd_properties{CPUWeight} = $cpuunits;
5561     } else {
5562         $systemd_properties{CPUShares} = $cpuunits;
5563     }
5564
5565     if (my $cpulimit = $conf->{cpulimit}) {
5566         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5567     }
5568     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5569
5570     my $run_qemu = sub {
5571         PVE::Tools::run_fork sub {
5572             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5573
5574             my $tpmpid;
5575             if (my $tpm = $conf->{tpmstate0}) {
5576                 # start the TPM emulator so QEMU can connect on start
5577                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5578             }
5579
5580             my $exitcode = run_command($cmd, %run_params);
5581             if ($exitcode) {
5582                 if ($tpmpid) {
5583                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5584                     kill 'TERM', $tpmpid;
5585                 }
5586                 die "QEMU exited with code $exitcode\n";
5587             }
5588         };
5589     };
5590
5591     if ($conf->{hugepages}) {
5592
5593         my $code = sub {
5594             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5595             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5596
5597             PVE::QemuServer::Memory::hugepages_mount();
5598             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5599
5600             eval { $run_qemu->() };
5601             if (my $err = $@) {
5602                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5603                     if !$conf->{keephugepages};
5604                 die $err;
5605             }
5606
5607             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5608                 if !$conf->{keephugepages};
5609         };
5610         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5611
5612     } else {
5613         eval { $run_qemu->() };
5614     }
5615
5616     if (my $err = $@) {
5617         # deactivate volumes if start fails
5618         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5619         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5620
5621         die "start failed: $err";
5622     }
5623
5624     # re-reserve all PCI IDs now that we can know the actual VM PID
5625     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5626     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5627     warn $@ if $@;
5628
5629     print "migration listens on $migrate_uri\n" if $migrate_uri;
5630     $res->{migrate_uri} = $migrate_uri;
5631
5632     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5633         eval { mon_cmd($vmid, "cont"); };
5634         warn $@ if $@;
5635     }
5636
5637     #start nbd server for storage migration
5638     if (my $nbd = $migrate_opts->{nbd}) {
5639         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5640
5641         my $migrate_storage_uri;
5642         # nbd_protocol_version > 0 for unix socket support
5643         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5644             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5645             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5646             $migrate_storage_uri = "nbd:unix:$socket_path";
5647         } else {
5648             my $nodename = nodename();
5649             my $localip = $get_migration_ip->($nodename);
5650             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5651             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5652
5653             mon_cmd($vmid, "nbd-server-start", addr => {
5654                 type => 'inet',
5655                 data => {
5656                     host => "${localip}",
5657                     port => "${storage_migrate_port}",
5658                 },
5659             });
5660             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5661             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5662         }
5663
5664         $res->{migrate_storage_uri} = $migrate_storage_uri;
5665
5666         foreach my $opt (sort keys %$nbd) {
5667             my $drivestr = $nbd->{$opt}->{drivestr};
5668             my $volid = $nbd->{$opt}->{volid};
5669             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5670             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5671             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5672             print "re-using replicated volume: $opt - $volid\n"
5673                 if $nbd->{$opt}->{replicated};
5674
5675             $res->{drives}->{$opt} = $nbd->{$opt};
5676             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5677         }
5678     }
5679
5680     if ($migratedfrom) {
5681         eval {
5682             set_migration_caps($vmid);
5683         };
5684         warn $@ if $@;
5685
5686         if ($spice_port) {
5687             print "spice listens on port $spice_port\n";
5688             $res->{spice_port} = $spice_port;
5689             if ($migrate_opts->{spice_ticket}) {
5690                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5691                     $migrate_opts->{spice_ticket});
5692                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5693             }
5694         }
5695
5696     } else {
5697         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5698             if !$statefile && $conf->{balloon};
5699
5700         foreach my $opt (keys %$conf) {
5701             next if $opt !~  m/^net\d+$/;
5702             my $nicconf = parse_net($conf->{$opt});
5703             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5704         }
5705     }
5706
5707     mon_cmd($vmid, 'qom-set',
5708                 path => "machine/peripheral/balloon0",
5709                 property => "guest-stats-polling-interval",
5710                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5711
5712     if ($resume) {
5713         print "Resumed VM, removing state\n";
5714         if (my $vmstate = $conf->{vmstate}) {
5715             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5716             PVE::Storage::vdisk_free($storecfg, $vmstate);
5717         }
5718         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5719         PVE::QemuConfig->write_config($vmid, $conf);
5720     }
5721
5722     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5723
5724     return $res;
5725 }
5726
5727 sub vm_commandline {
5728     my ($storecfg, $vmid, $snapname) = @_;
5729
5730     my $conf = PVE::QemuConfig->load_config($vmid);
5731
5732     my ($forcemachine, $forcecpu);
5733     if ($snapname) {
5734         my $snapshot = $conf->{snapshots}->{$snapname};
5735         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5736
5737         # check for machine or CPU overrides in snapshot
5738         $forcemachine = $snapshot->{runningmachine};
5739         $forcecpu = $snapshot->{runningcpu};
5740
5741         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5742
5743         $conf = $snapshot;
5744     }
5745
5746     my $defaults = load_defaults();
5747
5748     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5749
5750     return PVE::Tools::cmd2string($cmd);
5751 }
5752
5753 sub vm_reset {
5754     my ($vmid, $skiplock) = @_;
5755
5756     PVE::QemuConfig->lock_config($vmid, sub {
5757
5758         my $conf = PVE::QemuConfig->load_config($vmid);
5759
5760         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5761
5762         mon_cmd($vmid, "system_reset");
5763     });
5764 }
5765
5766 sub get_vm_volumes {
5767     my ($conf) = @_;
5768
5769     my $vollist = [];
5770     foreach_volid($conf, sub {
5771         my ($volid, $attr) = @_;
5772
5773         return if $volid =~ m|^/|;
5774
5775         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5776         return if !$sid;
5777
5778         push @$vollist, $volid;
5779     });
5780
5781     return $vollist;
5782 }
5783
5784 sub vm_stop_cleanup {
5785     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5786
5787     eval {
5788
5789         if (!$keepActive) {
5790             my $vollist = get_vm_volumes($conf);
5791             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5792
5793             if (my $tpmdrive = $conf->{tpmstate0}) {
5794                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5795                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5796                 if ($storeid) {
5797                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5798                 }
5799             }
5800         }
5801
5802         foreach my $ext (qw(mon qmp pid vnc qga)) {
5803             unlink "/var/run/qemu-server/${vmid}.$ext";
5804         }
5805
5806         if ($conf->{ivshmem}) {
5807             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5808             # just delete it for now, VMs which have this already open do not
5809             # are affected, but new VMs will get a separated one. If this
5810             # becomes an issue we either add some sort of ref-counting or just
5811             # add a "don't delete on stop" flag to the ivshmem format.
5812             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5813         }
5814
5815         my $ids = [];
5816         foreach my $key (keys %$conf) {
5817             next if $key !~ m/^hostpci(\d+)$/;
5818             my $hostpciindex = $1;
5819             my $d = parse_hostpci($conf->{$key});
5820             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5821
5822             foreach my $pci (@{$d->{pciid}}) {
5823                 my $pciid = $pci->{id};
5824                 push @$ids, $pci->{id};
5825                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5826             }
5827         }
5828         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5829
5830         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5831     };
5832     warn $@ if $@; # avoid errors - just warn
5833 }
5834
5835 # call only in locked context
5836 sub _do_vm_stop {
5837     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5838
5839     my $pid = check_running($vmid, $nocheck);
5840     return if !$pid;
5841
5842     my $conf;
5843     if (!$nocheck) {
5844         $conf = PVE::QemuConfig->load_config($vmid);
5845         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5846         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5847             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5848             $timeout = $opts->{down} if $opts->{down};
5849         }
5850         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5851     }
5852
5853     eval {
5854         if ($shutdown) {
5855             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5856                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5857             } else {
5858                 mon_cmd($vmid, "system_powerdown");
5859             }
5860         } else {
5861             mon_cmd($vmid, "quit");
5862         }
5863     };
5864     my $err = $@;
5865
5866     if (!$err) {
5867         $timeout = 60 if !defined($timeout);
5868
5869         my $count = 0;
5870         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5871             $count++;
5872             sleep 1;
5873         }
5874
5875         if ($count >= $timeout) {
5876             if ($force) {
5877                 warn "VM still running - terminating now with SIGTERM\n";
5878                 kill 15, $pid;
5879             } else {
5880                 die "VM quit/powerdown failed - got timeout\n";
5881             }
5882         } else {
5883             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5884             return;
5885         }
5886     } else {
5887        if (!check_running($vmid, $nocheck)) {
5888             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5889             return;
5890        }
5891        if ($force) {
5892             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5893             kill 15, $pid;
5894         } else {
5895             die "VM quit/powerdown failed\n";
5896         }
5897     }
5898
5899     # wait again
5900     $timeout = 10;
5901
5902     my $count = 0;
5903     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5904         $count++;
5905         sleep 1;
5906     }
5907
5908     if ($count >= $timeout) {
5909         warn "VM still running - terminating now with SIGKILL\n";
5910         kill 9, $pid;
5911         sleep 1;
5912     }
5913
5914     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5915 }
5916
5917 # Note: use $nocheck to skip tests if VM configuration file exists.
5918 # We need that when migration VMs to other nodes (files already moved)
5919 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5920 sub vm_stop {
5921     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5922
5923     $force = 1 if !defined($force) && !$shutdown;
5924
5925     if ($migratedfrom){
5926         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5927         kill 15, $pid if $pid;
5928         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5929         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5930         return;
5931     }
5932
5933     PVE::QemuConfig->lock_config($vmid, sub {
5934         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5935    });
5936 }
5937
5938 sub vm_reboot {
5939     my ($vmid, $timeout) = @_;
5940
5941     PVE::QemuConfig->lock_config($vmid, sub {
5942         eval {
5943
5944             # only reboot if running, as qmeventd starts it again on a stop event
5945             return if !check_running($vmid);
5946
5947             create_reboot_request($vmid);
5948
5949             my $storecfg = PVE::Storage::config();
5950             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5951
5952         };
5953         if (my $err = $@) {
5954             # avoid that the next normal shutdown will be confused for a reboot
5955             clear_reboot_request($vmid);
5956             die $err;
5957         }
5958    });
5959 }
5960
5961 # note: if using the statestorage parameter, the caller has to check privileges
5962 sub vm_suspend {
5963     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5964
5965     my $conf;
5966     my $path;
5967     my $storecfg;
5968     my $vmstate;
5969
5970     PVE::QemuConfig->lock_config($vmid, sub {
5971
5972         $conf = PVE::QemuConfig->load_config($vmid);
5973
5974         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5975         PVE::QemuConfig->check_lock($conf)
5976             if !($skiplock || $is_backing_up);
5977
5978         die "cannot suspend to disk during backup\n"
5979             if $is_backing_up && $includestate;
5980
5981         if ($includestate) {
5982             $conf->{lock} = 'suspending';
5983             my $date = strftime("%Y-%m-%d", localtime(time()));
5984             $storecfg = PVE::Storage::config();
5985             if (!$statestorage) {
5986                 $statestorage = find_vmstate_storage($conf, $storecfg);
5987                 # check permissions for the storage
5988                 my $rpcenv = PVE::RPCEnvironment::get();
5989                 if ($rpcenv->{type} ne 'cli') {
5990                     my $authuser = $rpcenv->get_user();
5991                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5992                 }
5993             }
5994
5995
5996             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5997                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5998             $path = PVE::Storage::path($storecfg, $vmstate);
5999             PVE::QemuConfig->write_config($vmid, $conf);
6000         } else {
6001             mon_cmd($vmid, "stop");
6002         }
6003     });
6004
6005     if ($includestate) {
6006         # save vm state
6007         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6008
6009         eval {
6010             set_migration_caps($vmid, 1);
6011             mon_cmd($vmid, "savevm-start", statefile => $path);
6012             for(;;) {
6013                 my $state = mon_cmd($vmid, "query-savevm");
6014                 if (!$state->{status}) {
6015                     die "savevm not active\n";
6016                 } elsif ($state->{status} eq 'active') {
6017                     sleep(1);
6018                     next;
6019                 } elsif ($state->{status} eq 'completed') {
6020                     print "State saved, quitting\n";
6021                     last;
6022                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6023                     die "query-savevm failed with error '$state->{error}'\n"
6024                 } else {
6025                     die "query-savevm returned status '$state->{status}'\n";
6026                 }
6027             }
6028         };
6029         my $err = $@;
6030
6031         PVE::QemuConfig->lock_config($vmid, sub {
6032             $conf = PVE::QemuConfig->load_config($vmid);
6033             if ($err) {
6034                 # cleanup, but leave suspending lock, to indicate something went wrong
6035                 eval {
6036                     mon_cmd($vmid, "savevm-end");
6037                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6038                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6039                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6040                     PVE::QemuConfig->write_config($vmid, $conf);
6041                 };
6042                 warn $@ if $@;
6043                 die $err;
6044             }
6045
6046             die "lock changed unexpectedly\n"
6047                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6048
6049             mon_cmd($vmid, "quit");
6050             $conf->{lock} = 'suspended';
6051             PVE::QemuConfig->write_config($vmid, $conf);
6052         });
6053     }
6054 }
6055
6056 sub vm_resume {
6057     my ($vmid, $skiplock, $nocheck) = @_;
6058
6059     PVE::QemuConfig->lock_config($vmid, sub {
6060         my $res = mon_cmd($vmid, 'query-status');
6061         my $resume_cmd = 'cont';
6062         my $reset = 0;
6063
6064         if ($res->{status}) {
6065             return if $res->{status} eq 'running'; # job done, go home
6066             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6067             $reset = 1 if $res->{status} eq 'shutdown';
6068         }
6069
6070         if (!$nocheck) {
6071
6072             my $conf = PVE::QemuConfig->load_config($vmid);
6073
6074             PVE::QemuConfig->check_lock($conf)
6075                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6076         }
6077
6078         if ($reset) {
6079             # required if a VM shuts down during a backup and we get a resume
6080             # request before the backup finishes for example
6081             mon_cmd($vmid, "system_reset");
6082         }
6083         mon_cmd($vmid, $resume_cmd);
6084     });
6085 }
6086
6087 sub vm_sendkey {
6088     my ($vmid, $skiplock, $key) = @_;
6089
6090     PVE::QemuConfig->lock_config($vmid, sub {
6091
6092         my $conf = PVE::QemuConfig->load_config($vmid);
6093
6094         # there is no qmp command, so we use the human monitor command
6095         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6096         die $res if $res ne '';
6097     });
6098 }
6099
6100 # vzdump restore implementaion
6101
6102 sub tar_archive_read_firstfile {
6103     my $archive = shift;
6104
6105     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6106
6107     # try to detect archive type first
6108     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6109         die "unable to open file '$archive'\n";
6110     my $firstfile = <$fh>;
6111     kill 15, $pid;
6112     close $fh;
6113
6114     die "ERROR: archive contaions no data\n" if !$firstfile;
6115     chomp $firstfile;
6116
6117     return $firstfile;
6118 }
6119
6120 sub tar_restore_cleanup {
6121     my ($storecfg, $statfile) = @_;
6122
6123     print STDERR "starting cleanup\n";
6124
6125     if (my $fd = IO::File->new($statfile, "r")) {
6126         while (defined(my $line = <$fd>)) {
6127             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6128                 my $volid = $2;
6129                 eval {
6130                     if ($volid =~ m|^/|) {
6131                         unlink $volid || die 'unlink failed\n';
6132                     } else {
6133                         PVE::Storage::vdisk_free($storecfg, $volid);
6134                     }
6135                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6136                 };
6137                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6138             } else {
6139                 print STDERR "unable to parse line in statfile - $line";
6140             }
6141         }
6142         $fd->close();
6143     }
6144 }
6145
6146 sub restore_file_archive {
6147     my ($archive, $vmid, $user, $opts) = @_;
6148
6149     return restore_vma_archive($archive, $vmid, $user, $opts)
6150         if $archive eq '-';
6151
6152     my $info = PVE::Storage::archive_info($archive);
6153     my $format = $opts->{format} // $info->{format};
6154     my $comp = $info->{compression};
6155
6156     # try to detect archive format
6157     if ($format eq 'tar') {
6158         return restore_tar_archive($archive, $vmid, $user, $opts);
6159     } else {
6160         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6161     }
6162 }
6163
6164 # hepler to remove disks that will not be used after restore
6165 my $restore_cleanup_oldconf = sub {
6166     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6167
6168     PVE::QemuConfig->foreach_volume($oldconf, sub {
6169         my ($ds, $drive) = @_;
6170
6171         return if drive_is_cdrom($drive, 1);
6172
6173         my $volid = $drive->{file};
6174         return if !$volid || $volid =~ m|^/|;
6175
6176         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6177         return if !$path || !$owner || ($owner != $vmid);
6178
6179         # Note: only delete disk we want to restore
6180         # other volumes will become unused
6181         if ($virtdev_hash->{$ds}) {
6182             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6183             if (my $err = $@) {
6184                 warn $err;
6185             }
6186         }
6187     });
6188
6189     # delete vmstate files, after the restore we have no snapshots anymore
6190     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6191         my $snap = $oldconf->{snapshots}->{$snapname};
6192         if ($snap->{vmstate}) {
6193             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6194             if (my $err = $@) {
6195                 warn $err;
6196             }
6197         }
6198     }
6199 };
6200
6201 # Helper to parse vzdump backup device hints
6202 #
6203 # $rpcenv: Environment, used to ckeck storage permissions
6204 # $user: User ID, to check storage permissions
6205 # $storecfg: Storage configuration
6206 # $fh: the file handle for reading the configuration
6207 # $devinfo: should contain device sizes for all backu-up'ed devices
6208 # $options: backup options (pool, default storage)
6209 #
6210 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6211 my $parse_backup_hints = sub {
6212     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6213
6214     my $virtdev_hash = {};
6215
6216     while (defined(my $line = <$fh>)) {
6217         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6218             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6219             die "archive does not contain data for drive '$virtdev'\n"
6220                 if !$devinfo->{$devname};
6221
6222             if (defined($options->{storage})) {
6223                 $storeid = $options->{storage} || 'local';
6224             } elsif (!$storeid) {
6225                 $storeid = 'local';
6226             }
6227             $format = 'raw' if !$format;
6228             $devinfo->{$devname}->{devname} = $devname;
6229             $devinfo->{$devname}->{virtdev} = $virtdev;
6230             $devinfo->{$devname}->{format} = $format;
6231             $devinfo->{$devname}->{storeid} = $storeid;
6232
6233             # check permission on storage
6234             my $pool = $options->{pool}; # todo: do we need that?
6235             if ($user ne 'root@pam') {
6236                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6237             }
6238
6239             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6240         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6241             my $virtdev = $1;
6242             my $drive = parse_drive($virtdev, $2);
6243             if (drive_is_cloudinit($drive)) {
6244                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6245                 $storeid = $options->{storage} if defined ($options->{storage});
6246                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6247                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6248
6249                 $virtdev_hash->{$virtdev} = {
6250                     format => $format,
6251                     storeid => $storeid,
6252                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6253                     is_cloudinit => 1,
6254                 };
6255             }
6256         }
6257     }
6258
6259     return $virtdev_hash;
6260 };
6261
6262 # Helper to allocate and activate all volumes required for a restore
6263 #
6264 # $storecfg: Storage configuration
6265 # $virtdev_hash: as returned by parse_backup_hints()
6266 #
6267 # Returns: { $virtdev => $volid }
6268 my $restore_allocate_devices = sub {
6269     my ($storecfg, $virtdev_hash, $vmid) = @_;
6270
6271     my $map = {};
6272     foreach my $virtdev (sort keys %$virtdev_hash) {
6273         my $d = $virtdev_hash->{$virtdev};
6274         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6275         my $storeid = $d->{storeid};
6276         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6277
6278         # test if requested format is supported
6279         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6280         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6281         $d->{format} = $defFormat if !$supported;
6282
6283         my $name;
6284         if ($d->{is_cloudinit}) {
6285             $name = "vm-$vmid-cloudinit";
6286             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6287             if ($scfg->{path}) {
6288                 $name .= ".$d->{format}";
6289             }
6290         }
6291
6292         my $volid = PVE::Storage::vdisk_alloc(
6293             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6294
6295         print STDERR "new volume ID is '$volid'\n";
6296         $d->{volid} = $volid;
6297
6298         PVE::Storage::activate_volumes($storecfg, [$volid]);
6299
6300         $map->{$virtdev} = $volid;
6301     }
6302
6303     return $map;
6304 };
6305
6306 sub restore_update_config_line {
6307     my ($cookie, $map, $line, $unique) = @_;
6308
6309     return '' if $line =~ m/^\#qmdump\#/;
6310     return '' if $line =~ m/^\#vzdump\#/;
6311     return '' if $line =~ m/^lock:/;
6312     return '' if $line =~ m/^unused\d+:/;
6313     return '' if $line =~ m/^parent:/;
6314
6315     my $res = '';
6316
6317     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6318     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6319         # try to convert old 1.X settings
6320         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6321         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6322             my ($model, $macaddr) = split(/\=/, $devconfig);
6323             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6324             my $net = {
6325                 model => $model,
6326                 bridge => "vmbr$ind",
6327                 macaddr => $macaddr,
6328             };
6329             my $netstr = print_net($net);
6330
6331             $res .= "net$cookie->{netcount}: $netstr\n";
6332             $cookie->{netcount}++;
6333         }
6334     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6335         my ($id, $netstr) = ($1, $2);
6336         my $net = parse_net($netstr);
6337         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6338         $netstr = print_net($net);
6339         $res .= "$id: $netstr\n";
6340     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6341         my $virtdev = $1;
6342         my $value = $3;
6343         my $di = parse_drive($virtdev, $value);
6344         if (defined($di->{backup}) && !$di->{backup}) {
6345             $res .= "#$line";
6346         } elsif ($map->{$virtdev}) {
6347             delete $di->{format}; # format can change on restore
6348             $di->{file} = $map->{$virtdev};
6349             $value = print_drive($di);
6350             $res .= "$virtdev: $value\n";
6351         } else {
6352             $res .= $line;
6353         }
6354     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6355         my $vmgenid = $1;
6356         if ($vmgenid ne '0') {
6357             # always generate a new vmgenid if there was a valid one setup
6358             $vmgenid = generate_uuid();
6359         }
6360         $res .= "vmgenid: $vmgenid\n";
6361     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6362         my ($uuid, $uuid_str);
6363         UUID::generate($uuid);
6364         UUID::unparse($uuid, $uuid_str);
6365         my $smbios1 = parse_smbios1($2);
6366         $smbios1->{uuid} = $uuid_str;
6367         $res .= $1.print_smbios1($smbios1)."\n";
6368     } else {
6369         $res .= $line;
6370     }
6371
6372     return $res;
6373 }
6374
6375 my $restore_deactivate_volumes = sub {
6376     my ($storecfg, $devinfo) = @_;
6377
6378     my $vollist = [];
6379     foreach my $devname (keys %$devinfo) {
6380         my $volid = $devinfo->{$devname}->{volid};
6381         push @$vollist, $volid if $volid;
6382     }
6383
6384     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6385 };
6386
6387 my $restore_destroy_volumes = sub {
6388     my ($storecfg, $devinfo) = @_;
6389
6390     foreach my $devname (keys %$devinfo) {
6391         my $volid = $devinfo->{$devname}->{volid};
6392         next if !$volid;
6393         eval {
6394             if ($volid =~ m|^/|) {
6395                 unlink $volid || die 'unlink failed\n';
6396             } else {
6397                 PVE::Storage::vdisk_free($storecfg, $volid);
6398             }
6399             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6400         };
6401         print STDERR "unable to cleanup '$volid' - $@" if $@;
6402     }
6403 };
6404
6405 sub scan_volids {
6406     my ($cfg, $vmid) = @_;
6407
6408     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6409
6410     my $volid_hash = {};
6411     foreach my $storeid (keys %$info) {
6412         foreach my $item (@{$info->{$storeid}}) {
6413             next if !($item->{volid} && $item->{size});
6414             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6415             $volid_hash->{$item->{volid}} = $item;
6416         }
6417     }
6418
6419     return $volid_hash;
6420 }
6421
6422 sub update_disk_config {
6423     my ($vmid, $conf, $volid_hash) = @_;
6424
6425     my $changes;
6426     my $prefix = "VM $vmid";
6427
6428     # used and unused disks
6429     my $referenced = {};
6430
6431     # Note: it is allowed to define multiple storages with same path (alias), so
6432     # we need to check both 'volid' and real 'path' (two different volid can point
6433     # to the same path).
6434
6435     my $referencedpath = {};
6436
6437     # update size info
6438     PVE::QemuConfig->foreach_volume($conf, sub {
6439         my ($opt, $drive) = @_;
6440
6441         my $volid = $drive->{file};
6442         return if !$volid;
6443         my $volume = $volid_hash->{$volid};
6444
6445         # mark volid as "in-use" for next step
6446         $referenced->{$volid} = 1;
6447         if ($volume && (my $path = $volume->{path})) {
6448             $referencedpath->{$path} = 1;
6449         }
6450
6451         return if drive_is_cdrom($drive);
6452         return if !$volume;
6453
6454         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6455         if (defined($updated)) {
6456             $changes = 1;
6457             $conf->{$opt} = print_drive($updated);
6458             print "$prefix ($opt): $msg\n";
6459         }
6460     });
6461
6462     # remove 'unusedX' entry if volume is used
6463     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6464         my ($opt, $drive) = @_;
6465
6466         my $volid = $drive->{file};
6467         return if !$volid;
6468
6469         my $path;
6470         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6471         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6472             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6473             $changes = 1;
6474             delete $conf->{$opt};
6475         }
6476
6477         $referenced->{$volid} = 1;
6478         $referencedpath->{$path} = 1 if $path;
6479     });
6480
6481     foreach my $volid (sort keys %$volid_hash) {
6482         next if $volid =~ m/vm-$vmid-state-/;
6483         next if $referenced->{$volid};
6484         my $path = $volid_hash->{$volid}->{path};
6485         next if !$path; # just to be sure
6486         next if $referencedpath->{$path};
6487         $changes = 1;
6488         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6489         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6490         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6491     }
6492
6493     return $changes;
6494 }
6495
6496 sub rescan {
6497     my ($vmid, $nolock, $dryrun) = @_;
6498
6499     my $cfg = PVE::Storage::config();
6500
6501     print "rescan volumes...\n";
6502     my $volid_hash = scan_volids($cfg, $vmid);
6503
6504     my $updatefn =  sub {
6505         my ($vmid) = @_;
6506
6507         my $conf = PVE::QemuConfig->load_config($vmid);
6508
6509         PVE::QemuConfig->check_lock($conf);
6510
6511         my $vm_volids = {};
6512         foreach my $volid (keys %$volid_hash) {
6513             my $info = $volid_hash->{$volid};
6514             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6515         }
6516
6517         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6518
6519         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6520     };
6521
6522     if (defined($vmid)) {
6523         if ($nolock) {
6524             &$updatefn($vmid);
6525         } else {
6526             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6527         }
6528     } else {
6529         my $vmlist = config_list();
6530         foreach my $vmid (keys %$vmlist) {
6531             if ($nolock) {
6532                 &$updatefn($vmid);
6533             } else {
6534                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6535             }
6536         }
6537     }
6538 }
6539
6540 sub restore_proxmox_backup_archive {
6541     my ($archive, $vmid, $user, $options) = @_;
6542
6543     my $storecfg = PVE::Storage::config();
6544
6545     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6546     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6547
6548     my $fingerprint = $scfg->{fingerprint};
6549     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6550
6551     my $repo = PVE::PBSClient::get_repository($scfg);
6552
6553     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6554     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6555     local $ENV{PBS_PASSWORD} = $password;
6556     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6557
6558     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6559         PVE::Storage::parse_volname($storecfg, $archive);
6560
6561     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6562
6563     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6564
6565     my $tmpdir = "/var/tmp/vzdumptmp$$";
6566     rmtree $tmpdir;
6567     mkpath $tmpdir;
6568
6569     my $conffile = PVE::QemuConfig->config_file($vmid);
6570      # disable interrupts (always do cleanups)
6571     local $SIG{INT} =
6572         local $SIG{TERM} =
6573         local $SIG{QUIT} =
6574         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6575
6576     # Note: $oldconf is undef if VM does not exists
6577     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6578     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6579     my $new_conf_raw = '';
6580
6581     my $rpcenv = PVE::RPCEnvironment::get();
6582     my $devinfo = {};
6583
6584     eval {
6585         # enable interrupts
6586         local $SIG{INT} =
6587             local $SIG{TERM} =
6588             local $SIG{QUIT} =
6589             local $SIG{HUP} =
6590             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6591
6592         my $cfgfn = "$tmpdir/qemu-server.conf";
6593         my $firewall_config_fn = "$tmpdir/fw.conf";
6594         my $index_fn = "$tmpdir/index.json";
6595
6596         my $cmd = "restore";
6597
6598         my $param = [$pbs_backup_name, "index.json", $index_fn];
6599         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6600         my $index = PVE::Tools::file_get_contents($index_fn);
6601         $index = decode_json($index);
6602
6603         # print Dumper($index);
6604         foreach my $info (@{$index->{files}}) {
6605             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6606                 my $devname = $1;
6607                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6608                     $devinfo->{$devname}->{size} = $1;
6609                 } else {
6610                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6611                 }
6612             }
6613         }
6614
6615         my $is_qemu_server_backup = scalar(
6616             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6617         );
6618         if (!$is_qemu_server_backup) {
6619             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6620         }
6621         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6622
6623         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6624         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6625
6626         if ($has_firewall_config) {
6627             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6628             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6629
6630             my $pve_firewall_dir = '/etc/pve/firewall';
6631             mkdir $pve_firewall_dir; # make sure the dir exists
6632             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6633         }
6634
6635         my $fh = IO::File->new($cfgfn, "r") ||
6636             die "unable to read qemu-server.conf - $!\n";
6637
6638         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6639
6640         # fixme: rate limit?
6641
6642         # create empty/temp config
6643         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6644
6645         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6646
6647         # allocate volumes
6648         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6649
6650         foreach my $virtdev (sort keys %$virtdev_hash) {
6651             my $d = $virtdev_hash->{$virtdev};
6652             next if $d->{is_cloudinit}; # no need to restore cloudinit
6653
6654             # this fails if storage is unavailable
6655             my $volid = $d->{volid};
6656             my $path = PVE::Storage::path($storecfg, $volid);
6657
6658             # for live-restore we only want to preload the efidisk and TPM state
6659             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6660
6661             my $pbs_restore_cmd = [
6662                 '/usr/bin/pbs-restore',
6663                 '--repository', $repo,
6664                 $pbs_backup_name,
6665                 "$d->{devname}.img.fidx",
6666                 $path,
6667                 '--verbose',
6668                 ];
6669
6670             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6671             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6672
6673             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6674                 push @$pbs_restore_cmd, '--skip-zero';
6675             }
6676
6677             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6678             print "restore proxmox backup image: $dbg_cmdstring\n";
6679             run_command($pbs_restore_cmd);
6680         }
6681
6682         $fh->seek(0, 0) || die "seek failed - $!\n";
6683
6684         my $cookie = { netcount => 0 };
6685         while (defined(my $line = <$fh>)) {
6686             $new_conf_raw .= restore_update_config_line(
6687                 $cookie,
6688                 $map,
6689                 $line,
6690                 $options->{unique},
6691             );
6692         }
6693
6694         $fh->close();
6695     };
6696     my $err = $@;
6697
6698     if ($err || !$options->{live}) {
6699         $restore_deactivate_volumes->($storecfg, $devinfo);
6700     }
6701
6702     rmtree $tmpdir;
6703
6704     if ($err) {
6705         $restore_destroy_volumes->($storecfg, $devinfo);
6706         die $err;
6707     }
6708
6709     if ($options->{live}) {
6710         # keep lock during live-restore
6711         $new_conf_raw .= "\nlock: create";
6712     }
6713
6714     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6715
6716     PVE::Cluster::cfs_update(); # make sure we read new file
6717
6718     eval { rescan($vmid, 1); };
6719     warn $@ if $@;
6720
6721     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6722
6723     if ($options->{live}) {
6724         # enable interrupts
6725         local $SIG{INT} =
6726             local $SIG{TERM} =
6727             local $SIG{QUIT} =
6728             local $SIG{HUP} =
6729             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6730
6731         my $conf = PVE::QemuConfig->load_config($vmid);
6732         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6733
6734         # these special drives are already restored before start
6735         delete $devinfo->{'drive-efidisk0'};
6736         delete $devinfo->{'drive-tpmstate0-backup'};
6737         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6738
6739         PVE::QemuConfig->remove_lock($vmid, "create");
6740     }
6741 }
6742
6743 sub pbs_live_restore {
6744     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6745
6746     print "starting VM for live-restore\n";
6747     print "repository: '$repo', snapshot: '$snap'\n";
6748
6749     my $pbs_backing = {};
6750     for my $ds (keys %$restored_disks) {
6751         $ds =~ m/^drive-(.*)$/;
6752         my $confname = $1;
6753         $pbs_backing->{$confname} = {
6754             repository => $repo,
6755             snapshot => $snap,
6756             archive => "$ds.img.fidx",
6757         };
6758         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6759
6760         my $drive = parse_drive($confname, $conf->{$confname});
6761         print "restoring '$ds' to '$drive->{file}'\n";
6762     }
6763
6764     my $drives_streamed = 0;
6765     eval {
6766         # make sure HA doesn't interrupt our restore by stopping the VM
6767         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6768             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6769         }
6770
6771         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6772         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6773         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6774
6775         my $qmeventd_fd = register_qmeventd_handle($vmid);
6776
6777         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6778         # this will effectively collapse the backing image chain consisting of
6779         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6780         # removes itself once all backing images vanish with 'auto-remove=on')
6781         my $jobs = {};
6782         for my $ds (sort keys %$restored_disks) {
6783             my $job_id = "restore-$ds";
6784             mon_cmd($vmid, 'block-stream',
6785                 'job-id' => $job_id,
6786                 device => "$ds",
6787             );
6788             $jobs->{$job_id} = {};
6789         }
6790
6791         mon_cmd($vmid, 'cont');
6792         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6793
6794         print "restore-drive jobs finished successfully, removing all tracking block devices"
6795             ." to disconnect from Proxmox Backup Server\n";
6796
6797         for my $ds (sort keys %$restored_disks) {
6798             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6799         }
6800
6801         close($qmeventd_fd);
6802     };
6803
6804     my $err = $@;
6805
6806     if ($err) {
6807         warn "An error occured during live-restore: $err\n";
6808         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6809         die "live-restore failed\n";
6810     }
6811 }
6812
6813 sub restore_vma_archive {
6814     my ($archive, $vmid, $user, $opts, $comp) = @_;
6815
6816     my $readfrom = $archive;
6817
6818     my $cfg = PVE::Storage::config();
6819     my $commands = [];
6820     my $bwlimit = $opts->{bwlimit};
6821
6822     my $dbg_cmdstring = '';
6823     my $add_pipe = sub {
6824         my ($cmd) = @_;
6825         push @$commands, $cmd;
6826         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6827         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6828         $readfrom = '-';
6829     };
6830
6831     my $input = undef;
6832     if ($archive eq '-') {
6833         $input = '<&STDIN';
6834     } else {
6835         # If we use a backup from a PVE defined storage we also consider that
6836         # storage's rate limit:
6837         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6838         if (defined($volid)) {
6839             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6840             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6841             if ($readlimit) {
6842                 print STDERR "applying read rate limit: $readlimit\n";
6843                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6844                 $add_pipe->($cstream);
6845             }
6846         }
6847     }
6848
6849     if ($comp) {
6850         my $info = PVE::Storage::decompressor_info('vma', $comp);
6851         my $cmd = $info->{decompressor};
6852         push @$cmd, $readfrom;
6853         $add_pipe->($cmd);
6854     }
6855
6856     my $tmpdir = "/var/tmp/vzdumptmp$$";
6857     rmtree $tmpdir;
6858
6859     # disable interrupts (always do cleanups)
6860     local $SIG{INT} =
6861         local $SIG{TERM} =
6862         local $SIG{QUIT} =
6863         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6864
6865     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6866     POSIX::mkfifo($mapfifo, 0600);
6867     my $fifofh;
6868     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6869
6870     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6871
6872     my $oldtimeout;
6873     my $timeout = 5;
6874
6875     my $devinfo = {};
6876
6877     my $rpcenv = PVE::RPCEnvironment::get();
6878
6879     my $conffile = PVE::QemuConfig->config_file($vmid);
6880
6881     # Note: $oldconf is undef if VM does not exist
6882     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6883     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6884     my $new_conf_raw = '';
6885
6886     my %storage_limits;
6887
6888     my $print_devmap = sub {
6889         my $cfgfn = "$tmpdir/qemu-server.conf";
6890
6891         # we can read the config - that is already extracted
6892         my $fh = IO::File->new($cfgfn, "r") ||
6893             die "unable to read qemu-server.conf - $!\n";
6894
6895         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6896         if (-f $fwcfgfn) {
6897             my $pve_firewall_dir = '/etc/pve/firewall';
6898             mkdir $pve_firewall_dir; # make sure the dir exists
6899             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6900         }
6901
6902         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6903
6904         foreach my $info (values %{$virtdev_hash}) {
6905             my $storeid = $info->{storeid};
6906             next if defined($storage_limits{$storeid});
6907
6908             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6909             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6910             $storage_limits{$storeid} = $limit * 1024;
6911         }
6912
6913         foreach my $devname (keys %$devinfo) {
6914             die "found no device mapping information for device '$devname'\n"
6915                 if !$devinfo->{$devname}->{virtdev};
6916         }
6917
6918         # create empty/temp config
6919         if ($oldconf) {
6920             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6921             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6922         }
6923
6924         # allocate volumes
6925         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6926
6927         # print restore information to $fifofh
6928         foreach my $virtdev (sort keys %$virtdev_hash) {
6929             my $d = $virtdev_hash->{$virtdev};
6930             next if $d->{is_cloudinit}; # no need to restore cloudinit
6931
6932             my $storeid = $d->{storeid};
6933             my $volid = $d->{volid};
6934
6935             my $map_opts = '';
6936             if (my $limit = $storage_limits{$storeid}) {
6937                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6938             }
6939
6940             my $write_zeros = 1;
6941             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6942                 $write_zeros = 0;
6943             }
6944
6945             my $path = PVE::Storage::path($cfg, $volid);
6946
6947             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6948
6949             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6950         }
6951
6952         $fh->seek(0, 0) || die "seek failed - $!\n";
6953
6954         my $cookie = { netcount => 0 };
6955         while (defined(my $line = <$fh>)) {
6956             $new_conf_raw .= restore_update_config_line(
6957                 $cookie,
6958                 $map,
6959                 $line,
6960                 $opts->{unique},
6961             );
6962         }
6963
6964         $fh->close();
6965     };
6966
6967     eval {
6968         # enable interrupts
6969         local $SIG{INT} =
6970             local $SIG{TERM} =
6971             local $SIG{QUIT} =
6972             local $SIG{HUP} =
6973             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6974         local $SIG{ALRM} = sub { die "got timeout\n"; };
6975
6976         $oldtimeout = alarm($timeout);
6977
6978         my $parser = sub {
6979             my $line = shift;
6980
6981             print "$line\n";
6982
6983             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6984                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6985                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6986             } elsif ($line =~ m/^CTIME: /) {
6987                 # we correctly received the vma config, so we can disable
6988                 # the timeout now for disk allocation (set to 10 minutes, so
6989                 # that we always timeout if something goes wrong)
6990                 alarm(600);
6991                 &$print_devmap();
6992                 print $fifofh "done\n";
6993                 my $tmp = $oldtimeout || 0;
6994                 $oldtimeout = undef;
6995                 alarm($tmp);
6996                 close($fifofh);
6997                 $fifofh = undef;
6998             }
6999         };
7000
7001         print "restore vma archive: $dbg_cmdstring\n";
7002         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7003     };
7004     my $err = $@;
7005
7006     alarm($oldtimeout) if $oldtimeout;
7007
7008     $restore_deactivate_volumes->($cfg, $devinfo);
7009
7010     close($fifofh) if $fifofh;
7011     unlink $mapfifo;
7012     rmtree $tmpdir;
7013
7014     if ($err) {
7015         $restore_destroy_volumes->($cfg, $devinfo);
7016         die $err;
7017     }
7018
7019     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7020
7021     PVE::Cluster::cfs_update(); # make sure we read new file
7022
7023     eval { rescan($vmid, 1); };
7024     warn $@ if $@;
7025
7026     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7027 }
7028
7029 sub restore_tar_archive {
7030     my ($archive, $vmid, $user, $opts) = @_;
7031
7032     if ($archive ne '-') {
7033         my $firstfile = tar_archive_read_firstfile($archive);
7034         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7035             if $firstfile ne 'qemu-server.conf';
7036     }
7037
7038     my $storecfg = PVE::Storage::config();
7039
7040     # avoid zombie disks when restoring over an existing VM -> cleanup first
7041     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7042     # skiplock=1 because qmrestore has set the 'create' lock itself already
7043     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7044     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7045
7046     my $tocmd = "/usr/lib/qemu-server/qmextract";
7047
7048     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7049     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7050     $tocmd .= ' --prealloc' if $opts->{prealloc};
7051     $tocmd .= ' --info' if $opts->{info};
7052
7053     # tar option "xf" does not autodetect compression when read from STDIN,
7054     # so we pipe to zcat
7055     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7056         PVE::Tools::shellquote("--to-command=$tocmd");
7057
7058     my $tmpdir = "/var/tmp/vzdumptmp$$";
7059     mkpath $tmpdir;
7060
7061     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7062     local $ENV{VZDUMP_VMID} = $vmid;
7063     local $ENV{VZDUMP_USER} = $user;
7064
7065     my $conffile = PVE::QemuConfig->config_file($vmid);
7066     my $new_conf_raw = '';
7067
7068     # disable interrupts (always do cleanups)
7069     local $SIG{INT} =
7070         local $SIG{TERM} =
7071         local $SIG{QUIT} =
7072         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7073
7074     eval {
7075         # enable interrupts
7076         local $SIG{INT} =
7077             local $SIG{TERM} =
7078             local $SIG{QUIT} =
7079             local $SIG{HUP} =
7080             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7081
7082         if ($archive eq '-') {
7083             print "extracting archive from STDIN\n";
7084             run_command($cmd, input => "<&STDIN");
7085         } else {
7086             print "extracting archive '$archive'\n";
7087             run_command($cmd);
7088         }
7089
7090         return if $opts->{info};
7091
7092         # read new mapping
7093         my $map = {};
7094         my $statfile = "$tmpdir/qmrestore.stat";
7095         if (my $fd = IO::File->new($statfile, "r")) {
7096             while (defined (my $line = <$fd>)) {
7097                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7098                     $map->{$1} = $2 if $1;
7099                 } else {
7100                     print STDERR "unable to parse line in statfile - $line\n";
7101                 }
7102             }
7103             $fd->close();
7104         }
7105
7106         my $confsrc = "$tmpdir/qemu-server.conf";
7107
7108         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7109
7110         my $cookie = { netcount => 0 };
7111         while (defined (my $line = <$srcfd>)) {
7112             $new_conf_raw .= restore_update_config_line(
7113                 $cookie,
7114                 $map,
7115                 $line,
7116                 $opts->{unique},
7117             );
7118         }
7119
7120         $srcfd->close();
7121     };
7122     if (my $err = $@) {
7123         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7124         die $err;
7125     }
7126
7127     rmtree $tmpdir;
7128
7129     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7130
7131     PVE::Cluster::cfs_update(); # make sure we read new file
7132
7133     eval { rescan($vmid, 1); };
7134     warn $@ if $@;
7135 };
7136
7137 sub foreach_storage_used_by_vm {
7138     my ($conf, $func) = @_;
7139
7140     my $sidhash = {};
7141
7142     PVE::QemuConfig->foreach_volume($conf, sub {
7143         my ($ds, $drive) = @_;
7144         return if drive_is_cdrom($drive);
7145
7146         my $volid = $drive->{file};
7147
7148         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7149         $sidhash->{$sid} = $sid if $sid;
7150     });
7151
7152     foreach my $sid (sort keys %$sidhash) {
7153         &$func($sid);
7154     }
7155 }
7156
7157 my $qemu_snap_storage = {
7158     rbd => 1,
7159 };
7160 sub do_snapshots_with_qemu {
7161     my ($storecfg, $volid, $deviceid) = @_;
7162
7163     return if $deviceid =~ m/tpmstate0/;
7164
7165     my $storage_name = PVE::Storage::parse_volume_id($volid);
7166     my $scfg = $storecfg->{ids}->{$storage_name};
7167     die "could not find storage '$storage_name'\n" if !defined($scfg);
7168
7169     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7170         return 1;
7171     }
7172
7173     if ($volid =~ m/\.(qcow2|qed)$/){
7174         return 1;
7175     }
7176
7177     return;
7178 }
7179
7180 sub qga_check_running {
7181     my ($vmid, $nowarn) = @_;
7182
7183     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7184     if ($@) {
7185         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7186         return 0;
7187     }
7188     return 1;
7189 }
7190
7191 sub template_create {
7192     my ($vmid, $conf, $disk) = @_;
7193
7194     my $storecfg = PVE::Storage::config();
7195
7196     PVE::QemuConfig->foreach_volume($conf, sub {
7197         my ($ds, $drive) = @_;
7198
7199         return if drive_is_cdrom($drive);
7200         return if $disk && $ds ne $disk;
7201
7202         my $volid = $drive->{file};
7203         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7204
7205         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7206         $drive->{file} = $voliddst;
7207         $conf->{$ds} = print_drive($drive);
7208         PVE::QemuConfig->write_config($vmid, $conf);
7209     });
7210 }
7211
7212 sub convert_iscsi_path {
7213     my ($path) = @_;
7214
7215     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7216         my $portal = $1;
7217         my $target = $2;
7218         my $lun = $3;
7219
7220         my $initiator_name = get_initiator_name();
7221
7222         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7223                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7224     }
7225
7226     die "cannot convert iscsi path '$path', unkown format\n";
7227 }
7228
7229 sub qemu_img_convert {
7230     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7231
7232     my $storecfg = PVE::Storage::config();
7233     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7234     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7235
7236     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7237
7238     my $cachemode;
7239     my $src_path;
7240     my $src_is_iscsi = 0;
7241     my $src_format;
7242
7243     if ($src_storeid) {
7244         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7245         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7246         $src_format = qemu_img_format($src_scfg, $src_volname);
7247         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7248         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7249         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7250     } elsif (-f $src_volid) {
7251         $src_path = $src_volid;
7252         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7253             $src_format = $1;
7254         }
7255     }
7256
7257     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7258
7259     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7260     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7261     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7262     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7263
7264     my $cmd = [];
7265     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7266     push @$cmd, '-l', "snapshot.name=$snapname"
7267         if $snapname && $src_format && $src_format eq "qcow2";
7268     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7269     push @$cmd, '-T', $cachemode if defined($cachemode);
7270
7271     if ($src_is_iscsi) {
7272         push @$cmd, '--image-opts';
7273         $src_path = convert_iscsi_path($src_path);
7274     } elsif ($src_format) {
7275         push @$cmd, '-f', $src_format;
7276     }
7277
7278     if ($dst_is_iscsi) {
7279         push @$cmd, '--target-image-opts';
7280         $dst_path = convert_iscsi_path($dst_path);
7281     } else {
7282         push @$cmd, '-O', $dst_format;
7283     }
7284
7285     push @$cmd, $src_path;
7286
7287     if (!$dst_is_iscsi && $is_zero_initialized) {
7288         push @$cmd, "zeroinit:$dst_path";
7289     } else {
7290         push @$cmd, $dst_path;
7291     }
7292
7293     my $parser = sub {
7294         my $line = shift;
7295         if($line =~ m/\((\S+)\/100\%\)/){
7296             my $percent = $1;
7297             my $transferred = int($size * $percent / 100);
7298             my $total_h = render_bytes($size, 1);
7299             my $transferred_h = render_bytes($transferred, 1);
7300
7301             print "transferred $transferred_h of $total_h ($percent%)\n";
7302         }
7303
7304     };
7305
7306     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7307     my $err = $@;
7308     die "copy failed: $err" if $err;
7309 }
7310
7311 sub qemu_img_format {
7312     my ($scfg, $volname) = @_;
7313
7314     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7315         return $1;
7316     } else {
7317         return "raw";
7318     }
7319 }
7320
7321 sub qemu_drive_mirror {
7322     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7323
7324     $jobs = {} if !$jobs;
7325
7326     my $qemu_target;
7327     my $format;
7328     $jobs->{"drive-$drive"} = {};
7329
7330     if ($dst_volid =~ /^nbd:/) {
7331         $qemu_target = $dst_volid;
7332         $format = "nbd";
7333     } else {
7334         my $storecfg = PVE::Storage::config();
7335         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7336
7337         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7338
7339         $format = qemu_img_format($dst_scfg, $dst_volname);
7340
7341         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7342
7343         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7344     }
7345
7346     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7347     $opts->{format} = $format if $format;
7348
7349     if (defined($src_bitmap)) {
7350         $opts->{sync} = 'incremental';
7351         $opts->{bitmap} = $src_bitmap;
7352         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7353     }
7354
7355     if (defined($bwlimit)) {
7356         $opts->{speed} = $bwlimit * 1024;
7357         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7358     } else {
7359         print "drive mirror is starting for drive-$drive\n";
7360     }
7361
7362     # if a job already runs for this device we get an error, catch it for cleanup
7363     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7364     if (my $err = $@) {
7365         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7366         warn "$@\n" if $@;
7367         die "mirroring error: $err\n";
7368     }
7369
7370     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7371 }
7372
7373 # $completion can be either
7374 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7375 # 'cancel': wait until all jobs are ready, block-job-cancel them
7376 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7377 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7378 sub qemu_drive_mirror_monitor {
7379     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7380
7381     $completion //= 'complete';
7382     $op //= "mirror";
7383
7384     eval {
7385         my $err_complete = 0;
7386
7387         my $starttime = time ();
7388         while (1) {
7389             die "block job ('$op') timed out\n" if $err_complete > 300;
7390
7391             my $stats = mon_cmd($vmid, "query-block-jobs");
7392             my $ctime = time();
7393
7394             my $running_jobs = {};
7395             for my $stat (@$stats) {
7396                 next if $stat->{type} ne $op;
7397                 $running_jobs->{$stat->{device}} = $stat;
7398             }
7399
7400             my $readycounter = 0;
7401
7402             for my $job_id (sort keys %$jobs) {
7403                 my $job = $running_jobs->{$job_id};
7404
7405                 my $vanished = !defined($job);
7406                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7407                 if($complete || ($vanished && $completion eq 'auto')) {
7408                     print "$job_id: $op-job finished\n";
7409                     delete $jobs->{$job_id};
7410                     next;
7411                 }
7412
7413                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7414
7415                 my $busy = $job->{busy};
7416                 my $ready = $job->{ready};
7417                 if (my $total = $job->{len}) {
7418                     my $transferred = $job->{offset} || 0;
7419                     my $remaining = $total - $transferred;
7420                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7421
7422                     my $duration = $ctime - $starttime;
7423                     my $total_h = render_bytes($total, 1);
7424                     my $transferred_h = render_bytes($transferred, 1);
7425
7426                     my $status = sprintf(
7427                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7428                         render_duration($duration),
7429                     );
7430
7431                     if ($ready) {
7432                         if ($busy) {
7433                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7434                         } else {
7435                             $status .= ", ready";
7436                         }
7437                     }
7438                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7439                     $jobs->{$job_id}->{ready} = $ready;
7440                 }
7441
7442                 $readycounter++ if $job->{ready};
7443             }
7444
7445             last if scalar(keys %$jobs) == 0;
7446
7447             if ($readycounter == scalar(keys %$jobs)) {
7448                 print "all '$op' jobs are ready\n";
7449
7450                 # do the complete later (or has already been done)
7451                 last if $completion eq 'skip' || $completion eq 'auto';
7452
7453                 if ($vmiddst && $vmiddst != $vmid) {
7454                     my $agent_running = $qga && qga_check_running($vmid);
7455                     if ($agent_running) {
7456                         print "freeze filesystem\n";
7457                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7458                     } else {
7459                         print "suspend vm\n";
7460                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7461                     }
7462
7463                     # if we clone a disk for a new target vm, we don't switch the disk
7464                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7465
7466                     if ($agent_running) {
7467                         print "unfreeze filesystem\n";
7468                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7469                     } else {
7470                         print "resume vm\n";
7471                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7472                     }
7473
7474                     last;
7475                 } else {
7476
7477                     for my $job_id (sort keys %$jobs) {
7478                         # try to switch the disk if source and destination are on the same guest
7479                         print "$job_id: Completing block job_id...\n";
7480
7481                         my $op;
7482                         if ($completion eq 'complete') {
7483                             $op = 'block-job-complete';
7484                         } elsif ($completion eq 'cancel') {
7485                             $op = 'block-job-cancel';
7486                         } else {
7487                             die "invalid completion value: $completion\n";
7488                         }
7489                         eval { mon_cmd($vmid, $op, device => $job_id) };
7490                         if ($@ =~ m/cannot be completed/) {
7491                             print "$job_id: block job cannot be completed, trying again.\n";
7492                             $err_complete++;
7493                         }else {
7494                             print "$job_id: Completed successfully.\n";
7495                             $jobs->{$job_id}->{complete} = 1;
7496                         }
7497                     }
7498                 }
7499             }
7500             sleep 1;
7501         }
7502     };
7503     my $err = $@;
7504
7505     if ($err) {
7506         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7507         die "block job ($op) error: $err";
7508     }
7509 }
7510
7511 sub qemu_blockjobs_cancel {
7512     my ($vmid, $jobs) = @_;
7513
7514     foreach my $job (keys %$jobs) {
7515         print "$job: Cancelling block job\n";
7516         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7517         $jobs->{$job}->{cancel} = 1;
7518     }
7519
7520     while (1) {
7521         my $stats = mon_cmd($vmid, "query-block-jobs");
7522
7523         my $running_jobs = {};
7524         foreach my $stat (@$stats) {
7525             $running_jobs->{$stat->{device}} = $stat;
7526         }
7527
7528         foreach my $job (keys %$jobs) {
7529
7530             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7531                 print "$job: Done.\n";
7532                 delete $jobs->{$job};
7533             }
7534         }
7535
7536         last if scalar(keys %$jobs) == 0;
7537
7538         sleep 1;
7539     }
7540 }
7541
7542 sub clone_disk {
7543     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7544         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7545
7546     my $newvolid;
7547
7548     if (!$full) {
7549         print "create linked clone of drive $drivename ($drive->{file})\n";
7550         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7551         push @$newvollist, $newvolid;
7552     } else {
7553
7554         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7555         $storeid = $storage if $storage;
7556
7557         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7558
7559         print "create full clone of drive $drivename ($drive->{file})\n";
7560         my $name = undef;
7561         my $size = undef;
7562         if (drive_is_cloudinit($drive)) {
7563             $name = "vm-$newvmid-cloudinit";
7564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7565             if ($scfg->{path}) {
7566                 $name .= ".$dst_format";
7567             }
7568             $snapname = undef;
7569             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7570         } elsif ($drivename eq 'efidisk0') {
7571             $size = get_efivars_size($conf);
7572         } elsif ($drivename eq 'tpmstate0') {
7573             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7574         } else {
7575             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7576         }
7577         $newvolid = PVE::Storage::vdisk_alloc(
7578             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7579         );
7580         push @$newvollist, $newvolid;
7581
7582         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7583
7584         if (drive_is_cloudinit($drive)) {
7585             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7586             # if this is the case, we have to complete any block-jobs still there from
7587             # previous drive-mirrors
7588             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7589                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7590             }
7591             goto no_data_clone;
7592         }
7593
7594         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7595         if (!$running || $snapname) {
7596             # TODO: handle bwlimits
7597             if ($drivename eq 'efidisk0') {
7598                 # the relevant data on the efidisk may be smaller than the source
7599                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7600                 # that is given by the OVMF_VARS.fd
7601                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7602                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7603
7604                 # better for Ceph if block size is not too small, see bug #3324
7605                 my $bs = 1024*1024;
7606
7607                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7608                     "if=$src_path", "of=$dst_path"]);
7609             } else {
7610                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7611             }
7612         } else {
7613
7614             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7615
7616             my $kvmver = get_running_qemu_version ($vmid);
7617             if (!min_version($kvmver, 2, 7)) {
7618                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7619                     if $drive->{iothread};
7620             }
7621
7622             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7623                 $completion, $qga, $bwlimit);
7624         }
7625     }
7626
7627 no_data_clone:
7628     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7629
7630     my $disk = $drive;
7631     $disk->{format} = undef;
7632     $disk->{file} = $newvolid;
7633     $disk->{size} = $size if defined($size);
7634
7635     return $disk;
7636 }
7637
7638 sub get_running_qemu_version {
7639     my ($vmid) = @_;
7640     my $res = mon_cmd($vmid, "query-version");
7641     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7642 }
7643
7644 sub qemu_use_old_bios_files {
7645     my ($machine_type) = @_;
7646
7647     return if !$machine_type;
7648
7649     my $use_old_bios_files = undef;
7650
7651     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7652         $machine_type = $1;
7653         $use_old_bios_files = 1;
7654     } else {
7655         my $version = extract_version($machine_type, kvm_user_version());
7656         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7657         # load new efi bios files on migration. So this hack is required to allow
7658         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7659         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7660         $use_old_bios_files = !min_version($version, 2, 4);
7661     }
7662
7663     return ($use_old_bios_files, $machine_type);
7664 }
7665
7666 sub get_efivars_size {
7667     my ($conf) = @_;
7668     my $arch = get_vm_arch($conf);
7669     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7670     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7671     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7672     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7673     return -s $ovmf_vars;
7674 }
7675
7676 sub update_efidisk_size {
7677     my ($conf) = @_;
7678
7679     return if !defined($conf->{efidisk0});
7680
7681     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7682     $disk->{size} = get_efivars_size($conf);
7683     $conf->{efidisk0} = print_drive($disk);
7684
7685     return;
7686 }
7687
7688 sub update_tpmstate_size {
7689     my ($conf) = @_;
7690
7691     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7692     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7693     $conf->{tpmstate0} = print_drive($disk);
7694 }
7695
7696 sub create_efidisk($$$$$$$) {
7697     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7698
7699     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7700     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7701
7702     my $vars_size_b = -s $ovmf_vars;
7703     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7704     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7705     PVE::Storage::activate_volumes($storecfg, [$volid]);
7706
7707     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7708     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7709
7710     return ($volid, $size/1024);
7711 }
7712
7713 sub vm_iothreads_list {
7714     my ($vmid) = @_;
7715
7716     my $res = mon_cmd($vmid, 'query-iothreads');
7717
7718     my $iothreads = {};
7719     foreach my $iothread (@$res) {
7720         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7721     }
7722
7723     return $iothreads;
7724 }
7725
7726 sub scsihw_infos {
7727     my ($conf, $drive) = @_;
7728
7729     my $maxdev = 0;
7730
7731     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7732         $maxdev = 7;
7733     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7734         $maxdev = 1;
7735     } else {
7736         $maxdev = 256;
7737     }
7738
7739     my $controller = int($drive->{index} / $maxdev);
7740     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7741         ? "virtioscsi"
7742         : "scsihw";
7743
7744     return ($maxdev, $controller, $controller_prefix);
7745 }
7746
7747 sub windows_version {
7748     my ($ostype) = @_;
7749
7750     return 0 if !$ostype;
7751
7752     my $winversion = 0;
7753
7754     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7755         $winversion = 5;
7756     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7757         $winversion = 6;
7758     } elsif ($ostype =~ m/^win(\d+)$/) {
7759         $winversion = $1;
7760     }
7761
7762     return $winversion;
7763 }
7764
7765 sub resolve_dst_disk_format {
7766         my ($storecfg, $storeid, $src_volname, $format) = @_;
7767         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7768
7769         if (!$format) {
7770             # if no target format is specified, use the source disk format as hint
7771             if ($src_volname) {
7772                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7773                 $format = qemu_img_format($scfg, $src_volname);
7774             } else {
7775                 return $defFormat;
7776             }
7777         }
7778
7779         # test if requested format is supported - else use default
7780         my $supported = grep { $_ eq $format } @$validFormats;
7781         $format = $defFormat if !$supported;
7782         return $format;
7783 }
7784
7785 # NOTE: if this logic changes, please update docs & possibly gui logic
7786 sub find_vmstate_storage {
7787     my ($conf, $storecfg) = @_;
7788
7789     # first, return storage from conf if set
7790     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7791
7792     my ($target, $shared, $local);
7793
7794     foreach_storage_used_by_vm($conf, sub {
7795         my ($sid) = @_;
7796         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7797         my $dst = $scfg->{shared} ? \$shared : \$local;
7798         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7799     });
7800
7801     # second, use shared storage where VM has at least one disk
7802     # third, use local storage where VM has at least one disk
7803     # fall back to local storage
7804     $target = $shared // $local // 'local';
7805
7806     return $target;
7807 }
7808
7809 sub generate_uuid {
7810     my ($uuid, $uuid_str);
7811     UUID::generate($uuid);
7812     UUID::unparse($uuid, $uuid_str);
7813     return $uuid_str;
7814 }
7815
7816 sub generate_smbios1_uuid {
7817     return "uuid=".generate_uuid();
7818 }
7819
7820 sub nbd_stop {
7821     my ($vmid) = @_;
7822
7823     mon_cmd($vmid, 'nbd-server-stop');
7824 }
7825
7826 sub create_reboot_request {
7827     my ($vmid) = @_;
7828     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7829         or die "failed to create reboot trigger file: $!\n";
7830     close($fh);
7831 }
7832
7833 sub clear_reboot_request {
7834     my ($vmid) = @_;
7835     my $path = "/run/qemu-server/$vmid.reboot";
7836     my $res = 0;
7837
7838     $res = unlink($path);
7839     die "could not remove reboot request for $vmid: $!"
7840         if !$res && $! != POSIX::ENOENT;
7841
7842     return $res;
7843 }
7844
7845 sub bootorder_from_legacy {
7846     my ($conf, $bootcfg) = @_;
7847
7848     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7849     my $bootindex_hash = {};
7850     my $i = 1;
7851     foreach my $o (split(//, $boot)) {
7852         $bootindex_hash->{$o} = $i*100;
7853         $i++;
7854     }
7855
7856     my $bootorder = {};
7857
7858     PVE::QemuConfig->foreach_volume($conf, sub {
7859         my ($ds, $drive) = @_;
7860
7861         if (drive_is_cdrom ($drive, 1)) {
7862             if ($bootindex_hash->{d}) {
7863                 $bootorder->{$ds} = $bootindex_hash->{d};
7864                 $bootindex_hash->{d} += 1;
7865             }
7866         } elsif ($bootindex_hash->{c}) {
7867             $bootorder->{$ds} = $bootindex_hash->{c}
7868                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7869             $bootindex_hash->{c} += 1;
7870         }
7871     });
7872
7873     if ($bootindex_hash->{n}) {
7874         for (my $i = 0; $i < $MAX_NETS; $i++) {
7875             my $netname = "net$i";
7876             next if !$conf->{$netname};
7877             $bootorder->{$netname} = $bootindex_hash->{n};
7878             $bootindex_hash->{n} += 1;
7879         }
7880     }
7881
7882     return $bootorder;
7883 }
7884
7885 # Generate default device list for 'boot: order=' property. Matches legacy
7886 # default boot order, but with explicit device names. This is important, since
7887 # the fallback for when neither 'order' nor the old format is specified relies
7888 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7889 sub get_default_bootdevices {
7890     my ($conf) = @_;
7891
7892     my @ret = ();
7893
7894     # harddisk
7895     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7896     push @ret, $first if $first;
7897
7898     # cdrom
7899     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7900     push @ret, $first if $first;
7901
7902     # network
7903     for (my $i = 0; $i < $MAX_NETS; $i++) {
7904         my $netname = "net$i";
7905         next if !$conf->{$netname};
7906         push @ret, $netname;
7907         last;
7908     }
7909
7910     return \@ret;
7911 }
7912
7913 sub device_bootorder {
7914     my ($conf) = @_;
7915
7916     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7917
7918     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7919
7920     my $bootorder = {};
7921     if (!defined($boot) || $boot->{legacy}) {
7922         $bootorder = bootorder_from_legacy($conf, $boot);
7923     } elsif ($boot->{order}) {
7924         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7925         for my $dev (PVE::Tools::split_list($boot->{order})) {
7926             $bootorder->{$dev} = $i++;
7927         }
7928     }
7929
7930     return $bootorder;
7931 }
7932
7933 sub register_qmeventd_handle {
7934     my ($vmid) = @_;
7935
7936     my $fh;
7937     my $peer = "/var/run/qmeventd.sock";
7938     my $count = 0;
7939
7940     for (;;) {
7941         $count++;
7942         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7943         last if $fh;
7944         if ($! != EINTR && $! != EAGAIN) {
7945             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7946         }
7947         if ($count > 4) {
7948             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7949               . "after $count retries\n";
7950         }
7951         usleep(25000);
7952     }
7953
7954     # send handshake to mark VM as backing up
7955     print $fh to_json({vzdump => {vmid => "$vmid"}});
7956
7957     # return handle to be closed later when inhibit is no longer required
7958     return $fh;
7959 }
7960
7961 # bash completion helper
7962
7963 sub complete_backup_archives {
7964     my ($cmdname, $pname, $cvalue) = @_;
7965
7966     my $cfg = PVE::Storage::config();
7967
7968     my $storeid;
7969
7970     if ($cvalue =~ m/^([^:]+):/) {
7971         $storeid = $1;
7972     }
7973
7974     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7975
7976     my $res = [];
7977     foreach my $id (keys %$data) {
7978         foreach my $item (@{$data->{$id}}) {
7979             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7980             push @$res, $item->{volid} if defined($item->{volid});
7981         }
7982     }
7983
7984     return $res;
7985 }
7986
7987 my $complete_vmid_full = sub {
7988     my ($running) = @_;
7989
7990     my $idlist = vmstatus();
7991
7992     my $res = [];
7993
7994     foreach my $id (keys %$idlist) {
7995         my $d = $idlist->{$id};
7996         if (defined($running)) {
7997             next if $d->{template};
7998             next if $running && $d->{status} ne 'running';
7999             next if !$running && $d->{status} eq 'running';
8000         }
8001         push @$res, $id;
8002
8003     }
8004     return $res;
8005 };
8006
8007 sub complete_vmid {
8008     return &$complete_vmid_full();
8009 }
8010
8011 sub complete_vmid_stopped {
8012     return &$complete_vmid_full(0);
8013 }
8014
8015 sub complete_vmid_running {
8016     return &$complete_vmid_full(1);
8017 }
8018
8019 sub complete_storage {
8020
8021     my $cfg = PVE::Storage::config();
8022     my $ids = $cfg->{ids};
8023
8024     my $res = [];
8025     foreach my $sid (keys %$ids) {
8026         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8027         next if !$ids->{$sid}->{content}->{images};
8028         push @$res, $sid;
8029     }
8030
8031     return $res;
8032 }
8033
8034 sub complete_migration_storage {
8035     my ($cmd, $param, $current_value, $all_args) = @_;
8036
8037     my $targetnode = @$all_args[1];
8038
8039     my $cfg = PVE::Storage::config();
8040     my $ids = $cfg->{ids};
8041
8042     my $res = [];
8043     foreach my $sid (keys %$ids) {
8044         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8045         next if !$ids->{$sid}->{content}->{images};
8046         push @$res, $sid;
8047     }
8048
8049     return $res;
8050 }
8051
8052 sub vm_is_paused {
8053     my ($vmid) = @_;
8054     my $qmpstatus = eval {
8055         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8056         mon_cmd($vmid, "query-status");
8057     };
8058     warn "$@\n" if $@;
8059     return $qmpstatus && $qmpstatus->{status} eq "paused";
8060 }
8061
8062 sub check_volume_storage_type {
8063     my ($storecfg, $vol) = @_;
8064
8065     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8066     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8067     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8068
8069     die "storage '$storeid' does not support content-type '$vtype'\n"
8070         if !$scfg->{content}->{$vtype};
8071
8072     return 1;
8073 }
8074
8075 1;