PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122
 123 sub map_storage {
 124     my ($map, $source) = @_;
 125
 126     return $source if !defined($map);
 127
 128     return $map->{entries}->{$source}
 129         if $map->{entries} && defined($map->{entries}->{$source});
 130
 131     return $map->{default} if $map->{default};
 132
 133     # identity (fallback)
 134     return $source;
 135 }
 136
 137 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 138     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 139     type => 'string',
 140     format => 'storage-pair-list',
 141     optional => 1,
 142 });
 143
 144 #no warnings 'redefine';
 145
 146 my $nodename_cache;
 147 sub nodename {
 148     $nodename_cache //= PVE::INotify::nodename();
 149     return $nodename_cache;
 150 }
 151
 152 my $watchdog_fmt = {
 153     model => {
 154         default_key => 1,
 155         type => 'string',
 156         enum => [qw(i6300esb ib700)],
 157         description => "Watchdog type to emulate.",
 158         default => 'i6300esb',
 159         optional => 1,
 160     },
 161     action => {
 162         type => 'string',
 163         enum => [qw(reset shutdown poweroff pause debug none)],
 164         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 165         optional => 1,
 166     },
 167 };
 168 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 169
 170 my $agent_fmt = {
 171     enabled => {
 172         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 173         type => 'boolean',
 174         default => 0,
 175         default_key => 1,
 176     },
 177     fstrim_cloned_disks => {
 178         description => "Run fstrim after moving a disk or migrating the VM.",
 179         type => 'boolean',
 180         optional => 1,
 181         default => 0
 182     },
 183     type => {
 184         description => "Select the agent type",
 185         type => 'string',
 186         default => 'virtio',
 187         optional => 1,
 188         enum => [qw(virtio isa)],
 189     },
 190 };
 191
 192 my $vga_fmt = {
 193     type => {
 194         description => "Select the VGA type.",
 195         type => 'string',
 196         default => 'std',
 197         optional => 1,
 198         default_key => 1,
 199         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 200     },
 201     memory => {
 202         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 203         type => 'integer',
 204         optional => 1,
 205         minimum => 4,
 206         maximum => 512,
 207     },
 208 };
 209
 210 my $ivshmem_fmt = {
 211     size => {
 212         type => 'integer',
 213         minimum => 1,
 214         description => "The size of the file in MB.",
 215     },
 216     name => {
 217         type => 'string',
 218         pattern => '[a-zA-Z0-9\-]+',
 219         optional => 1,
 220         format_description => 'string',
 221         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 222     },
 223 };
 224
 225 my $audio_fmt = {
 226     device => {
 227         type => 'string',
 228         enum => [qw(ich9-intel-hda intel-hda AC97)],
 229         description =>  "Configure an audio device."
 230     },
 231     driver =>  {
 232         type => 'string',
 233         enum => ['spice', 'none'],
 234         default => 'spice',
 235         optional => 1,
 236         description => "Driver backend for the audio device."
 237     },
 238 };
 239
 240 my $spice_enhancements_fmt = {
 241     foldersharing => {
 242         type => 'boolean',
 243         optional => 1,
 244         default => '0',
 245         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 246     },
 247     videostreaming =>  {
 248         type => 'string',
 249         enum => ['off', 'all', 'filter'],
 250         default => 'off',
 251         optional => 1,
 252         description => "Enable video streaming. Uses compression for detected video streams."
 253     },
 254 };
 255
 256 my $rng_fmt = {
 257     source => {
 258         type => 'string',
 259         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 260         default_key => 1,
 261         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 262             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 263             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 264             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 265             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 266             ." a hardware RNG from the host.",
 267     },
 268     max_bytes => {
 269         type => 'integer',
 270         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 271             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 272             ." `0` to disable limiting (potentially dangerous!).",
 273         optional => 1,
 274
 275         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 276         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 277         # reading from /dev/urandom
 278         default => 1024,
 279     },
 280     period => {
 281         type => 'integer',
 282         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 283             ." the guest to retrieve another 'max_bytes' of entropy.",
 284         optional => 1,
 285         default => 1000,
 286     },
 287 };
 288
 289 my $meta_info_fmt = {
 290     'ctime' => {
 291         type => 'integer',
 292         description => "The guest creation timestamp as UNIX epoch time",
 293         minimum => 0,
 294         optional => 1,
 295     },
 296     'creation-qemu' => {
 297         type => 'string',
 298         description => "The QEMU (machine) version from the time this VM was created.",
 299         pattern => '\d+(\.\d+)+',
 300         optional => 1,
 301     },
 302 };
 303
 304 my $confdesc = {
 305     onboot => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Specifies whether a VM will be started during system bootup.",
 309         default => 0,
 310     },
 311     autostart => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Automatic restart after crash (currently ignored).",
 315         default => 0,
 316     },
 317     hotplug => {
 318         optional => 1,
 319         type => 'string', format => 'pve-hotplug-features',
 320         description => "Selectively enable hotplug features. This is a comma separated list of"
 321             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 322             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 323         default => 'network,disk,usb',
 324     },
 325     reboot => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 329         default => 1,
 330     },
 331     lock => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Lock/unlock the VM.",
 335         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 336     },
 337     cpulimit => {
 338         optional => 1,
 339         type => 'number',
 340         description => "Limit of CPU usage.",
 341         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 342             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 343         minimum => 0,
 344         maximum => 128,
 345         default => 0,
 346     },
 347     cpuunits => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 351         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 352             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 353             ." weights of all the other running VMs.",
 354         minimum => 2,
 355         maximum => 262144,
 356         default => 'cgroup v1: 1024, cgroup v2: 100',
 357     },
 358     memory => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 362             ." you use the balloon device.",
 363         minimum => 16,
 364         default => 512,
 365     },
 366     balloon => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 370         minimum => 0,
 371     },
 372     shares => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 376             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 377             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 378         minimum => 0,
 379         maximum => 50000,
 380         default => 1000,
 381     },
 382     keyboard => {
 383         optional => 1,
 384         type => 'string',
 385         description => "Keyboard layout for VNC server. The default is read from the"
 386             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 387         enum => PVE::Tools::kvmkeymaplist(),
 388         default => undef,
 389     },
 390     name => {
 391         optional => 1,
 392         type => 'string', format => 'dns-name',
 393         description => "Set a name for the VM. Only used on the configuration web interface.",
 394     },
 395     scsihw => {
 396         optional => 1,
 397         type => 'string',
 398         description => "SCSI controller model",
 399         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 400         default => 'lsi',
 401     },
 402     description => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Description for the VM. Shown in the web-interface VM's summary."
 406             ." This is saved as comment inside the configuration file.",
 407         maxLength => 1024 * 8,
 408     },
 409     ostype => {
 410         optional => 1,
 411         type => 'string',
 412         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 413         description => "Specify guest operating system.",
 414         verbose_description => <<EODESC,
 415 Specify guest operating system. This is used to enable special
 416 optimization/features for specific operating systems:
 417
 418 [horizontal]
 419 other;; unspecified OS
 420 wxp;; Microsoft Windows XP
 421 w2k;; Microsoft Windows 2000
 422 w2k3;; Microsoft Windows 2003
 423 w2k8;; Microsoft Windows 2008
 424 wvista;; Microsoft Windows Vista
 425 win7;; Microsoft Windows 7
 426 win8;; Microsoft Windows 8/2012/2012r2
 427 win10;; Microsoft Windows 10/2016/2019
 428 win11;; Microsoft Windows 11/2022
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6 - 5.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string', format => 'pve-qm-boot',
 437         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 438             ." key or 'legacy=' is deprecated.",
 439     },
 440     bootdisk => {
 441         optional => 1,
 442         type => 'string', format => 'pve-qm-bootdisk',
 443         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 444         pattern => '(ide|sata|scsi|virtio)\d+',
 445     },
 446     smp => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "The number of CPUs. Please use option -sockets instead.",
 450         minimum => 1,
 451         default => 1,
 452     },
 453     sockets => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPU sockets.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     cores => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of cores per socket.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     numa => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable NUMA.",
 471         default => 0,
 472     },
 473     hugepages => {
 474         optional => 1,
 475         type => 'string',
 476         description => "Enable/disable hugepages memory.",
 477         enum => [qw(any 2 1024)],
 478     },
 479     keephugepages => {
 480         optional => 1,
 481         type => 'boolean',
 482         default => 0,
 483         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 484             ." after VM shutdown and can be used for subsequent starts.",
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 521             ." the `ostype` indicates a Microsoft Windows OS.",
 522     },
 523     freeze => {
 524         optional => 1,
 525         type => 'boolean',
 526         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 527     },
 528     vga => {
 529         optional => 1,
 530         type => 'string', format => $vga_fmt,
 531         description => "Configure the VGA hardware.",
 532         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 533             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 534             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 535             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 536             ." display server. For win* OS you can select how many independent displays you want,"
 537             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 538             ." using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 545             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 546             ." else the watchdog will reset the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are:"
 553             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 582             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 583             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 584             ." may consider disabling this to save some context switches. This is turned off by"
 585             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => 'pve-vm-cpu-conf',
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally"
 627             ." for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 635             ." for snapshots.",
 636     }),
 637     runningcpu => {
 638         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 639             ." internally for snapshots.",
 640         optional => 1,
 641         type => 'string',
 642         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 643         format_description => 'QEMU -cpu parameter'
 644     },
 645     machine => get_standard_option('pve-qemu-machine'),
 646     arch => {
 647         description => "Virtual processor architecture. Defaults to the host.",
 648         optional => 1,
 649         type => 'string',
 650         enum => [qw(x86_64 aarch64)],
 651     },
 652     smbios1 => {
 653         description => "Specify SMBIOS type 1 fields.",
 654         type => 'string', format => 'pve-qm-smbios1',
 655         maxLength => 512,
 656         optional => 1,
 657     },
 658     protection => {
 659         optional => 1,
 660         type => 'boolean',
 661         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 662             ." remove disk operations.",
 663         default => 0,
 664     },
 665     bios => {
 666         optional => 1,
 667         type => 'string',
 668         enum => [ qw(seabios ovmf) ],
 669         description => "Select BIOS implementation.",
 670         default => 'seabios',
 671     },
 672     vmgenid => {
 673         type => 'string',
 674         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 675         format_description => 'UUID',
 676         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 677             ." to disable explicitly.",
 678         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 679             ." value identifier to the guest OS. This allows to notify the guest operating system"
 680             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 681             ." execution or creation from a template). The guest operating system notices the"
 682             ." change, and is then able to react as appropriate by marking its copies of"
 683             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 684             ."Note that auto-creation only works when done through API/CLI create or update methods"
 685             .", but not when manually editing the config file.",
 686         default => "1 (autogenerated)",
 687         optional => 1,
 688     },
 689     hookscript => {
 690         type => 'string',
 691         format => 'pve-volume-id',
 692         optional => 1,
 693         description => "Script that will be executed during various steps in the vms lifetime.",
 694     },
 695     ivshmem => {
 696         type => 'string',
 697         format => $ivshmem_fmt,
 698         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 699             ." the host.",
 700         optional => 1,
 701     },
 702     audio0 => {
 703         type => 'string',
 704         format => $audio_fmt,
 705         description => "Configure a audio device, useful in combination with QXL/Spice.",
 706         optional => 1
 707     },
 708     spice_enhancements => {
 709         type => 'string',
 710         format => $spice_enhancements_fmt,
 711         description => "Configure additional enhancements for SPICE.",
 712         optional => 1
 713     },
 714     tags => {
 715         type => 'string', format => 'pve-tag-list',
 716         description => 'Tags of the VM. This is only meta information.',
 717         optional => 1,
 718     },
 719     rng0 => {
 720         type => 'string',
 721         format => $rng_fmt,
 722         description => "Configure a VirtIO-based Random Number Generator.",
 723         optional => 1,
 724     },
 725     meta => {
 726         type => 'string',
 727         format => $meta_info_fmt,
 728         description => "Some (read-only) meta-information about this guest.",
 729         optional => 1,
 730     },
 731 };
 732
 733 my $cicustom_fmt = {
 734     meta => {
 735         type => 'string',
 736         optional => 1,
 737         description => 'Specify a custom file containing all meta data passed to the VM via"
 738             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 739         format => 'pve-volume-id',
 740         format_description => 'volume',
 741     },
 742     network => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all network data passed to the VM via'
 746             .' cloud-init.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     user => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'Specify a custom file containing all user data passed to the VM via'
 754             .' cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758     vendor => {
 759     type => 'string',
 760     optional => 1,
 761     description => 'Specify a custom file containing all vendor data passed to the VM via'
 762      .' cloud-init.',
 763     format => 'pve-volume-id',
 764     format_description => 'volume',
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 768
 769 my $confdesc_cloudinit = {
 770     citype => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'Specifies the cloud-init configuration format. The default depends on the'
 774             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 775             .' and `configdrive2` for windows.',
 776         enum => ['configdrive2', 'nocloud', 'opennebula'],
 777     },
 778     ciuser => {
 779         optional => 1,
 780         type => 'string',
 781         description => "cloud-init: User name to change ssh keys and password for instead of the"
 782             ." image's configured default user.",
 783     },
 784     cipassword => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Password to assign the user. Using this is generally not'
 788             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 789             .' support hashed passwords.',
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => "cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.",
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.",
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_USB_DEVICES = 5;
 842 my $MAX_NETS = 32;
 843 my $MAX_SERIAL_PORTS = 4;
 844 my $MAX_PARALLEL_PORTS = 3;
 845 my $MAX_NUMA = 8;
 846
 847 my $numa_fmt = {
 848     cpus => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "CPUs accessing this NUMA node.",
 852         format_description => "id[-id];...",
 853     },
 854     memory => {
 855         type => "number",
 856         description => "Amount of memory this NUMA node provides.",
 857         optional => 1,
 858     },
 859     hostnodes => {
 860         type => "string",
 861         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 862         description => "Host NUMA nodes to use.",
 863         format_description => "id[-id];...",
 864         optional => 1,
 865     },
 866     policy => {
 867         type => 'string',
 868         enum => [qw(preferred bind interleave)],
 869         description => "NUMA allocation policy.",
 870         optional => 1,
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 874 my $numadesc = {
 875     optional => 1,
 876     type => 'string', format => $numa_fmt,
 877     description => "NUMA topology.",
 878 };
 879 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 880
 881 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 882     $confdesc->{"numa$i"} = $numadesc;
 883 }
 884
 885 my $nic_model_list = [
 886     'e1000',
 887     'e1000-82540em',
 888     'e1000-82544gc',
 889     'e1000-82545em',
 890     'e1000e',
 891     'i82551',
 892     'i82557b',
 893     'i82559er',
 894     'ne2k_isa',
 895     'ne2k_pci',
 896     'pcnet',
 897     'rtl8139',
 898     'virtio',
 899     'vmxnet3',
 900 ];
 901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 902
 903 my $net_fmt_bridge_descr = <<__EOD__;
 904 Bridge to attach the network device to. The Proxmox VE standard bridge
 905 is called 'vmbr0'.
 906
 907 If you do not specify a bridge, we create a kvm user (NATed) network
 908 device, which provides DHCP and DNS services. The following addresses
 909 are used:
 910
 911  10.0.2.2   Gateway
 912  10.0.2.3   DNS Server
 913  10.0.2.4   SMB Server
 914
 915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 916 __EOD__
 917
 918 my $net_fmt = {
 919     macaddr  => get_standard_option('mac-addr', {
 920         description => "MAC address. That address must be unique withing your network. This is"
 921             ." automatically generated if not specified.",
 922     }),
 923     model => {
 924         type => 'string',
 925         description => "Network Card Model. The 'virtio' model provides the best performance with"
 926             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 927             ." best to use 'e1000'.",
 928         enum => $nic_model_list,
 929         default_key => 1,
 930     },
 931     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 932     bridge => {
 933         type => 'string',
 934         description => $net_fmt_bridge_descr,
 935         format_description => 'bridge',
 936         pattern => '[-_.\w\d]+',
 937         optional => 1,
 938     },
 939     queues => {
 940         type => 'integer',
 941         minimum => 0, maximum => 16,
 942         description => 'Number of packet queues to be used on the device.',
 943         optional => 1,
 944     },
 945     rate => {
 946         type => 'number',
 947         minimum => 0,
 948         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 949         optional => 1,
 950     },
 951     tag => {
 952         type => 'integer',
 953         minimum => 1, maximum => 4094,
 954         description => 'VLAN tag to apply to packets on this interface.',
 955         optional => 1,
 956     },
 957     trunks => {
 958         type => 'string',
 959         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 960         description => 'VLAN trunks to pass through this interface.',
 961         format_description => 'vlanid[;vlanid...]',
 962         optional => 1,
 963     },
 964     firewall => {
 965         type => 'boolean',
 966         description => 'Whether this interface should be protected by the firewall.',
 967         optional => 1,
 968     },
 969     link_down => {
 970         type => 'boolean',
 971         description => 'Whether this interface should be disconnected (like pulling the plug).',
 972         optional => 1,
 973     },
 974     mtu => {
 975         type => 'integer',
 976         minimum => 1, maximum => 65520,
 977         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 978         optional => 1,
 979     },
 980 };
 981
 982 my $netdesc = {
 983     optional => 1,
 984     type => 'string', format => $net_fmt,
 985     description => "Specify network devices.",
 986 };
 987
 988 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 989
 990 my $ipconfig_fmt = {
 991     ip => {
 992         type => 'string',
 993         format => 'pve-ipv4-config',
 994         format_description => 'IPv4Format/CIDR',
 995         description => 'IPv4 address in CIDR format.',
 996         optional => 1,
 997         default => 'dhcp',
 998     },
 999     gw => {
1000         type => 'string',
1001         format => 'ipv4',
1002         format_description => 'GatewayIPv4',
1003         description => 'Default gateway for IPv4 traffic.',
1004         optional => 1,
1005         requires => 'ip',
1006     },
1007     ip6 => {
1008         type => 'string',
1009         format => 'pve-ipv6-config',
1010         format_description => 'IPv6Format/CIDR',
1011         description => 'IPv6 address in CIDR format.',
1012         optional => 1,
1013         default => 'dhcp',
1014     },
1015     gw6 => {
1016         type => 'string',
1017         format => 'ipv6',
1018         format_description => 'GatewayIPv6',
1019         description => 'Default gateway for IPv6 traffic.',
1020         optional => 1,
1021         requires => 'ip6',
1022     },
1023 };
1024 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1025 my $ipconfigdesc = {
1026     optional => 1,
1027     type => 'string', format => 'pve-qm-ipconfig',
1028     description => <<'EODESCR',
1029 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1030
1031 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1032
1033 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1034 gateway should be provided.
1035 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1036 cloud-init 19.4 or newer.
1037
1038 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1039 dhcp on IPv4.
1040 EODESCR
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1043
1044 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1045     $confdesc->{"net$i"} = $netdesc;
1046     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1047 }
1048
1049 foreach my $key (keys %$confdesc_cloudinit) {
1050     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1051 }
1052
1053 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1054 sub verify_volume_id_or_qm_path {
1055     my ($volid, $noerr) = @_;
1056
1057     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1058         return $volid;
1059     }
1060
1061     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1062     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1063     if ($@) {
1064         return if $noerr;
1065         die $@;
1066     }
1067     return $volid;
1068 }
1069
1070 my $usb_fmt = {
1071     host => {
1072         default_key => 1,
1073         type => 'string', format => 'pve-qm-usb-device',
1074         format_description => 'HOSTUSBDEVICE|spice',
1075         description => <<EODESCR,
1076 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1077
1078  'bus-port(.port)*' (decimal numbers) or
1079  'vendor_id:product_id' (hexadeciaml numbers) or
1080  'spice'
1081
1082 You can use the 'lsusb -t' command to list existing usb devices.
1083
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1086
1087 The value 'spice' can be used to add a usb redirection devices for spice.
1088 EODESCR
1089     },
1090     usb3 => {
1091         optional => 1,
1092         type => 'boolean',
1093         description => "Specifies whether if given host option is a USB3 device or port.",
1094         default => 0,
1095     },
1096 };
1097
1098 my $usbdesc = {
1099     optional => 1,
1100     type => 'string', format => $usb_fmt,
1101     description => "Configure an USB device (n is 0 to 4).",
1102 };
1103 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1104
1105 my $serialdesc = {
1106         optional => 1,
1107         type => 'string',
1108         pattern => '(/dev/.+|socket)',
1109         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1110         verbose_description =>  <<EODESCR,
1111 Create a serial device inside the VM (n is 0 to 3), and pass through a
1112 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1113 host side (use 'qm terminal' to open a terminal connection).
1114
1115 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1116 use with special care.
1117
1118 CAUTION: Experimental! User reported problems with this option.
1119 EODESCR
1120 };
1121
1122 my $paralleldesc= {
1123         optional => 1,
1124         type => 'string',
1125         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1126         description =>  "Map host parallel devices (n is 0 to 2).",
1127         verbose_description =>  <<EODESCR,
1128 Map host parallel devices (n is 0 to 2).
1129
1130 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1131 machines - use with special care.
1132
1133 CAUTION: Experimental! User reported problems with this option.
1134 EODESCR
1135 };
1136
1137 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1138     $confdesc->{"parallel$i"} = $paralleldesc;
1139 }
1140
1141 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1142     $confdesc->{"serial$i"} = $serialdesc;
1143 }
1144
1145 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1146     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1147 }
1148
1149 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1150     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1151 }
1152
1153 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1154     $confdesc->{"usb$i"} = $usbdesc;
1155 }
1156
1157 my $boot_fmt = {
1158     legacy => {
1159         optional => 1,
1160         default_key => 1,
1161         type => 'string',
1162         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1163                      . " Deprecated, use 'order=' instead.",
1164         pattern => '[acdn]{1,4}',
1165         format_description => "[acdn]{1,4}",
1166
1167         # note: this is also the fallback if boot: is not given at all
1168         default => 'cdn',
1169     },
1170     order => {
1171         optional => 1,
1172         type => 'string',
1173         format => 'pve-qm-bootdev-list',
1174         format_description => "device[;device...]",
1175         description => <<EODESC,
1176 The guest will attempt to boot from devices in the order they appear here.
1177
1178 Disks, optical drives and passed-through storage USB devices will be directly
1179 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1180 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1181
1182 Note that only devices in this list will be marked as bootable and thus loaded
1183 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1184 (e.g. software-raid), you need to specify all of them here.
1185
1186 Overrides the deprecated 'legacy=[acdn]*' value when given.
1187 EODESC
1188     },
1189 };
1190 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1191
1192 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1193 sub verify_bootdev {
1194     my ($dev, $noerr) = @_;
1195
1196     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1197     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1198
1199     my $check = sub {
1200         my ($base) = @_;
1201         return 0 if $dev !~ m/^$base\d+$/;
1202         return 0 if !$confdesc->{$dev};
1203         return 1;
1204     };
1205
1206     return $dev if $check->("net");
1207     return $dev if $check->("usb");
1208     return $dev if $check->("hostpci");
1209
1210     return if $noerr;
1211     die "invalid boot device '$dev'\n";
1212 }
1213
1214 sub print_bootorder {
1215     my ($devs) = @_;
1216     return "" if !@$devs;
1217     my $data = { order => join(';', @$devs) };
1218     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1219 }
1220
1221 my $kvm_api_version = 0;
1222
1223 sub kvm_version {
1224     return $kvm_api_version if $kvm_api_version;
1225
1226     open my $fh, '<', '/dev/kvm' or return;
1227
1228     # 0xae00 => KVM_GET_API_VERSION
1229     $kvm_api_version = ioctl($fh, 0xae00, 0);
1230     close($fh);
1231
1232     return $kvm_api_version;
1233 }
1234
1235 my $kvm_user_version = {};
1236 my $kvm_mtime = {};
1237
1238 sub kvm_user_version {
1239     my ($binary) = @_;
1240
1241     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1242     my $st = stat($binary);
1243
1244     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1245     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1246         $cachedmtime == $st->mtime;
1247
1248     $kvm_user_version->{$binary} = 'unknown';
1249     $kvm_mtime->{$binary} = $st->mtime;
1250
1251     my $code = sub {
1252         my $line = shift;
1253         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1254             $kvm_user_version->{$binary} = $2;
1255         }
1256     };
1257
1258     eval { run_command([$binary, '--version'], outfunc => $code); };
1259     warn $@ if $@;
1260
1261     return $kvm_user_version->{$binary};
1262
1263 }
1264 my sub extract_version {
1265     my ($machine_type, $version) = @_;
1266     $version = kvm_user_version() if !defined($version);
1267     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1268 }
1269
1270 sub kernel_has_vhost_net {
1271     return -c '/dev/vhost-net';
1272 }
1273
1274 sub option_exists {
1275     my $key = shift;
1276     return defined($confdesc->{$key});
1277 }
1278
1279 my $cdrom_path;
1280 sub get_cdrom_path {
1281
1282     return  $cdrom_path if $cdrom_path;
1283
1284     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1285     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1286     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1287 }
1288
1289 sub get_iso_path {
1290     my ($storecfg, $vmid, $cdrom) = @_;
1291
1292     if ($cdrom eq 'cdrom') {
1293         return get_cdrom_path();
1294     } elsif ($cdrom eq 'none') {
1295         return '';
1296     } elsif ($cdrom =~ m|^/|) {
1297         return $cdrom;
1298     } else {
1299         return PVE::Storage::path($storecfg, $cdrom);
1300     }
1301 }
1302
1303 # try to convert old style file names to volume IDs
1304 sub filename_to_volume_id {
1305     my ($vmid, $file, $media) = @_;
1306
1307      if (!($file eq 'none' || $file eq 'cdrom' ||
1308           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1309
1310         return if $file =~ m|/|;
1311
1312         if ($media && $media eq 'cdrom') {
1313             $file = "local:iso/$file";
1314         } else {
1315             $file = "local:$vmid/$file";
1316         }
1317     }
1318
1319     return $file;
1320 }
1321
1322 sub verify_media_type {
1323     my ($opt, $vtype, $media) = @_;
1324
1325     return if !$media;
1326
1327     my $etype;
1328     if ($media eq 'disk') {
1329         $etype = 'images';
1330     } elsif ($media eq 'cdrom') {
1331         $etype = 'iso';
1332     } else {
1333         die "internal error";
1334     }
1335
1336     return if ($vtype eq $etype);
1337
1338     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1339 }
1340
1341 sub cleanup_drive_path {
1342     my ($opt, $storecfg, $drive) = @_;
1343
1344     # try to convert filesystem paths to volume IDs
1345
1346     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1347         ($drive->{file} !~ m|^/dev/.+|) &&
1348         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1349         ($drive->{file} !~ m/^\d+$/)) {
1350         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1351         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1352             if !$vtype;
1353         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1354         verify_media_type($opt, $vtype, $drive->{media});
1355         $drive->{file} = $volid;
1356     }
1357
1358     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1359 }
1360
1361 sub parse_hotplug_features {
1362     my ($data) = @_;
1363
1364     my $res = {};
1365
1366     return $res if $data eq '0';
1367
1368     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1369
1370     foreach my $feature (PVE::Tools::split_list($data)) {
1371         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1372             $res->{$1} = 1;
1373         } else {
1374             die "invalid hotplug feature '$feature'\n";
1375         }
1376     }
1377     return $res;
1378 }
1379
1380 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1381 sub pve_verify_hotplug_features {
1382     my ($value, $noerr) = @_;
1383
1384     return $value if parse_hotplug_features($value);
1385
1386     return if $noerr;
1387
1388     die "unable to parse hotplug option\n";
1389 }
1390
1391 sub scsi_inquiry {
1392     my($fh, $noerr) = @_;
1393
1394     my $SG_IO = 0x2285;
1395     my $SG_GET_VERSION_NUM = 0x2282;
1396
1397     my $versionbuf = "\x00" x 8;
1398     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1399     if (!$ret) {
1400         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1401         return;
1402     }
1403     my $version = unpack("I", $versionbuf);
1404     if ($version < 30000) {
1405         die "scsi generic interface too old\n"  if !$noerr;
1406         return;
1407     }
1408
1409     my $buf = "\x00" x 36;
1410     my $sensebuf = "\x00" x 8;
1411     my $cmd = pack("C x3 C x1", 0x12, 36);
1412
1413     # see /usr/include/scsi/sg.h
1414     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1415
1416     my $packet = pack(
1417         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1418     );
1419
1420     $ret = ioctl($fh, $SG_IO, $packet);
1421     if (!$ret) {
1422         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1423         return;
1424     }
1425
1426     my @res = unpack($sg_io_hdr_t, $packet);
1427     if ($res[17] || $res[18]) {
1428         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1429         return;
1430     }
1431
1432     my $res = {};
1433     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1434
1435     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1436     $res->{type} &= 0x1F;
1437
1438     return $res;
1439 }
1440
1441 sub path_is_scsi {
1442     my ($path) = @_;
1443
1444     my $fh = IO::File->new("+<$path") || return;
1445     my $res = scsi_inquiry($fh, 1);
1446     close($fh);
1447
1448     return $res;
1449 }
1450
1451 sub print_tabletdevice_full {
1452     my ($conf, $arch) = @_;
1453
1454     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1455
1456     # we use uhci for old VMs because tablet driver was buggy in older qemu
1457     my $usbbus;
1458     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1459         $usbbus = 'ehci';
1460     } else {
1461         $usbbus = 'uhci';
1462     }
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_keyboarddevice_full {
1468     my ($conf, $arch) = @_;
1469
1470     return if $arch ne 'aarch64';
1471
1472     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1473 }
1474
1475 my sub get_drive_id {
1476     my ($drive) = @_;
1477     return "$drive->{interface}$drive->{index}";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     my $drive_id = get_drive_id($drive);
1487     if ($drive->{interface} eq 'virtio') {
1488         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1489         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1490         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1491     } elsif ($drive->{interface} eq 'scsi') {
1492
1493         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1494         my $unit = $drive->{index} % $maxdev;
1495         my $devicetype = 'hd';
1496         my $path = '';
1497         if (drive_is_cdrom($drive)) {
1498             $devicetype = 'cd';
1499         } else {
1500             if ($drive->{file} =~ m|^/|) {
1501                 $path = $drive->{file};
1502                 if (my $info = path_is_scsi($path)) {
1503                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1504                         $devicetype = 'block';
1505                     } elsif ($info->{type} == 1) { # tape
1506                         $devicetype = 'generic';
1507                     }
1508                 }
1509             } else {
1510                  $path = PVE::Storage::path($storecfg, $drive->{file});
1511             }
1512
1513             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1514             my $version = extract_version($machine_type, kvm_user_version());
1515             if ($path =~ m/^iscsi\:\/\// &&
1516                !min_version($version, 4, 1)) {
1517                 $devicetype = 'generic';
1518             }
1519         }
1520
1521         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1522             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1523         } else {
1524             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1525                 .",lun=$drive->{index}";
1526         }
1527         $device .= ",drive=drive-$drive_id,id=$drive_id";
1528
1529         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1530             $device .= ",rotation_rate=1";
1531         }
1532         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1533
1534     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1535         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype";
1541         if ($drive->{interface} eq 'ide') {
1542             $device .= ",bus=ide.$controller,unit=$unit";
1543         } else {
1544             $device .= ",bus=ahci$controller.$unit";
1545         }
1546         $device .= ",drive=drive-$drive_id,id=$drive_id";
1547
1548         if ($devicetype eq 'hd') {
1549             if (my $model = $drive->{model}) {
1550                 $model = URI::Escape::uri_unescape($model);
1551                 $device .= ",model=$model";
1552             }
1553             if ($drive->{ssd}) {
1554                 $device .= ",rotation_rate=1";
1555             }
1556         }
1557         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1558     } elsif ($drive->{interface} eq 'usb') {
1559         die "implement me";
1560         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1561     } else {
1562         die "unsupported interface type";
1563     }
1564
1565     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1566
1567     if (my $serial = $drive->{serial}) {
1568         $serial = URI::Escape::uri_unescape($serial);
1569         $device .= ",serial=$serial";
1570     }
1571
1572
1573     return $device;
1574 }
1575
1576 sub get_initiator_name {
1577     my $initiator;
1578
1579     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1580     while (defined(my $line = <$fh>)) {
1581         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1582         $initiator = $1;
1583         last;
1584     }
1585     $fh->close();
1586
1587     return $initiator;
1588 }
1589
1590 sub print_drive_commandline_full {
1591     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1592
1593     my $path;
1594     my $volid = $drive->{file};
1595     my $format = $drive->{format};
1596     my $drive_id = get_drive_id($drive);
1597
1598     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1599     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1600
1601     if (drive_is_cdrom($drive)) {
1602         $path = get_iso_path($storecfg, $vmid, $volid);
1603         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1604     } else {
1605         if ($storeid) {
1606             $path = PVE::Storage::path($storecfg, $volid);
1607             $format //= qemu_img_format($scfg, $volname);
1608         } else {
1609             $path = $volid;
1610             $format //= "raw";
1611         }
1612    }
1613
1614    my $is_rbd = $path =~ m/^rbd:/;
1615
1616     my $opts = '';
1617     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1618     foreach my $o (@qemu_drive_options) {
1619         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1620     }
1621
1622     # snapshot only accepts on|off
1623     if (defined($drive->{snapshot})) {
1624         my $v = $drive->{snapshot} ? 'on' : 'off';
1625         $opts .= ",snapshot=$v";
1626     }
1627
1628     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1629         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1630     }
1631
1632     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1633         my ($dir, $qmpname) = @$type;
1634         if (my $v = $drive->{"mbps$dir"}) {
1635             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1636         }
1637         if (my $v = $drive->{"mbps${dir}_max"}) {
1638             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1639         }
1640         if (my $v = $drive->{"bps${dir}_max_length"}) {
1641             $opts .= ",throttling.bps$qmpname-max-length=$v";
1642         }
1643         if (my $v = $drive->{"iops${dir}"}) {
1644             $opts .= ",throttling.iops$qmpname=$v";
1645         }
1646         if (my $v = $drive->{"iops${dir}_max"}) {
1647             $opts .= ",throttling.iops$qmpname-max=$v";
1648         }
1649         if (my $v = $drive->{"iops${dir}_max_length"}) {
1650             $opts .= ",throttling.iops$qmpname-max-length=$v";
1651         }
1652     }
1653
1654     if ($pbs_name) {
1655         $format = "rbd" if $is_rbd;
1656         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1657             if !$format;
1658         $opts .= ",format=alloc-track,file.driver=$format";
1659     } elsif ($format) {
1660         $opts .= ",format=$format";
1661     }
1662
1663     my $cache_direct = 0;
1664
1665     if (my $cache = $drive->{cache}) {
1666         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1667     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1668         $opts .= ",cache=none";
1669         $cache_direct = 1;
1670     }
1671
1672     # io_uring with cache mode writeback or writethrough on krbd will hang...
1673     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1674
1675     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1676     # sometimes, just plain disable...
1677     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1678
1679     if (!$drive->{aio}) {
1680         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1681             # io_uring supports all cache modes
1682             $opts .= ",aio=io_uring";
1683         } else {
1684             # aio native works only with O_DIRECT
1685             if($cache_direct) {
1686                 $opts .= ",aio=native";
1687             } else {
1688                 $opts .= ",aio=threads";
1689             }
1690         }
1691     }
1692
1693     if (!drive_is_cdrom($drive)) {
1694         my $detectzeroes;
1695         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1696             $detectzeroes = 'off';
1697         } elsif ($drive->{discard}) {
1698             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1699         } else {
1700             # This used to be our default with discard not being specified:
1701             $detectzeroes = 'on';
1702         }
1703
1704         # note: 'detect-zeroes' works per blockdev and we want it to persist
1705         # after the alloc-track is removed, so put it on 'file' directly
1706         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1707         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1708     }
1709
1710     if ($pbs_name) {
1711         $opts .= ",backing=$pbs_name";
1712         $opts .= ",auto-remove=on";
1713     }
1714
1715     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1716     my $file_param = "file";
1717     if ($pbs_name) {
1718         # non-rbd drivers require the underlying file to be a seperate block
1719         # node, so add a second .file indirection
1720         $file_param .= ".file" if !$is_rbd;
1721         $file_param .= ".filename";
1722     }
1723     my $pathinfo = $path ? "$file_param=$path," : '';
1724
1725     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1726 }
1727
1728 sub print_pbs_blockdev {
1729     my ($pbs_conf, $pbs_name) = @_;
1730     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1731     $blockdev .= ",repository=$pbs_conf->{repository}";
1732     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1733     $blockdev .= ",archive=$pbs_conf->{archive}";
1734     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1735     return $blockdev;
1736 }
1737
1738 sub print_netdevice_full {
1739     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1740
1741     my $device = $net->{model};
1742     if ($net->{model} eq 'virtio') {
1743          $device = 'virtio-net-pci';
1744      };
1745
1746     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1747     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1748     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1749         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1750         # and out of each queue plus one config interrupt and control vector queue
1751         my $vectors = $net->{queues} * 2 + 2;
1752         $tmpstr .= ",vectors=$vectors,mq=on";
1753     }
1754     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1755
1756     if (my $mtu = $net->{mtu}) {
1757         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1758             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1759             if ($mtu == 1) {
1760                  $mtu = $bridge_mtu;
1761             } elsif ($mtu < 576) {
1762                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1763             } elsif ($mtu > $bridge_mtu) {
1764                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1765             }
1766             $tmpstr .= ",host_mtu=$mtu";
1767         } else {
1768             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1769         }
1770     }
1771
1772     if ($use_old_bios_files) {
1773         my $romfile;
1774         if ($device eq 'virtio-net-pci') {
1775             $romfile = 'pxe-virtio.rom';
1776         } elsif ($device eq 'e1000') {
1777             $romfile = 'pxe-e1000.rom';
1778         } elsif ($device eq 'e1000e') {
1779             $romfile = 'pxe-e1000e.rom';
1780         } elsif ($device eq 'ne2k') {
1781             $romfile = 'pxe-ne2k_pci.rom';
1782         } elsif ($device eq 'pcnet') {
1783             $romfile = 'pxe-pcnet.rom';
1784         } elsif ($device eq 'rtl8139') {
1785             $romfile = 'pxe-rtl8139.rom';
1786         }
1787         $tmpstr .= ",romfile=$romfile" if $romfile;
1788     }
1789
1790     return $tmpstr;
1791 }
1792
1793 sub print_netdev_full {
1794     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1795
1796     my $i = '';
1797     if ($netid =~ m/^net(\d+)$/) {
1798         $i = int($1);
1799     }
1800
1801     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1802
1803     my $ifname = "tap${vmid}i$i";
1804
1805     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1806     die "interface name '$ifname' is too long (max 15 character)\n"
1807         if length($ifname) >= 16;
1808
1809     my $vhostparam = '';
1810     if (is_native($arch)) {
1811         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1812     }
1813
1814     my $vmname = $conf->{name} || "vm$vmid";
1815
1816     my $netdev = "";
1817     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1818
1819     if ($net->{bridge}) {
1820         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1821             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1822     } else {
1823         $netdev = "type=user,id=$netid,hostname=$vmname";
1824     }
1825
1826     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1827
1828     return $netdev;
1829 }
1830
1831 my $vga_map = {
1832     'cirrus' => 'cirrus-vga',
1833     'std' => 'VGA',
1834     'vmware' => 'vmware-svga',
1835     'virtio' => 'virtio-vga',
1836 };
1837
1838 sub print_vga_device {
1839     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1840
1841     my $type = $vga_map->{$vga->{type}};
1842     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1843         $type = 'virtio-gpu';
1844     }
1845     my $vgamem_mb = $vga->{memory};
1846
1847     my $max_outputs = '';
1848     if ($qxlnum) {
1849         $type = $id ? 'qxl' : 'qxl-vga';
1850
1851         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1852             # set max outputs so linux can have up to 4 qxl displays with one device
1853             if (min_version($machine_version, 4, 1)) {
1854                 $max_outputs = ",max_outputs=4";
1855             }
1856         }
1857     }
1858
1859     die "no devicetype for $vga->{type}\n" if !$type;
1860
1861     my $memory = "";
1862     if ($vgamem_mb) {
1863         if ($vga->{type} eq 'virtio') {
1864             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1865             $memory = ",max_hostmem=$bytes";
1866         } elsif ($qxlnum) {
1867             # from https://www.spice-space.org/multiple-monitors.html
1868             $memory = ",vgamem_mb=$vga->{memory}";
1869             my $ram = $vgamem_mb * 4;
1870             my $vram = $vgamem_mb * 2;
1871             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1872         } else {
1873             $memory = ",vgamem_mb=$vga->{memory}";
1874         }
1875     } elsif ($qxlnum && $id) {
1876         $memory = ",ram_size=67108864,vram_size=33554432";
1877     }
1878
1879     my $edidoff = "";
1880     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1881         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1882     }
1883
1884     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1885     my $vgaid = "vga" . ($id // '');
1886     my $pciaddr;
1887     if ($q35 && $vgaid eq 'vga') {
1888         # the first display uses pcie.0 bus on q35 machines
1889         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1890     } else {
1891         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 sub parse_number_sets {
1898     my ($set) = @_;
1899     my $res = [];
1900     foreach my $part (split(/;/, $set)) {
1901         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1902             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1903             push @$res, [ $1, $2 ];
1904         } else {
1905             die "invalid range: $part\n";
1906         }
1907     }
1908     return $res;
1909 }
1910
1911 sub parse_numa {
1912     my ($data) = @_;
1913
1914     my $res = parse_property_string($numa_fmt, $data);
1915     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1916     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1917     return $res;
1918 }
1919
1920 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1921 sub parse_net {
1922     my ($data) = @_;
1923
1924     my $res = eval { parse_property_string($net_fmt, $data) };
1925     if ($@) {
1926         warn $@;
1927         return;
1928     }
1929     if (!defined($res->{macaddr})) {
1930         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1931         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1932     }
1933     return $res;
1934 }
1935
1936 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1937 sub parse_ipconfig {
1938     my ($data) = @_;
1939
1940     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1941     if ($@) {
1942         warn $@;
1943         return;
1944     }
1945
1946     if ($res->{gw} && !$res->{ip}) {
1947         warn 'gateway specified without specifying an IP address';
1948         return;
1949     }
1950     if ($res->{gw6} && !$res->{ip6}) {
1951         warn 'IPv6 gateway specified without specifying an IPv6 address';
1952         return;
1953     }
1954     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1955         warn 'gateway specified together with DHCP';
1956         return;
1957     }
1958     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1959         # gw6 + auto/dhcp
1960         warn "IPv6 gateway specified together with $res->{ip6} address";
1961         return;
1962     }
1963
1964     if (!$res->{ip} && !$res->{ip6}) {
1965         return { ip => 'dhcp', ip6 => 'dhcp' };
1966     }
1967
1968     return $res;
1969 }
1970
1971 sub print_net {
1972     my $net = shift;
1973
1974     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1975 }
1976
1977 sub add_random_macs {
1978     my ($settings) = @_;
1979
1980     foreach my $opt (keys %$settings) {
1981         next if $opt !~ m/^net(\d+)$/;
1982         my $net = parse_net($settings->{$opt});
1983         next if !$net;
1984         $settings->{$opt} = print_net($net);
1985     }
1986 }
1987
1988 sub vm_is_volid_owner {
1989     my ($storecfg, $vmid, $volid) = @_;
1990
1991     if ($volid !~  m|^/|) {
1992         my ($path, $owner);
1993         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1994         if ($owner && ($owner == $vmid)) {
1995             return 1;
1996         }
1997     }
1998
1999     return;
2000 }
2001
2002 sub vmconfig_register_unused_drive {
2003     my ($storecfg, $vmid, $conf, $drive) = @_;
2004
2005     if (drive_is_cloudinit($drive)) {
2006         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2007         warn $@ if $@;
2008     } elsif (!drive_is_cdrom($drive)) {
2009         my $volid = $drive->{file};
2010         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2011             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2012         }
2013     }
2014 }
2015
2016 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2017 my $smbios1_fmt = {
2018     uuid => {
2019         type => 'string',
2020         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2021         format_description => 'UUID',
2022         description => "Set SMBIOS1 UUID.",
2023         optional => 1,
2024     },
2025     version => {
2026         type => 'string',
2027         pattern => '[A-Za-z0-9+\/]+={0,2}',
2028         format_description => 'Base64 encoded string',
2029         description => "Set SMBIOS1 version.",
2030         optional => 1,
2031     },
2032     serial => {
2033         type => 'string',
2034         pattern => '[A-Za-z0-9+\/]+={0,2}',
2035         format_description => 'Base64 encoded string',
2036         description => "Set SMBIOS1 serial number.",
2037         optional => 1,
2038     },
2039     manufacturer => {
2040         type => 'string',
2041         pattern => '[A-Za-z0-9+\/]+={0,2}',
2042         format_description => 'Base64 encoded string',
2043         description => "Set SMBIOS1 manufacturer.",
2044         optional => 1,
2045     },
2046     product => {
2047         type => 'string',
2048         pattern => '[A-Za-z0-9+\/]+={0,2}',
2049         format_description => 'Base64 encoded string',
2050         description => "Set SMBIOS1 product ID.",
2051         optional => 1,
2052     },
2053     sku => {
2054         type => 'string',
2055         pattern => '[A-Za-z0-9+\/]+={0,2}',
2056         format_description => 'Base64 encoded string',
2057         description => "Set SMBIOS1 SKU string.",
2058         optional => 1,
2059     },
2060     family => {
2061         type => 'string',
2062         pattern => '[A-Za-z0-9+\/]+={0,2}',
2063         format_description => 'Base64 encoded string',
2064         description => "Set SMBIOS1 family string.",
2065         optional => 1,
2066     },
2067     base64 => {
2068         type => 'boolean',
2069         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2070         optional => 1,
2071     },
2072 };
2073
2074 sub parse_smbios1 {
2075     my ($data) = @_;
2076
2077     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub print_smbios1 {
2083     my ($smbios1) = @_;
2084     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2085 }
2086
2087 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2088
2089 sub parse_watchdog {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 sub parse_guest_agent {
2100     my ($conf) = @_;
2101
2102     return {} if !defined($conf->{agent});
2103
2104     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2105     warn $@ if $@;
2106
2107     # if the agent is disabled ignore the other potentially set properties
2108     return {} if !$res->{enabled};
2109     return $res;
2110 }
2111
2112 sub get_qga_key {
2113     my ($conf, $key) = @_;
2114     return undef if !defined($conf->{agent});
2115
2116     my $agent = parse_guest_agent($conf);
2117     return $agent->{$key};
2118 }
2119
2120 sub parse_vga {
2121     my ($value) = @_;
2122
2123     return {} if !$value;
2124     my $res = eval { parse_property_string($vga_fmt, $value) };
2125     warn $@ if $@;
2126     return $res;
2127 }
2128
2129 sub parse_rng {
2130     my ($value) = @_;
2131
2132     return if !$value;
2133
2134     my $res = eval { parse_property_string($rng_fmt, $value) };
2135     warn $@ if $@;
2136     return $res;
2137 }
2138
2139 sub parse_meta_info {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub new_meta_info_string {
2150     my () = @_; # for now do not allow to override any value
2151
2152     return PVE::JSONSchema::print_property_string(
2153         {
2154             'creation-qemu' => kvm_user_version(),
2155             ctime => "". int(time()),
2156         },
2157         $meta_info_fmt
2158     );
2159 }
2160
2161 sub qemu_created_version_fixups {
2162     my ($conf, $forcemachine, $kvmver) = @_;
2163
2164     my $meta = parse_meta_info($conf->{meta}) // {};
2165     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2166
2167     # check if we need to apply some handling for VMs that always use the latest machine version but
2168     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2169     # would be required (we do not want to pin machine version for non-windows OS type)
2170     if (
2171         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2172         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2173         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2174         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2175     ) {
2176         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2177         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2178             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2179             # and thus with the predictable interface naming of systemd
2180             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2181         }
2182     }
2183     return;
2184 }
2185
2186 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2187 sub verify_usb_device {
2188     my ($value, $noerr) = @_;
2189
2190     return $value if parse_usb_device($value);
2191
2192     return if $noerr;
2193
2194     die "unable to parse usb device\n";
2195 }
2196
2197 # add JSON properties for create and set function
2198 sub json_config_properties {
2199     my $prop = shift;
2200
2201     my $skip_json_config_opts = {
2202         parent => 1,
2203         snaptime => 1,
2204         vmstate => 1,
2205         runningmachine => 1,
2206         runningcpu => 1,
2207         meta => 1,
2208     };
2209
2210     foreach my $opt (keys %$confdesc) {
2211         next if $skip_json_config_opts->{$opt};
2212         $prop->{$opt} = $confdesc->{$opt};
2213     }
2214
2215     return $prop;
2216 }
2217
2218 # return copy of $confdesc_cloudinit to generate documentation
2219 sub cloudinit_config_properties {
2220
2221     return dclone($confdesc_cloudinit);
2222 }
2223
2224 sub check_type {
2225     my ($key, $value) = @_;
2226
2227     die "unknown setting '$key'\n" if !$confdesc->{$key};
2228
2229     my $type = $confdesc->{$key}->{type};
2230
2231     if (!defined($value)) {
2232         die "got undefined value\n";
2233     }
2234
2235     if ($value =~ m/[\n\r]/) {
2236         die "property contains a line feed\n";
2237     }
2238
2239     if ($type eq 'boolean') {
2240         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2241         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2242         die "type check ('boolean') failed - got '$value'\n";
2243     } elsif ($type eq 'integer') {
2244         return int($1) if $value =~ m/^(\d+)$/;
2245         die "type check ('integer') failed - got '$value'\n";
2246     } elsif ($type eq 'number') {
2247         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2248         die "type check ('number') failed - got '$value'\n";
2249     } elsif ($type eq 'string') {
2250         if (my $fmt = $confdesc->{$key}->{format}) {
2251             PVE::JSONSchema::check_format($fmt, $value);
2252             return $value;
2253         }
2254         $value =~ s/^\"(.*)\"$/$1/;
2255         return $value;
2256     } else {
2257         die "internal error"
2258     }
2259 }
2260
2261 sub destroy_vm {
2262     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2263
2264     my $conf = PVE::QemuConfig->load_config($vmid);
2265
2266     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2267
2268     if ($conf->{template}) {
2269         # check if any base image is still used by a linked clone
2270         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2271                 my ($ds, $drive) = @_;
2272                 return if drive_is_cdrom($drive);
2273
2274                 my $volid = $drive->{file};
2275                 return if !$volid || $volid =~ m|^/|;
2276
2277                 die "base volume '$volid' is still in use by linked cloned\n"
2278                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2279
2280         });
2281     }
2282
2283     my $volids = {};
2284     my $remove_owned_drive = sub {
2285         my ($ds, $drive) = @_;
2286         return if drive_is_cdrom($drive, 1);
2287
2288         my $volid = $drive->{file};
2289         return if !$volid || $volid =~ m|^/|;
2290         return if $volids->{$volid};
2291
2292         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2293         return if !$path || !$owner || ($owner != $vmid);
2294
2295         $volids->{$volid} = 1;
2296         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2297         warn "Could not remove disk '$volid', check manually: $@" if $@;
2298     };
2299
2300     # only remove disks owned by this VM (referenced in the config)
2301     my $include_opts = {
2302         include_unused => 1,
2303         extra_keys => ['vmstate'],
2304     };
2305     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2306
2307     for my $snap (values %{$conf->{snapshots}}) {
2308         next if !defined($snap->{vmstate});
2309         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2310         next if !defined($drive);
2311         $remove_owned_drive->('vmstate', $drive);
2312     }
2313
2314     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2315
2316     if ($purge_unreferenced) { # also remove unreferenced disk
2317         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2318         PVE::Storage::foreach_volid($vmdisks, sub {
2319             my ($volid, $sid, $volname, $d) = @_;
2320             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2321             warn $@ if $@;
2322         });
2323     }
2324
2325     if (defined $replacement_conf) {
2326         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2327     } else {
2328         PVE::QemuConfig->destroy_config($vmid);
2329     }
2330 }
2331
2332 sub parse_vm_config {
2333     my ($filename, $raw) = @_;
2334
2335     return if !defined($raw);
2336
2337     my $res = {
2338         digest => Digest::SHA::sha1_hex($raw),
2339         snapshots => {},
2340         pending => {},
2341     };
2342
2343     $filename =~ m|/qemu-server/(\d+)\.conf$|
2344         || die "got strange filename '$filename'";
2345
2346     my $vmid = $1;
2347
2348     my $conf = $res;
2349     my $descr;
2350     my $section = '';
2351
2352     my @lines = split(/\n/, $raw);
2353     foreach my $line (@lines) {
2354         next if $line =~ m/^\s*$/;
2355
2356         if ($line =~ m/^\[PENDING\]\s*$/i) {
2357             $section = 'pending';
2358             if (defined($descr)) {
2359                 $descr =~ s/\s+$//;
2360                 $conf->{description} = $descr;
2361             }
2362             $descr = undef;
2363             $conf = $res->{$section} = {};
2364             next;
2365
2366         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2367             $section = $1;
2368             if (defined($descr)) {
2369                 $descr =~ s/\s+$//;
2370                 $conf->{description} = $descr;
2371             }
2372             $descr = undef;
2373             $conf = $res->{snapshots}->{$section} = {};
2374             next;
2375         }
2376
2377         if ($line =~ m/^\#(.*)\s*$/) {
2378             $descr = '' if !defined($descr);
2379             $descr .= PVE::Tools::decode_text($1) . "\n";
2380             next;
2381         }
2382
2383         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2384             $descr = '' if !defined($descr);
2385             $descr .= PVE::Tools::decode_text($2);
2386         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2387             $conf->{snapstate} = $1;
2388         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             $conf->{$key} = $value;
2392         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2393             my $value = $1;
2394             if ($section eq 'pending') {
2395                 $conf->{delete} = $value; # we parse this later
2396             } else {
2397                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2398             }
2399         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2400             my $key = $1;
2401             my $value = $2;
2402             eval { $value = check_type($key, $value); };
2403             if ($@) {
2404                 warn "vm $vmid - unable to parse value of '$key' - $@";
2405             } else {
2406                 $key = 'ide2' if $key eq 'cdrom';
2407                 my $fmt = $confdesc->{$key}->{format};
2408                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2409                     my $v = parse_drive($key, $value);
2410                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2411                         $v->{file} = $volid;
2412                         $value = print_drive($v);
2413                     } else {
2414                         warn "vm $vmid - unable to parse value of '$key'\n";
2415                         next;
2416                     }
2417                 }
2418
2419                 $conf->{$key} = $value;
2420             }
2421         } else {
2422             warn "vm $vmid - unable to parse config: $line\n";
2423         }
2424     }
2425
2426     if (defined($descr)) {
2427         $descr =~ s/\s+$//;
2428         $conf->{description} = $descr;
2429     }
2430     delete $res->{snapstate}; # just to be sure
2431
2432     return $res;
2433 }
2434
2435 sub write_vm_config {
2436     my ($filename, $conf) = @_;
2437
2438     delete $conf->{snapstate}; # just to be sure
2439
2440     if ($conf->{cdrom}) {
2441         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2442         $conf->{ide2} = $conf->{cdrom};
2443         delete $conf->{cdrom};
2444     }
2445
2446     # we do not use 'smp' any longer
2447     if ($conf->{sockets}) {
2448         delete $conf->{smp};
2449     } elsif ($conf->{smp}) {
2450         $conf->{sockets} = $conf->{smp};
2451         delete $conf->{cores};
2452         delete $conf->{smp};
2453     }
2454
2455     my $used_volids = {};
2456
2457     my $cleanup_config = sub {
2458         my ($cref, $pending, $snapname) = @_;
2459
2460         foreach my $key (keys %$cref) {
2461             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2462                 $key eq 'snapstate' || $key eq 'pending';
2463             my $value = $cref->{$key};
2464             if ($key eq 'delete') {
2465                 die "propertry 'delete' is only allowed in [PENDING]\n"
2466                     if !$pending;
2467                 # fixme: check syntax?
2468                 next;
2469             }
2470             eval { $value = check_type($key, $value); };
2471             die "unable to parse value of '$key' - $@" if $@;
2472
2473             $cref->{$key} = $value;
2474
2475             if (!$snapname && is_valid_drivename($key)) {
2476                 my $drive = parse_drive($key, $value);
2477                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2478             }
2479         }
2480     };
2481
2482     &$cleanup_config($conf);
2483
2484     &$cleanup_config($conf->{pending}, 1);
2485
2486     foreach my $snapname (keys %{$conf->{snapshots}}) {
2487         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2488         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2489     }
2490
2491     # remove 'unusedX' settings if we re-add a volume
2492     foreach my $key (keys %$conf) {
2493         my $value = $conf->{$key};
2494         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2495             delete $conf->{$key};
2496         }
2497     }
2498
2499     my $generate_raw_config = sub {
2500         my ($conf, $pending) = @_;
2501
2502         my $raw = '';
2503
2504         # add description as comment to top of file
2505         if (defined(my $descr = $conf->{description})) {
2506             if ($descr) {
2507                 foreach my $cl (split(/\n/, $descr)) {
2508                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2509                 }
2510             } else {
2511                 $raw .= "#\n" if $pending;
2512             }
2513         }
2514
2515         foreach my $key (sort keys %$conf) {
2516             next if $key =~ /^(digest|description|pending|snapshots)$/;
2517             $raw .= "$key: $conf->{$key}\n";
2518         }
2519         return $raw;
2520     };
2521
2522     my $raw = &$generate_raw_config($conf);
2523
2524     if (scalar(keys %{$conf->{pending}})){
2525         $raw .= "\n[PENDING]\n";
2526         $raw .= &$generate_raw_config($conf->{pending}, 1);
2527     }
2528
2529     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2530         $raw .= "\n[$snapname]\n";
2531         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2532     }
2533
2534     return $raw;
2535 }
2536
2537 sub load_defaults {
2538
2539     my $res = {};
2540
2541     # we use static defaults from our JSON schema configuration
2542     foreach my $key (keys %$confdesc) {
2543         if (defined(my $default = $confdesc->{$key}->{default})) {
2544             $res->{$key} = $default;
2545         }
2546     }
2547
2548     return $res;
2549 }
2550
2551 sub config_list {
2552     my $vmlist = PVE::Cluster::get_vmlist();
2553     my $res = {};
2554     return $res if !$vmlist || !$vmlist->{ids};
2555     my $ids = $vmlist->{ids};
2556     my $nodename = nodename();
2557
2558     foreach my $vmid (keys %$ids) {
2559         my $d = $ids->{$vmid};
2560         next if !$d->{node} || $d->{node} ne $nodename;
2561         next if !$d->{type} || $d->{type} ne 'qemu';
2562         $res->{$vmid}->{exists} = 1;
2563     }
2564     return $res;
2565 }
2566
2567 # test if VM uses local resources (to prevent migration)
2568 sub check_local_resources {
2569     my ($conf, $noerr) = @_;
2570
2571     my @loc_res = ();
2572
2573     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2574     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2575
2576     push @loc_res, "ivshmem" if $conf->{ivshmem};
2577
2578     foreach my $k (keys %$conf) {
2579         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2580         # sockets are safe: they will recreated be on the target side post-migrate
2581         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2582         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2583     }
2584
2585     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2586
2587     return \@loc_res;
2588 }
2589
2590 # check if used storages are available on all nodes (use by migrate)
2591 sub check_storage_availability {
2592     my ($storecfg, $conf, $node) = @_;
2593
2594     PVE::QemuConfig->foreach_volume($conf, sub {
2595         my ($ds, $drive) = @_;
2596
2597         my $volid = $drive->{file};
2598         return if !$volid;
2599
2600         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2601         return if !$sid;
2602
2603         # check if storage is available on both nodes
2604         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2605         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2606
2607         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2608
2609         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2610             if !$scfg->{content}->{$vtype};
2611    });
2612 }
2613
2614 # list nodes where all VM images are available (used by has_feature API)
2615 sub shared_nodes {
2616     my ($conf, $storecfg) = @_;
2617
2618     my $nodelist = PVE::Cluster::get_nodelist();
2619     my $nodehash = { map { $_ => 1 } @$nodelist };
2620     my $nodename = nodename();
2621
2622     PVE::QemuConfig->foreach_volume($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         if ($storeid) {
2630             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2631             if ($scfg->{disable}) {
2632                 $nodehash = {};
2633             } elsif (my $avail = $scfg->{nodes}) {
2634                 foreach my $node (keys %$nodehash) {
2635                     delete $nodehash->{$node} if !$avail->{$node};
2636                 }
2637             } elsif (!$scfg->{shared}) {
2638                 foreach my $node (keys %$nodehash) {
2639                     delete $nodehash->{$node} if $node ne $nodename
2640                 }
2641             }
2642         }
2643     });
2644
2645     return $nodehash
2646 }
2647
2648 sub check_local_storage_availability {
2649     my ($conf, $storecfg) = @_;
2650
2651     my $nodelist = PVE::Cluster::get_nodelist();
2652     my $nodehash = { map { $_ => {} } @$nodelist };
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663
2664             if ($scfg->{disable}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2667                 }
2668             } elsif (my $avail = $scfg->{nodes}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     if (!$avail->{$node}) {
2671                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2672                     }
2673                 }
2674             }
2675         }
2676     });
2677
2678     foreach my $node (values %$nodehash) {
2679         if (my $unavail = $node->{unavailable_storages}) {
2680             $node->{unavailable_storages} = [ sort keys %$unavail ];
2681         }
2682     }
2683
2684     return $nodehash
2685 }
2686
2687 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2688 sub check_running {
2689     my ($vmid, $nocheck, $node) = @_;
2690
2691     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2692     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2693 }
2694
2695 sub vzlist {
2696
2697     my $vzlist = config_list();
2698
2699     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2700
2701     while (defined(my $de = $fd->read)) {
2702         next if $de !~ m/^(\d+)\.pid$/;
2703         my $vmid = $1;
2704         next if !defined($vzlist->{$vmid});
2705         if (my $pid = check_running($vmid)) {
2706             $vzlist->{$vmid}->{pid} = $pid;
2707         }
2708     }
2709
2710     return $vzlist;
2711 }
2712
2713 our $vmstatus_return_properties = {
2714     vmid => get_standard_option('pve-vmid'),
2715     status => {
2716         description => "Qemu process status.",
2717         type => 'string',
2718         enum => ['stopped', 'running'],
2719     },
2720     maxmem => {
2721         description => "Maximum memory in bytes.",
2722         type => 'integer',
2723         optional => 1,
2724         renderer => 'bytes',
2725     },
2726     maxdisk => {
2727         description => "Root disk size in bytes.",
2728         type => 'integer',
2729         optional => 1,
2730         renderer => 'bytes',
2731     },
2732     name => {
2733         description => "VM name.",
2734         type => 'string',
2735         optional => 1,
2736     },
2737     qmpstatus => {
2738         description => "Qemu QMP agent status.",
2739         type => 'string',
2740         optional => 1,
2741     },
2742     pid => {
2743         description => "PID of running qemu process.",
2744         type => 'integer',
2745         optional => 1,
2746     },
2747     uptime => {
2748         description => "Uptime.",
2749         type => 'integer',
2750         optional => 1,
2751         renderer => 'duration',
2752     },
2753     cpus => {
2754         description => "Maximum usable CPUs.",
2755         type => 'number',
2756         optional => 1,
2757     },
2758     lock => {
2759         description => "The current config lock, if any.",
2760         type => 'string',
2761         optional => 1,
2762     },
2763     tags => {
2764         description  => "The current configured tags, if any",
2765         type => 'string',
2766         optional => 1,
2767     },
2768     'running-machine' => {
2769         description => "The currently running machine type (if running).",
2770         type => 'string',
2771         optional => 1,
2772     },
2773     'running-qemu' => {
2774         description => "The currently running QEMU version (if running).",
2775         type => 'string',
2776         optional => 1,
2777     },
2778 };
2779
2780 my $last_proc_pid_stat;
2781
2782 # get VM status information
2783 # This must be fast and should not block ($full == false)
2784 # We only query KVM using QMP if $full == true (this can be slow)
2785 sub vmstatus {
2786     my ($opt_vmid, $full) = @_;
2787
2788     my $res = {};
2789
2790     my $storecfg = PVE::Storage::config();
2791
2792     my $list = vzlist();
2793     my $defaults = load_defaults();
2794
2795     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2796
2797     my $cpucount = $cpuinfo->{cpus} || 1;
2798
2799     foreach my $vmid (keys %$list) {
2800         next if $opt_vmid && ($vmid ne $opt_vmid);
2801
2802         my $conf = PVE::QemuConfig->load_config($vmid);
2803
2804         my $d = { vmid => int($vmid) };
2805         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2806
2807         # fixme: better status?
2808         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2809
2810         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2811         if (defined($size)) {
2812             $d->{disk} = 0; # no info available
2813             $d->{maxdisk} = $size;
2814         } else {
2815             $d->{disk} = 0;
2816             $d->{maxdisk} = 0;
2817         }
2818
2819         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2820             * ($conf->{cores} || $defaults->{cores});
2821         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2822         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2823
2824         $d->{name} = $conf->{name} || "VM $vmid";
2825         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2826             : $defaults->{memory}*(1024*1024);
2827
2828         if ($conf->{balloon}) {
2829             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2830             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2831                 : $defaults->{shares};
2832         }
2833
2834         $d->{uptime} = 0;
2835         $d->{cpu} = 0;
2836         $d->{mem} = 0;
2837
2838         $d->{netout} = 0;
2839         $d->{netin} = 0;
2840
2841         $d->{diskread} = 0;
2842         $d->{diskwrite} = 0;
2843
2844         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2845
2846         $d->{serial} = 1 if conf_has_serial($conf);
2847         $d->{lock} = $conf->{lock} if $conf->{lock};
2848         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2849
2850         $res->{$vmid} = $d;
2851     }
2852
2853     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2854     foreach my $dev (keys %$netdev) {
2855         next if $dev !~ m/^tap([1-9]\d*)i/;
2856         my $vmid = $1;
2857         my $d = $res->{$vmid};
2858         next if !$d;
2859
2860         $d->{netout} += $netdev->{$dev}->{receive};
2861         $d->{netin} += $netdev->{$dev}->{transmit};
2862
2863         if ($full) {
2864             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2865             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2866         }
2867
2868     }
2869
2870     my $ctime = gettimeofday;
2871
2872     foreach my $vmid (keys %$list) {
2873
2874         my $d = $res->{$vmid};
2875         my $pid = $d->{pid};
2876         next if !$pid;
2877
2878         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2879         next if !$pstat; # not running
2880
2881         my $used = $pstat->{utime} + $pstat->{stime};
2882
2883         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2884
2885         if ($pstat->{vsize}) {
2886             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2887         }
2888
2889         my $old = $last_proc_pid_stat->{$pid};
2890         if (!$old) {
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => 0,
2895             };
2896             next;
2897         }
2898
2899         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2900
2901         if ($dtime > 1000) {
2902             my $dutime = $used -  $old->{used};
2903
2904             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2905             $last_proc_pid_stat->{$pid} = {
2906                 time => $ctime,
2907                 used => $used,
2908                 cpu => $d->{cpu},
2909             };
2910         } else {
2911             $d->{cpu} = $old->{cpu};
2912         }
2913     }
2914
2915     return $res if !$full;
2916
2917     my $qmpclient = PVE::QMPClient->new();
2918
2919     my $ballooncb = sub {
2920         my ($vmid, $resp) = @_;
2921
2922         my $info = $resp->{'return'};
2923         return if !$info->{max_mem};
2924
2925         my $d = $res->{$vmid};
2926
2927         # use memory assigned to VM
2928         $d->{maxmem} = $info->{max_mem};
2929         $d->{balloon} = $info->{actual};
2930
2931         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2932             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2933             $d->{freemem} = $info->{free_mem};
2934         }
2935
2936         $d->{ballooninfo} = $info;
2937     };
2938
2939     my $blockstatscb = sub {
2940         my ($vmid, $resp) = @_;
2941         my $data = $resp->{'return'} || [];
2942         my $totalrdbytes = 0;
2943         my $totalwrbytes = 0;
2944
2945         for my $blockstat (@$data) {
2946             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2947             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2948
2949             $blockstat->{device} =~ s/drive-//;
2950             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2951         }
2952         $res->{$vmid}->{diskread} = $totalrdbytes;
2953         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2954     };
2955
2956     my $machinecb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} || [];
2959
2960         $res->{$vmid}->{'running-machine'} =
2961             PVE::QemuServer::Machine::current_from_query_machines($data);
2962     };
2963
2964     my $versioncb = sub {
2965         my ($vmid, $resp) = @_;
2966         my $data = $resp->{'return'} // {};
2967         my $version = 'unknown';
2968
2969         if (my $v = $data->{qemu}) {
2970             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2971         }
2972
2973         $res->{$vmid}->{'running-qemu'} = $version;
2974     };
2975
2976     my $statuscb = sub {
2977         my ($vmid, $resp) = @_;
2978
2979         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2980         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2981         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2982         # this fails if ballon driver is not loaded, so this must be
2983         # the last commnand (following command are aborted if this fails).
2984         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2985
2986         my $status = 'unknown';
2987         if (!defined($status = $resp->{'return'}->{status})) {
2988             warn "unable to get VM status\n";
2989             return;
2990         }
2991
2992         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2993     };
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; # not running
2998         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2999     }
3000
3001     $qmpclient->queue_execute(undef, 2);
3002
3003     foreach my $vmid (keys %$list) {
3004         next if $opt_vmid && ($vmid ne $opt_vmid);
3005         next if !$res->{$vmid}->{pid}; #not running
3006
3007         # we can't use the $qmpclient since it might have already aborted on
3008         # 'query-balloon', but this might also fail for older versions...
3009         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3010         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3011     }
3012
3013     foreach my $vmid (keys %$list) {
3014         next if $opt_vmid && ($vmid ne $opt_vmid);
3015         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3016     }
3017
3018     return $res;
3019 }
3020
3021 sub conf_has_serial {
3022     my ($conf) = @_;
3023
3024     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3025         if ($conf->{"serial$i"}) {
3026             return 1;
3027         }
3028     }
3029
3030     return 0;
3031 }
3032
3033 sub conf_has_audio {
3034     my ($conf, $id) = @_;
3035
3036     $id //= 0;
3037     my $audio = $conf->{"audio$id"};
3038     return if !defined($audio);
3039
3040     my $audioproperties = parse_property_string($audio_fmt, $audio);
3041     my $audiodriver = $audioproperties->{driver} // 'spice';
3042
3043     return {
3044         dev => $audioproperties->{device},
3045         dev_id => "audiodev$id",
3046         backend => $audiodriver,
3047         backend_id => "$audiodriver-backend${id}",
3048     };
3049 }
3050
3051 sub audio_devs {
3052     my ($audio, $audiopciaddr, $machine_version) = @_;
3053
3054     my $devs = [];
3055
3056     my $id = $audio->{dev_id};
3057     my $audiodev = "";
3058     if (min_version($machine_version, 4, 2)) {
3059         $audiodev = ",audiodev=$audio->{backend_id}";
3060     }
3061
3062     if ($audio->{dev} eq 'AC97') {
3063         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3064     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3065         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3066         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3067         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3068     } else {
3069         die "unkown audio device '$audio->{dev}', implement me!";
3070     }
3071
3072     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3073
3074     return $devs;
3075 }
3076
3077 sub get_tpm_paths {
3078     my ($vmid) = @_;
3079     return {
3080         socket => "/var/run/qemu-server/$vmid.swtpm",
3081         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3082     };
3083 }
3084
3085 sub add_tpm_device {
3086     my ($vmid, $devices, $conf) = @_;
3087
3088     return if !$conf->{tpmstate0};
3089
3090     my $paths = get_tpm_paths($vmid);
3091
3092     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3093     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3094     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3095 }
3096
3097 sub start_swtpm {
3098     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3099
3100     return if !$tpmdrive;
3101
3102     my $state;
3103     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3104     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3105     if ($storeid) {
3106         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3107     } else {
3108         $state = $tpm->{file};
3109     }
3110
3111     my $paths = get_tpm_paths($vmid);
3112
3113     # during migration, we will get state from remote
3114     #
3115     if (!$migration) {
3116         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3117         my $setup_cmd = [
3118             "swtpm_setup",
3119             "--tpmstate",
3120             "file://$state",
3121             "--createek",
3122             "--create-ek-cert",
3123             "--create-platform-cert",
3124             "--lock-nvram",
3125             "--config",
3126             "/etc/swtpm_setup.conf", # do not use XDG configs
3127             "--runas",
3128             "0", # force creation as root, error if not possible
3129             "--not-overwrite", # ignore existing state, do not modify
3130         ];
3131
3132         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3133         # TPM 2.0 supports ECC crypto, use if possible
3134         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3135
3136         run_command($setup_cmd, outfunc => sub {
3137             print "swtpm_setup: $1\n";
3138         });
3139     }
3140
3141     my $emulator_cmd = [
3142         "swtpm",
3143         "socket",
3144         "--tpmstate",
3145         "backend-uri=file://$state,mode=0600",
3146         "--ctrl",
3147         "type=unixio,path=$paths->{socket},mode=0600",
3148         "--pid",
3149         "file=$paths->{pid}",
3150         "--terminate", # terminate on QEMU disconnect
3151         "--daemon",
3152     ];
3153     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3154     run_command($emulator_cmd, outfunc => sub { print $1; });
3155
3156     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3157     while (! -e $paths->{pid}) {
3158         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3159         usleep(50_000);
3160     }
3161
3162     # return untainted PID of swtpm daemon so it can be killed on error
3163     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3164     return $1;
3165 }
3166
3167 sub vga_conf_has_spice {
3168     my ($vga) = @_;
3169
3170     my $vgaconf = parse_vga($vga);
3171     my $vgatype = $vgaconf->{type};
3172     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3173
3174     return $1 || 1;
3175 }
3176
3177 sub is_native($) {
3178     my ($arch) = @_;
3179     return get_host_arch() eq $arch;
3180 }
3181
3182 sub get_vm_arch {
3183     my ($conf) = @_;
3184     return $conf->{arch} // get_host_arch();
3185 }
3186
3187 my $default_machines = {
3188     x86_64 => 'pc',
3189     aarch64 => 'virt',
3190 };
3191
3192 sub get_installed_machine_version {
3193     my ($kvmversion) = @_;
3194     $kvmversion = kvm_user_version() if !defined($kvmversion);
3195     $kvmversion =~ m/^(\d+\.\d+)/;
3196     return $1;
3197 }
3198
3199 sub windows_get_pinned_machine_version {
3200     my ($machine, $base_version, $kvmversion) = @_;
3201
3202     my $pin_version = $base_version;
3203     if (!defined($base_version) ||
3204         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3205     ) {
3206         $pin_version = get_installed_machine_version($kvmversion);
3207     }
3208     if (!$machine || $machine eq 'pc') {
3209         $machine = "pc-i440fx-$pin_version";
3210     } elsif ($machine eq 'q35') {
3211         $machine = "pc-q35-$pin_version";
3212     } elsif ($machine eq 'virt') {
3213         $machine = "virt-$pin_version";
3214     } else {
3215         warn "unknown machine type '$machine', not touching that!\n";
3216     }
3217
3218     return $machine;
3219 }
3220
3221 sub get_vm_machine {
3222     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3223
3224     my $machine = $forcemachine || $conf->{machine};
3225
3226     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3227         $kvmversion //= kvm_user_version();
3228         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3229         # layout which confuses windows quite a bit and may result in various regressions..
3230         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3231         if (windows_version($conf->{ostype})) {
3232             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3233         }
3234         $arch //= 'x86_64';
3235         $machine ||= $default_machines->{$arch};
3236         if ($add_pve_version) {
3237             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3238             $machine .= "+pve$pvever";
3239         }
3240     }
3241
3242     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3243         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3244         $machine = $1 if $is_pxe;
3245
3246         # for version-pinned machines that do not include a pve-version (e.g.
3247         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3248         $machine .= '+pve0';
3249
3250         $machine .= '.pxe' if $is_pxe;
3251     }
3252
3253     return $machine;
3254 }
3255
3256 sub get_ovmf_files($$$) {
3257     my ($arch, $efidisk, $smm) = @_;
3258
3259     my $types = $OVMF->{$arch}
3260         or die "no OVMF images known for architecture '$arch'\n";
3261
3262     my $type = 'default';
3263     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3264         $type = $smm ? "4m" : "4m-no-smm";
3265         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3266     }
3267
3268     return $types->{$type}->@*;
3269 }
3270
3271 my $Arch2Qemu = {
3272     aarch64 => '/usr/bin/qemu-system-aarch64',
3273     x86_64 => '/usr/bin/qemu-system-x86_64',
3274 };
3275 sub get_command_for_arch($) {
3276     my ($arch) = @_;
3277     return '/usr/bin/kvm' if is_native($arch);
3278
3279     my $cmd = $Arch2Qemu->{$arch}
3280         or die "don't know how to emulate architecture '$arch'\n";
3281     return $cmd;
3282 }
3283
3284 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3285 # to use in a QEMU command line (-cpu element), first array_intersect the result
3286 # of query_supported_ with query_understood_. This is necessary because:
3287 #
3288 # a) query_understood_ returns flags the host cannot use and
3289 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3290 #    flags, but CPU settings - with most of them being flags. Those settings
3291 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3292 #
3293 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3294 # expensive. If you need the value returned from this, you can get it much
3295 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3296 # $accel being 'kvm' or 'tcg'.
3297 #
3298 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3299 # changes, automatically populating pmxcfs.
3300 #
3301 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3302 # since kvm and tcg machines support different flags
3303 #
3304 sub query_supported_cpu_flags {
3305     my ($arch) = @_;
3306
3307     $arch //= get_host_arch();
3308     my $default_machine = $default_machines->{$arch};
3309
3310     my $flags = {};
3311
3312     # FIXME: Once this is merged, the code below should work for ARM as well:
3313     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3314     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3315         $arch eq "aarch64";
3316
3317     my $kvm_supported = defined(kvm_version());
3318     my $qemu_cmd = get_command_for_arch($arch);
3319     my $fakevmid = -1;
3320     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3321
3322     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3323     my $query_supported_run_qemu = sub {
3324         my ($kvm) = @_;
3325
3326         my $flags = {};
3327         my $cmd = [
3328             $qemu_cmd,
3329             '-machine', $default_machine,
3330             '-display', 'none',
3331             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3332             '-mon', 'chardev=qmp,mode=control',
3333             '-pidfile', $pidfile,
3334             '-S', '-daemonize'
3335         ];
3336
3337         if (!$kvm) {
3338             push @$cmd, '-accel', 'tcg';
3339         }
3340
3341         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3342         die "QEMU flag querying VM exited with code " . $rc if $rc;
3343
3344         eval {
3345             my $cmd_result = mon_cmd(
3346                 $fakevmid,
3347                 'query-cpu-model-expansion',
3348                 type => 'full',
3349                 model => { name => 'host' }
3350             );
3351
3352             my $props = $cmd_result->{model}->{props};
3353             foreach my $prop (keys %$props) {
3354                 next if $props->{$prop} ne '1';
3355                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3356                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3357                 # We only keep those with underscores, to match /proc/cpuinfo
3358                 $prop =~ s/\.|-/_/g;
3359                 $flags->{$prop} = 1;
3360             }
3361         };
3362         my $err = $@;
3363
3364         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3365         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3366
3367         die $err if $err;
3368
3369         return [ sort keys %$flags ];
3370     };
3371
3372     # We need to query QEMU twice, since KVM and TCG have different supported flags
3373     PVE::QemuConfig->lock_config($fakevmid, sub {
3374         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3375         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3376
3377         if ($kvm_supported) {
3378             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3379             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3380         }
3381     });
3382
3383     return $flags;
3384 }
3385
3386 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3387 my $understood_cpu_flag_dir = "/usr/share/kvm";
3388 sub query_understood_cpu_flags {
3389     my $arch = get_host_arch();
3390     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3391
3392     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3393         if ! -e $filepath;
3394
3395     my $raw = file_get_contents($filepath);
3396     $raw =~ s/^\s+|\s+$//g;
3397     my @flags = split(/\s+/, $raw);
3398
3399     return \@flags;
3400 }
3401
3402 my sub get_cpuunits {
3403     my ($conf) = @_;
3404     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3405 }
3406
3407 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3408 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3409 my sub should_disable_smm {
3410     my ($conf, $vga) = @_;
3411
3412     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3413         $vga->{type} && $vga->{type} =~ m/^serial\d+$/;
3414 }
3415
3416 sub config_to_command {
3417     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3418         $pbs_backing) = @_;
3419
3420     my $cmd = [];
3421     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3422     my $devices = [];
3423     my $bridges = {};
3424     my $ostype = $conf->{ostype};
3425     my $winversion = windows_version($ostype);
3426     my $kvm = $conf->{kvm};
3427     my $nodename = nodename();
3428
3429     my $arch = get_vm_arch($conf);
3430     my $kvm_binary = get_command_for_arch($arch);
3431     my $kvmver = kvm_user_version($kvm_binary);
3432
3433     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3434         $kvmver //= "undefined";
3435         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3436     }
3437
3438     my $add_pve_version = min_version($kvmver, 4, 1);
3439
3440     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3441     my $machine_version = extract_version($machine_type, $kvmver);
3442     $kvm //= 1 if is_native($arch);
3443
3444     $machine_version =~ m/(\d+)\.(\d+)/;
3445     my ($machine_major, $machine_minor) = ($1, $2);
3446
3447     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3448         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3449     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3450         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3451             ." please upgrade node '$nodename'\n"
3452     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3453         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3454         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3455             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3456             ." node '$nodename'\n";
3457     }
3458
3459     # if a specific +pve version is required for a feature, use $version_guard
3460     # instead of min_version to allow machines to be run with the minimum
3461     # required version
3462     my $required_pve_version = 0;
3463     my $version_guard = sub {
3464         my ($major, $minor, $pve) = @_;
3465         return 0 if !min_version($machine_version, $major, $minor, $pve);
3466         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3467         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3468         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3469         return 1;
3470     };
3471
3472     if ($kvm && !defined kvm_version()) {
3473         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3474             ." or enable in BIOS.\n";
3475     }
3476
3477     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3478     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3479     my $use_old_bios_files = undef;
3480     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3481
3482     my $cpuunits = get_cpuunits($conf);
3483
3484     push @$cmd, $kvm_binary;
3485
3486     push @$cmd, '-id', $vmid;
3487
3488     my $vmname = $conf->{name} || "vm$vmid";
3489
3490     push @$cmd, '-name', $vmname;
3491
3492     push @$cmd, '-no-shutdown';
3493
3494     my $use_virtio = 0;
3495
3496     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3497     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3498     push @$cmd, '-mon', "chardev=qmp,mode=control";
3499
3500     if (min_version($machine_version, 2, 12)) {
3501         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3502         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3503     }
3504
3505     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3506
3507     push @$cmd, '-daemonize';
3508
3509     if ($conf->{smbios1}) {
3510         my $smbios_conf = parse_smbios1($conf->{smbios1});
3511         if ($smbios_conf->{base64}) {
3512             # Do not pass base64 flag to qemu
3513             delete $smbios_conf->{base64};
3514             my $smbios_string = "";
3515             foreach my $key (keys %$smbios_conf) {
3516                 my $value;
3517                 if ($key eq "uuid") {
3518                     $value = $smbios_conf->{uuid}
3519                 } else {
3520                     $value = decode_base64($smbios_conf->{$key});
3521                 }
3522                 # qemu accepts any binary data, only commas need escaping by double comma
3523                 $value =~ s/,/,,/g;
3524                 $smbios_string .= "," . $key . "=" . $value if $value;
3525             }
3526             push @$cmd, '-smbios', "type=1" . $smbios_string;
3527         } else {
3528             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3529         }
3530     }
3531
3532     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3533         my $d;
3534         if (my $efidisk = $conf->{efidisk0}) {
3535             $d = parse_drive('efidisk0', $efidisk);
3536         }
3537
3538         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3540
3541         my ($path, $format);
3542         my $read_only_str = '';
3543         if ($d) {
3544             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3545             $format = $d->{format};
3546             if ($storeid) {
3547                 $path = PVE::Storage::path($storecfg, $d->{file});
3548                 if (!defined($format)) {
3549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3550                     $format = qemu_img_format($scfg, $volname);
3551                 }
3552             } else {
3553                 $path = $d->{file};
3554                 die "efidisk format must be specified\n"
3555                     if !defined($format);
3556             }
3557
3558             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3559         } else {
3560             warn "no efidisk configured! Using temporary efivars disk.\n";
3561             $path = "/tmp/$vmid-ovmf.fd";
3562             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3563             $format = 'raw';
3564         }
3565
3566         my $size_str = "";
3567
3568         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3569             $size_str = ",size=" . (-s $ovmf_vars);
3570         }
3571
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         my $cache = "";
3574         if ($path =~ m/^rbd:/) {
3575                 $cache = ',cache=writeback';
3576                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3577         }
3578
3579         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3580         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3581     }
3582
3583     if ($q35) { # tell QEMU to load q35 config early
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3593         push @$cmd, $fixups->@*;
3594     }
3595
3596     if ($conf->{vmgenid}) {
3597         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3598     }
3599
3600     # add usb controllers
3601     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3602         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3603     push @$devices, @usbcontrollers if @usbcontrollers;
3604     my $vga = parse_vga($conf->{vga});
3605
3606     my $qxlnum = vga_conf_has_spice($conf->{vga});
3607     $vga->{type} = 'qxl' if $qxlnum;
3608
3609     if (!$vga->{type}) {
3610         if ($arch eq 'aarch64') {
3611             $vga->{type} = 'virtio';
3612         } elsif (min_version($machine_version, 2, 9)) {
3613             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3614         } else {
3615             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3616         }
3617     }
3618
3619     # enable absolute mouse coordinates (needed by vnc)
3620     my $tablet = $conf->{tablet};
3621     if (!defined($tablet)) {
3622         $tablet = $defaults->{tablet};
3623         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3624         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3625     }
3626
3627     if ($tablet) {
3628         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3629         my $kbd = print_keyboarddevice_full($conf, $arch);
3630         push @$devices, '-device', $kbd if defined($kbd);
3631     }
3632
3633     my $bootorder = device_bootorder($conf);
3634
3635     # host pci device passthrough
3636     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3637         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3638
3639     # usb devices
3640     my $usb_dev_features = {};
3641     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3642
3643     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3644         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3645     push @$devices, @usbdevices if @usbdevices;
3646
3647     # serial devices
3648     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3649         my $path = $conf->{"serial$i"} or next;
3650         if ($path eq 'socket') {
3651             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3652             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3653             # On aarch64, serial0 is the UART device. Qemu only allows
3654             # connecting UART devices via the '-serial' command line, as
3655             # the device has a fixed slot on the hardware...
3656             if ($arch eq 'aarch64' && $i == 0) {
3657                 push @$devices, '-serial', "chardev:serial$i";
3658             } else {
3659                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3660             }
3661         } else {
3662             die "no such serial device\n" if ! -c $path;
3663             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3664             push @$devices, '-device', "isa-serial,chardev=serial$i";
3665         }
3666     }
3667
3668     # parallel devices
3669     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3670         if (my $path = $conf->{"parallel$i"}) {
3671             die "no such parallel device\n" if ! -c $path;
3672             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3673             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3674             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3675         }
3676     }
3677
3678     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3679         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3680         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3681         push @$devices, @$audio_devs;
3682     }
3683
3684     add_tpm_device($vmid, $devices, $conf);
3685
3686     my $sockets = 1;
3687     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3688     $sockets = $conf->{sockets} if  $conf->{sockets};
3689
3690     my $cores = $conf->{cores} || 1;
3691
3692     my $maxcpus = $sockets * $cores;
3693
3694     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3695
3696     my $allowed_vcpus = $cpuinfo->{cpus};
3697
3698     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3699
3700     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3701         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702         for (my $i = 2; $i <= $vcpus; $i++)  {
3703             my $cpustr = print_cpu_device($conf,$i);
3704             push @$cmd, '-device', $cpustr;
3705         }
3706
3707     } else {
3708
3709         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3710     }
3711     push @$cmd, '-nodefaults';
3712
3713     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3714
3715     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3716
3717     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3718
3719     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3720         push @$devices, '-device', print_vga_device(
3721             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3722         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3723         push @$cmd,  '-vnc', "unix:$socket,password=on";
3724     } else {
3725         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3726         push @$cmd, '-nographic';
3727     }
3728
3729     # time drift fix
3730     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3731     my $useLocaltime = $conf->{localtime};
3732
3733     if ($winversion >= 5) { # windows
3734         $useLocaltime = 1 if !defined($conf->{localtime});
3735
3736         # use time drift fix when acpi is enabled
3737         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3738             $tdf = 1 if !defined($conf->{tdf});
3739         }
3740     }
3741
3742     if ($winversion >= 6) {
3743         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3744         push @$cmd, '-no-hpet';
3745     }
3746
3747     push @$rtcFlags, 'driftfix=slew' if $tdf;
3748
3749     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3750         push @$rtcFlags, "base=$conf->{startdate}";
3751     } elsif ($useLocaltime) {
3752         push @$rtcFlags, 'base=localtime';
3753     }
3754
3755     if ($forcecpu) {
3756         push @$cmd, '-cpu', $forcecpu;
3757     } else {
3758         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3759     }
3760
3761     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3762
3763     push @$cmd, '-S' if $conf->{freeze};
3764
3765     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3766
3767     my $guest_agent = parse_guest_agent($conf);
3768
3769     if ($guest_agent->{enabled}) {
3770         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3771         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3772
3773         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3774             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3775             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3776             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3777         } elsif ($guest_agent->{type} eq 'isa') {
3778             push @$devices, '-device', "isa-serial,chardev=qga0";
3779         }
3780     }
3781
3782     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3783     if ($rng && $version_guard->(4, 1, 2)) {
3784         check_rng_source($rng->{source});
3785
3786         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3787         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3788         my $limiter_str = "";
3789         if ($max_bytes) {
3790             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3791         }
3792
3793         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3794         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3795         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3796     }
3797
3798     my $spice_port;
3799
3800     if ($qxlnum) {
3801         if ($qxlnum > 1) {
3802             if ($winversion){
3803                 for (my $i = 1; $i < $qxlnum; $i++){
3804                     push @$devices, '-device', print_vga_device(
3805                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3806                 }
3807             } else {
3808                 # assume other OS works like Linux
3809                 my ($ram, $vram) = ("134217728", "67108864");
3810                 if ($vga->{memory}) {
3811                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3812                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3813                 }
3814                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3815                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3816             }
3817         }
3818
3819         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3820
3821         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3822         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3823         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3824
3825         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3826         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3827         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3828
3829         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3830         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3831
3832         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3833         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3834         if ($spice_enhancement->{foldersharing}) {
3835             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3836             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3837         }
3838
3839         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3840         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3841             if $spice_enhancement->{videostreaming};
3842
3843         push @$devices, '-spice', "$spice_opts";
3844     }
3845
3846     # enable balloon by default, unless explicitly disabled
3847     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3848         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3849         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3850     }
3851
3852     if ($conf->{watchdog}) {
3853         my $wdopts = parse_watchdog($conf->{watchdog});
3854         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3855         my $watchdog = $wdopts->{model} || 'i6300esb';
3856         push @$devices, '-device', "$watchdog$pciaddr";
3857         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3858     }
3859
3860     my $vollist = [];
3861     my $scsicontroller = {};
3862     my $ahcicontroller = {};
3863     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3864
3865     # Add iscsi initiator name if available
3866     if (my $initiator = get_initiator_name()) {
3867         push @$devices, '-iscsi', "initiator-name=$initiator";
3868     }
3869
3870     PVE::QemuConfig->foreach_volume($conf, sub {
3871         my ($ds, $drive) = @_;
3872
3873         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3874             check_volume_storage_type($storecfg, $drive->{file});
3875             push @$vollist, $drive->{file};
3876         }
3877
3878         # ignore efidisk here, already added in bios/fw handling code above
3879         return if $drive->{interface} eq 'efidisk';
3880         # similar for TPM
3881         return if $drive->{interface} eq 'tpmstate';
3882
3883         $use_virtio = 1 if $ds =~ m/^virtio/;
3884
3885         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3886
3887         if ($drive->{interface} eq 'virtio'){
3888            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3889         }
3890
3891         if ($drive->{interface} eq 'scsi') {
3892
3893             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3894
3895             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3896                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3897
3898             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3899             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3900
3901             my $iothread = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3903                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3904                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3905             } elsif ($drive->{iothread}) {
3906                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3907             }
3908
3909             my $queues = '';
3910             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3911                 $queues = ",num_queues=$drive->{queues}";
3912             }
3913
3914             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3915                 if !$scsicontroller->{$controller};
3916             $scsicontroller->{$controller}=1;
3917         }
3918
3919         if ($drive->{interface} eq 'sata') {
3920             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3921             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3922             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3923                 if !$ahcicontroller->{$controller};
3924             $ahcicontroller->{$controller}=1;
3925         }
3926
3927         my $pbs_conf = $pbs_backing->{$ds};
3928         my $pbs_name = undef;
3929         if ($pbs_conf) {
3930             $pbs_name = "drive-$ds-pbs";
3931             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3932         }
3933
3934         my $drive_cmd = print_drive_commandline_full(
3935             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3936
3937         # extra protection for templates, but SATA and IDE don't support it..
3938         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3939
3940         push @$devices, '-drive',$drive_cmd;
3941         push @$devices, '-device', print_drivedevice_full(
3942             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3943     });
3944
3945     for (my $i = 0; $i < $MAX_NETS; $i++) {
3946         my $netname = "net$i";
3947
3948         next if !$conf->{$netname};
3949         my $d = parse_net($conf->{$netname});
3950         next if !$d;
3951
3952         $use_virtio = 1 if $d->{model} eq 'virtio';
3953
3954         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3955
3956         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3957         push @$devices, '-netdev', $netdevfull;
3958
3959         my $netdevicefull = print_netdevice_full(
3960             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3961
3962         push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3980             .",size=$ivshmem->{size}M";
3981     }
3982
3983     # pci.4 is nested in pci.1
3984     $bridges->{1} = 1 if $bridges->{4};
3985
3986     if (!$q35) { # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992     }
3993
3994     for my $k (sort {$b cmp $a} keys %$bridges) {
3995         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3996
3997         my $k_name = $k;
3998         if ($k == 2 && $legacy_igd) {
3999             $k_name = "$k-igd";
4000         }
4001         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4002         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4003
4004         if ($q35) { # add after -readconfig pve-q35.cfg
4005             splice @$devices, 2, 0, '-device', $devstr;
4006         } else {
4007             unshift @$devices, '-device', $devstr if $k > 0;
4008         }
4009     }
4010
4011     if (!$kvm) {
4012         push @$machineFlags, 'accel=tcg';
4013     }
4014
4015     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4016
4017     my $machine_type_min = $machine_type;
4018     if ($add_pve_version) {
4019         $machine_type_min =~ s/\+pve\d+$//;
4020         $machine_type_min .= "+pve$required_pve_version";
4021     }
4022     push @$machineFlags, "type=${machine_type_min}";
4023
4024     push @$cmd, @$devices;
4025     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4027     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4028
4029     if (my $vmstate = $conf->{vmstate}) {
4030         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4031         push @$vollist, $vmstate;
4032         push @$cmd, '-loadstate', $statepath;
4033         print "activating and using '$vmstate' as vmstate\n";
4034     }
4035
4036     if (PVE::QemuConfig->is_template($conf)) {
4037         # needed to workaround base volumes being read-only
4038         push @$cmd, '-snapshot';
4039     }
4040
4041     # add custom args
4042     if ($conf->{args}) {
4043         my $aa = PVE::Tools::split_args($conf->{args});
4044         push @$cmd, @$aa;
4045     }
4046
4047     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4048 }
4049
4050 sub check_rng_source {
4051     my ($source) = @_;
4052
4053     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4054     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4055         if ! -e $source;
4056
4057     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4058     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4059         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4060         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4061         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4062             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4063             ." to the host.\n";
4064     }
4065 }
4066
4067 sub spice_port {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-spice');
4071
4072     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4073 }
4074
4075 sub vm_devices_list {
4076     my ($vmid) = @_;
4077
4078     my $res = mon_cmd($vmid, 'query-pci');
4079     my $devices_to_check = [];
4080     my $devices = {};
4081     foreach my $pcibus (@$res) {
4082         push @$devices_to_check, @{$pcibus->{devices}},
4083     }
4084
4085     while (@$devices_to_check) {
4086         my $to_check = [];
4087         for my $d (@$devices_to_check) {
4088             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4089             next if !$d->{'pci_bridge'};
4090
4091             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4092             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4093         }
4094         $devices_to_check = $to_check;
4095     }
4096
4097     my $resblock = mon_cmd($vmid, 'query-block');
4098     foreach my $block (@$resblock) {
4099         if($block->{device} =~ m/^drive-(\S+)/){
4100                 $devices->{$1} = 1;
4101         }
4102     }
4103
4104     my $resmice = mon_cmd($vmid, 'query-mice');
4105     foreach my $mice (@$resmice) {
4106         if ($mice->{name} eq 'QEMU HID Tablet') {
4107             $devices->{tablet} = 1;
4108             last;
4109         }
4110     }
4111
4112     # for usb devices there is no query-usb
4113     # but we can iterate over the entries in
4114     # qom-list path=/machine/peripheral
4115     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4116     foreach my $per (@$resperipheral) {
4117         if ($per->{name} =~ m/^usb\d+$/) {
4118             $devices->{$per->{name}} = 1;
4119         }
4120     }
4121
4122     return $devices;
4123 }
4124
4125 sub vm_deviceplug {
4126     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4127
4128     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if defined($devices_list->{$deviceid});
4132
4133     # add PCI bridge if we need it for the device
4134     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4135
4136     if ($deviceid eq 'tablet') {
4137         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4138     } elsif ($deviceid eq 'keyboard') {
4139         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4140     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4141         die "usb hotplug currently not reliable\n";
4142         # since we can't reliably hot unplug all added usb devices and usb
4143         # passthrough breaks live migration we disable usb hotplugging for now
4144         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         qemu_iothread_add($vmid, $deviceid, $device);
4147
4148         qemu_driveadd($storecfg, $vmid, $device);
4149         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         eval { qemu_deviceaddverify($vmid, $deviceid); };
4153         if (my $err = $@) {
4154             eval { qemu_drivedel($vmid, $deviceid); };
4155             warn $@ if $@;
4156             die $err;
4157         }
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4177         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4178         qemu_driveadd($storecfg, $vmid, $device);
4179
4180         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4181         eval { qemu_deviceadd($vmid, $devicefull); };
4182         if (my $err = $@) {
4183             eval { qemu_drivedel($vmid, $deviceid); };
4184             warn $@ if $@;
4185             die $err;
4186         }
4187     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4188         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4189
4190         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4191         my $use_old_bios_files = undef;
4192         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4193
4194         my $netdevicefull = print_netdevice_full(
4195             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4196         qemu_deviceadd($vmid, $netdevicefull);
4197         eval {
4198             qemu_deviceaddverify($vmid, $deviceid);
4199             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4200         };
4201         if (my $err = $@) {
4202             eval { qemu_netdevdel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4207         my $bridgeid = $2;
4208         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4209         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         qemu_deviceaddverify($vmid, $deviceid);
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4228     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4229
4230     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4231         qemu_devicedel($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^usb\d+$/) {
4233         die "usb hotplug currently not reliable\n";
4234         # when unplugging usb devices this way, there may be remaining usb
4235         # controllers/hubs so we disable it for now
4236         #qemu_devicedel($vmid, $deviceid);
4237         #qemu_devicedelverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239         my $device = parse_drive($deviceid, $conf->{$deviceid});
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($vmid, $deviceid, $device);
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4249         my $device = parse_drive($deviceid, $conf->{$deviceid});
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_drivedel($vmid, $deviceid);
4253         qemu_deletescsihw($conf, $vmid, $deviceid);
4254
4255         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4256             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4257     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4258         qemu_devicedel($vmid, $deviceid);
4259         qemu_devicedelverify($vmid, $deviceid);
4260         qemu_netdevdel($vmid, $deviceid);
4261     } else {
4262         die "can't unplug device '$deviceid'\n";
4263     }
4264
4265     return 1;
4266 }
4267
4268 sub qemu_deviceadd {
4269     my ($vmid, $devicefull) = @_;
4270
4271     $devicefull = "driver=".$devicefull;
4272     my %options =  split(/[=,]/, $devicefull);
4273
4274     mon_cmd($vmid, "device_add" , %options);
4275 }
4276
4277 sub qemu_devicedel {
4278     my ($vmid, $deviceid) = @_;
4279
4280     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4281 }
4282
4283 sub qemu_iothread_add {
4284     my ($vmid, $deviceid, $device) = @_;
4285
4286     if ($device->{iothread}) {
4287         my $iothreads = vm_iothreads_list($vmid);
4288         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4289     }
4290 }
4291
4292 sub qemu_iothread_del {
4293     my ($vmid, $deviceid, $device) = @_;
4294
4295     if ($device->{iothread}) {
4296         my $iothreads = vm_iothreads_list($vmid);
4297         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4298     }
4299 }
4300
4301 sub qemu_objectadd {
4302     my ($vmid, $objectid, $qomtype) = @_;
4303
4304     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_objectdel {
4310     my ($vmid, $objectid) = @_;
4311
4312     mon_cmd($vmid, "object-del", id => $objectid);
4313
4314     return 1;
4315 }
4316
4317 sub qemu_driveadd {
4318     my ($storecfg, $vmid, $device) = @_;
4319
4320     my $kvmver = get_running_qemu_version($vmid);
4321     my $io_uring = min_version($kvmver, 6, 0);
4322     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4323     $drive =~ s/\\/\\\\/g;
4324     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4325
4326     # If the command succeeds qemu prints: "OK"
4327     return 1 if $ret =~ m/OK/s;
4328
4329     die "adding drive failed: $ret\n";
4330 }
4331
4332 sub qemu_drivedel {
4333     my ($vmid, $deviceid) = @_;
4334
4335     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4336     $ret =~ s/^\s+//;
4337
4338     return 1 if $ret eq "";
4339
4340     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4341     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4342
4343     die "deleting drive $deviceid failed : $ret\n";
4344 }
4345
4346 sub qemu_deviceaddverify {
4347     my ($vmid, $deviceid) = @_;
4348
4349     for (my $i = 0; $i <= 5; $i++) {
4350          my $devices_list = vm_devices_list($vmid);
4351          return 1 if defined($devices_list->{$deviceid});
4352          sleep 1;
4353     }
4354
4355     die "error on hotplug device '$deviceid'\n";
4356 }
4357
4358
4359 sub qemu_devicedelverify {
4360     my ($vmid, $deviceid) = @_;
4361
4362     # need to verify that the device is correctly removed as device_del
4363     # is async and empty return is not reliable
4364
4365     for (my $i = 0; $i <= 5; $i++) {
4366          my $devices_list = vm_devices_list($vmid);
4367          return 1 if !defined($devices_list->{$deviceid});
4368          sleep 1;
4369     }
4370
4371     die "error on hot-unplugging device '$deviceid'\n";
4372 }
4373
4374 sub qemu_findorcreatescsihw {
4375     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4376
4377     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4378
4379     my $scsihwid="$controller_prefix$controller";
4380     my $devices_list = vm_devices_list($vmid);
4381
4382     if (!defined($devices_list->{$scsihwid})) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4384     }
4385
4386     return 1;
4387 }
4388
4389 sub qemu_deletescsihw {
4390     my ($conf, $vmid, $opt) = @_;
4391
4392     my $device = parse_drive($opt, $conf->{$opt});
4393
4394     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4395         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4396         return 1;
4397     }
4398
4399     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4400
4401     my $devices_list = vm_devices_list($vmid);
4402     foreach my $opt (keys %{$devices_list}) {
4403         if (is_valid_drivename($opt)) {
4404             my $drive = parse_drive($opt, $conf->{$opt});
4405             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4406                 return 1;
4407             }
4408         }
4409     }
4410
4411     my $scsihwid="scsihw$controller";
4412
4413     vm_deviceunplug($vmid, $conf, $scsihwid);
4414
4415     return 1;
4416 }
4417
4418 sub qemu_add_pci_bridge {
4419     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4420
4421     my $bridges = {};
4422
4423     my $bridgeid;
4424
4425     print_pci_addr($device, $bridges, $arch, $machine_type);
4426
4427     while (my ($k, $v) = each %$bridges) {
4428         $bridgeid = $k;
4429     }
4430     return 1 if !defined($bridgeid) || $bridgeid < 1;
4431
4432     my $bridge = "pci.$bridgeid";
4433     my $devices_list = vm_devices_list($vmid);
4434
4435     if (!defined($devices_list->{$bridge})) {
4436         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4437     }
4438
4439     return 1;
4440 }
4441
4442 sub qemu_set_link_status {
4443     my ($vmid, $device, $up) = @_;
4444
4445     mon_cmd($vmid, "set_link", name => $device,
4446                up => $up ? JSON::true : JSON::false);
4447 }
4448
4449 sub qemu_netdevadd {
4450     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4451
4452     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4453     my %options =  split(/[=,]/, $netdev);
4454
4455     if (defined(my $vhost = $options{vhost})) {
4456         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4457     }
4458
4459     if (defined(my $queues = $options{queues})) {
4460         $options{queues} = $queues + 0;
4461     }
4462
4463     mon_cmd($vmid, "netdev_add",  %options);
4464     return 1;
4465 }
4466
4467 sub qemu_netdevdel {
4468     my ($vmid, $deviceid) = @_;
4469
4470     mon_cmd($vmid, "netdev_del", id => $deviceid);
4471 }
4472
4473 sub qemu_usb_hotplug {
4474     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4475
4476     return if !$device;
4477
4478     # remove the old one first
4479     vm_deviceunplug($vmid, $conf, $deviceid);
4480
4481     # check if xhci controller is necessary and available
4482     if ($device->{usb3}) {
4483
4484         my $devicelist = vm_devices_list($vmid);
4485
4486         if (!$devicelist->{xhci}) {
4487             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4488             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4489         }
4490     }
4491     my $d = parse_usb_device($device->{host});
4492     $d->{usb3} = $device->{usb3};
4493
4494     # add the new one
4495     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4496 }
4497
4498 sub qemu_cpu_hotplug {
4499     my ($vmid, $conf, $vcpus) = @_;
4500
4501     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4502
4503     my $sockets = 1;
4504     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4505     $sockets = $conf->{sockets} if  $conf->{sockets};
4506     my $cores = $conf->{cores} || 1;
4507     my $maxcpus = $sockets * $cores;
4508
4509     $vcpus = $maxcpus if !$vcpus;
4510
4511     die "you can't add more vcpus than maxcpus\n"
4512         if $vcpus > $maxcpus;
4513
4514     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4515
4516     if ($vcpus < $currentvcpus) {
4517
4518         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4519
4520             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4521                 qemu_devicedel($vmid, "cpu$i");
4522                 my $retry = 0;
4523                 my $currentrunningvcpus = undef;
4524                 while (1) {
4525                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4526                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4527                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4528                     $retry++;
4529                     sleep 1;
4530                 }
4531                 #update conf after each succesfull cpu unplug
4532                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4533                 PVE::QemuConfig->write_config($vmid, $conf);
4534             }
4535         } else {
4536             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4537         }
4538
4539         return;
4540     }
4541
4542     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4543     die "vcpus in running vm does not match its configuration\n"
4544         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4545
4546     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4549             my $cpustr = print_cpu_device($conf, $i);
4550             qemu_deviceadd($vmid, $cpustr);
4551
4552             my $retry = 0;
4553             my $currentrunningvcpus = undef;
4554             while (1) {
4555                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4556                 last if scalar(@{$currentrunningvcpus}) == $i;
4557                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4558                 sleep 1;
4559                 $retry++;
4560             }
4561             #update conf after each succesfull cpu hotplug
4562             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         }
4565     } else {
4566
4567         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4568             mon_cmd($vmid, "cpu-add", id => int($i));
4569         }
4570     }
4571 }
4572
4573 sub qemu_block_set_io_throttle {
4574     my ($vmid, $deviceid,
4575         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4576         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4577         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4578         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4579
4580     return if !check_running($vmid) ;
4581
4582     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4583         bps => int($bps),
4584         bps_rd => int($bps_rd),
4585         bps_wr => int($bps_wr),
4586         iops => int($iops),
4587         iops_rd => int($iops_rd),
4588         iops_wr => int($iops_wr),
4589         bps_max => int($bps_max),
4590         bps_rd_max => int($bps_rd_max),
4591         bps_wr_max => int($bps_wr_max),
4592         iops_max => int($iops_max),
4593         iops_rd_max => int($iops_rd_max),
4594         iops_wr_max => int($iops_wr_max),
4595         bps_max_length => int($bps_max_length),
4596         bps_rd_max_length => int($bps_rd_max_length),
4597         bps_wr_max_length => int($bps_wr_max_length),
4598         iops_max_length => int($iops_max_length),
4599         iops_rd_max_length => int($iops_rd_max_length),
4600         iops_wr_max_length => int($iops_wr_max_length),
4601     );
4602
4603 }
4604
4605 sub qemu_block_resize {
4606     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4607
4608     my $running = check_running($vmid);
4609
4610     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4611
4612     return if !$running;
4613
4614     my $padding = (1024 - $size % 1024) % 1024;
4615     $size = $size + $padding;
4616
4617     mon_cmd(
4618         $vmid,
4619         "block_resize",
4620         device => $deviceid,
4621         size => int($size),
4622         timeout => 60,
4623     );
4624 }
4625
4626 sub qemu_volume_snapshot {
4627     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4628
4629     my $running = check_running($vmid);
4630
4631     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4632         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4633     } else {
4634         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4635     }
4636 }
4637
4638 sub qemu_volume_snapshot_delete {
4639     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4640
4641     my $running = check_running($vmid);
4642
4643     if($running) {
4644
4645         $running = undef;
4646         my $conf = PVE::QemuConfig->load_config($vmid);
4647         PVE::QemuConfig->foreach_volume($conf, sub {
4648             my ($ds, $drive) = @_;
4649             $running = 1 if $drive->{file} eq $volid;
4650         });
4651     }
4652
4653     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4654         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4655     } else {
4656         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4657     }
4658 }
4659
4660 sub set_migration_caps {
4661     my ($vmid, $savevm) = @_;
4662
4663     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4664
4665     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4666     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4667
4668     my $cap_ref = [];
4669
4670     my $enabled_cap = {
4671         "auto-converge" => 1,
4672         "xbzrle" => 1,
4673         "x-rdma-pin-all" => 0,
4674         "zero-blocks" => 0,
4675         "compress" => 0,
4676         "dirty-bitmaps" => $dirty_bitmaps,
4677     };
4678
4679     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4680
4681     for my $supported_capability (@$supported_capabilities) {
4682         push @$cap_ref, {
4683             capability => $supported_capability->{capability},
4684             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4685         };
4686     }
4687
4688     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4689 }
4690
4691 sub foreach_volid {
4692     my ($conf, $func, @param) = @_;
4693
4694     my $volhash = {};
4695
4696     my $test_volid = sub {
4697         my ($key, $drive, $snapname) = @_;
4698
4699         my $volid = $drive->{file};
4700         return if !$volid;
4701
4702         $volhash->{$volid}->{cdrom} //= 1;
4703         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4704
4705         my $replicate = $drive->{replicate} // 1;
4706         $volhash->{$volid}->{replicate} //= 0;
4707         $volhash->{$volid}->{replicate} = 1 if $replicate;
4708
4709         $volhash->{$volid}->{shared} //= 0;
4710         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4711
4712         $volhash->{$volid}->{referenced_in_config} //= 0;
4713         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4714
4715         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4716             if defined($snapname);
4717
4718         my $size = $drive->{size};
4719         $volhash->{$volid}->{size} //= $size if $size;
4720
4721         $volhash->{$volid}->{is_vmstate} //= 0;
4722         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4723
4724         $volhash->{$volid}->{is_tpmstate} //= 0;
4725         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4726
4727         $volhash->{$volid}->{is_unused} //= 0;
4728         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4729
4730         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4731     };
4732
4733     my $include_opts = {
4734         extra_keys => ['vmstate'],
4735         include_unused => 1,
4736     };
4737
4738     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4739     foreach my $snapname (keys %{$conf->{snapshots}}) {
4740         my $snap = $conf->{snapshots}->{$snapname};
4741         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4742     }
4743
4744     foreach my $volid (keys %$volhash) {
4745         &$func($volid, $volhash->{$volid}, @param);
4746     }
4747 }
4748
4749 my $fast_plug_option = {
4750     'lock' => 1,
4751     'name' => 1,
4752     'onboot' => 1,
4753     'shares' => 1,
4754     'startup' => 1,
4755     'description' => 1,
4756     'protection' => 1,
4757     'vmstatestorage' => 1,
4758     'hookscript' => 1,
4759     'tags' => 1,
4760 };
4761
4762 # hotplug changes in [PENDING]
4763 # $selection hash can be used to only apply specified options, for
4764 # example: { cores => 1 } (only apply changed 'cores')
4765 # $errors ref is used to return error messages
4766 sub vmconfig_hotplug_pending {
4767     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4768
4769     my $defaults = load_defaults();
4770     my $arch = get_vm_arch($conf);
4771     my $machine_type = get_vm_machine($conf, undef, $arch);
4772
4773     # commit values which do not have any impact on running VM first
4774     # Note: those option cannot raise errors, we we do not care about
4775     # $selection and always apply them.
4776
4777     my $add_error = sub {
4778         my ($opt, $msg) = @_;
4779         $errors->{$opt} = "hotplug problem - $msg";
4780     };
4781
4782     my $changes = 0;
4783     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4784         if ($fast_plug_option->{$opt}) {
4785             $conf->{$opt} = $conf->{pending}->{$opt};
4786             delete $conf->{pending}->{$opt};
4787             $changes = 1;
4788         }
4789     }
4790
4791     if ($changes) {
4792         PVE::QemuConfig->write_config($vmid, $conf);
4793     }
4794
4795     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4796
4797     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4798     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4799     foreach my $opt (sort keys %$pending_delete_hash) {
4800         next if $selection && !$selection->{$opt};
4801         my $force = $pending_delete_hash->{$opt}->{force};
4802         eval {
4803             if ($opt eq 'hotplug') {
4804                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4805             } elsif ($opt eq 'tablet') {
4806                 die "skip\n" if !$hotplug_features->{usb};
4807                 if ($defaults->{tablet}) {
4808                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4810                         if $arch eq 'aarch64';
4811                 } else {
4812                     vm_deviceunplug($vmid, $conf, 'tablet');
4813                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4814                 }
4815             } elsif ($opt =~ m/^usb\d+/) {
4816                 die "skip\n";
4817                 # since we cannot reliably hot unplug usb devices we are disabling it
4818                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4819                 #vm_deviceunplug($vmid, $conf, $opt);
4820             } elsif ($opt eq 'vcpus') {
4821                 die "skip\n" if !$hotplug_features->{cpu};
4822                 qemu_cpu_hotplug($vmid, $conf, undef);
4823             } elsif ($opt eq 'balloon') {
4824                 # enable balloon device is not hotpluggable
4825                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4826                 # here we reset the ballooning value to memory
4827                 my $balloon = $conf->{memory} || $defaults->{memory};
4828                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4829             } elsif ($fast_plug_option->{$opt}) {
4830                 # do nothing
4831             } elsif ($opt =~ m/^net(\d+)$/) {
4832                 die "skip\n" if !$hotplug_features->{network};
4833                 vm_deviceunplug($vmid, $conf, $opt);
4834             } elsif (is_valid_drivename($opt)) {
4835                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4836                 vm_deviceunplug($vmid, $conf, $opt);
4837                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4838             } elsif ($opt =~ m/^memory$/) {
4839                 die "skip\n" if !$hotplug_features->{memory};
4840                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4841             } elsif ($opt eq 'cpuunits') {
4842                 $cgroup->change_cpu_shares(undef, 1024);
4843             } elsif ($opt eq 'cpulimit') {
4844                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4845             } else {
4846                 die "skip\n";
4847             }
4848         };
4849         if (my $err = $@) {
4850             &$add_error($opt, $err) if $err ne "skip\n";
4851         } else {
4852             delete $conf->{$opt};
4853             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4854         }
4855     }
4856
4857     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4858     $apply_pending_cloudinit = sub {
4859         return if $apply_pending_cloudinit_done; # once is enough
4860         $apply_pending_cloudinit_done = 1; # once is enough
4861
4862         my ($key, $value) = @_;
4863
4864         my @cloudinit_opts = keys %$confdesc_cloudinit;
4865         foreach my $opt (keys %{$conf->{pending}}) {
4866             next if !grep { $_ eq $opt } @cloudinit_opts;
4867             $conf->{$opt} = delete $conf->{pending}->{$opt};
4868         }
4869
4870         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4871         foreach my $opt (sort keys %$pending_delete_hash) {
4872             next if !grep { $_ eq $opt } @cloudinit_opts;
4873             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4874             delete $conf->{$opt};
4875         }
4876
4877         my $new_conf = { %$conf };
4878         $new_conf->{$key} = $value;
4879         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4880     };
4881
4882     foreach my $opt (keys %{$conf->{pending}}) {
4883         next if $selection && !$selection->{$opt};
4884         my $value = $conf->{pending}->{$opt};
4885         eval {
4886             if ($opt eq 'hotplug') {
4887                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4888             } elsif ($opt eq 'tablet') {
4889                 die "skip\n" if !$hotplug_features->{usb};
4890                 if ($value == 1) {
4891                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4892                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4893                         if $arch eq 'aarch64';
4894                 } elsif ($value == 0) {
4895                     vm_deviceunplug($vmid, $conf, 'tablet');
4896                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4897                 }
4898             } elsif ($opt =~ m/^usb\d+$/) {
4899                 die "skip\n";
4900                 # since we cannot reliably hot unplug usb devices we disable it for now
4901                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4902                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4903                 #die "skip\n" if !$d;
4904                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4905             } elsif ($opt eq 'vcpus') {
4906                 die "skip\n" if !$hotplug_features->{cpu};
4907                 qemu_cpu_hotplug($vmid, $conf, $value);
4908             } elsif ($opt eq 'balloon') {
4909                 # enable/disable balloning device is not hotpluggable
4910                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4911                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4912                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4913
4914                 # allow manual ballooning if shares is set to zero
4915                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4916                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4917                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4918                 }
4919             } elsif ($opt =~ m/^net(\d+)$/) {
4920                 # some changes can be done without hotplug
4921                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4922                                     $vmid, $opt, $value, $arch, $machine_type);
4923             } elsif (is_valid_drivename($opt)) {
4924                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4925                 # some changes can be done without hotplug
4926                 my $drive = parse_drive($opt, $value);
4927                 if (drive_is_cloudinit($drive)) {
4928                     &$apply_pending_cloudinit($opt, $value);
4929                 }
4930                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4931                                      $vmid, $opt, $value, $arch, $machine_type);
4932             } elsif ($opt =~ m/^memory$/) { #dimms
4933                 die "skip\n" if !$hotplug_features->{memory};
4934                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4935             } elsif ($opt eq 'cpuunits') {
4936                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4937             } elsif ($opt eq 'cpulimit') {
4938                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4939                 $cgroup->change_cpu_quota($cpulimit, 100000);
4940             } elsif ($opt eq 'agent') {
4941                 vmconfig_update_agent($conf, $opt, $value);
4942             } else {
4943                 die "skip\n";  # skip non-hot-pluggable options
4944             }
4945         };
4946         if (my $err = $@) {
4947             &$add_error($opt, $err) if $err ne "skip\n";
4948         } else {
4949             $conf->{$opt} = $value;
4950             delete $conf->{pending}->{$opt};
4951         }
4952     }
4953
4954     PVE::QemuConfig->write_config($vmid, $conf);
4955 }
4956
4957 sub try_deallocate_drive {
4958     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4959
4960     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4961         my $volid = $drive->{file};
4962         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4963             my $sid = PVE::Storage::parse_volume_id($volid);
4964             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4965
4966             # check if the disk is really unused
4967             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4968                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4969             PVE::Storage::vdisk_free($storecfg, $volid);
4970             return 1;
4971         } else {
4972             # If vm is not owner of this disk remove from config
4973             return 1;
4974         }
4975     }
4976
4977     return;
4978 }
4979
4980 sub vmconfig_delete_or_detach_drive {
4981     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4982
4983     my $drive = parse_drive($opt, $conf->{$opt});
4984
4985     my $rpcenv = PVE::RPCEnvironment::get();
4986     my $authuser = $rpcenv->get_user();
4987
4988     if ($force) {
4989         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4990         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4991     } else {
4992         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4993     }
4994 }
4995
4996
4997
4998 sub vmconfig_apply_pending {
4999     my ($vmid, $conf, $storecfg, $errors) = @_;
5000
5001     my $add_apply_error = sub {
5002         my ($opt, $msg) = @_;
5003         my $err_msg = "unable to apply pending change $opt : $msg";
5004         $errors->{$opt} = $err_msg;
5005         warn $err_msg;
5006     };
5007
5008     # cold plug
5009
5010     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5011     foreach my $opt (sort keys %$pending_delete_hash) {
5012         my $force = $pending_delete_hash->{$opt}->{force};
5013         eval {
5014             if ($opt =~ m/^unused/) {
5015                 die "internal error";
5016             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5017                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5018             }
5019         };
5020         if (my $err = $@) {
5021             $add_apply_error->($opt, $err);
5022         } else {
5023             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5024             delete $conf->{$opt};
5025         }
5026     }
5027
5028     PVE::QemuConfig->cleanup_pending($conf);
5029
5030     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5031         next if $opt eq 'delete'; # just to be sure
5032         eval {
5033             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5034                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5035             }
5036         };
5037         if (my $err = $@) {
5038             $add_apply_error->($opt, $err);
5039         } else {
5040             $conf->{$opt} = delete $conf->{pending}->{$opt};
5041         }
5042     }
5043
5044     # write all changes at once to avoid unnecessary i/o
5045     PVE::QemuConfig->write_config($vmid, $conf);
5046 }
5047
5048 sub vmconfig_update_net {
5049     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5050
5051     my $newnet = parse_net($value);
5052
5053     if ($conf->{$opt}) {
5054         my $oldnet = parse_net($conf->{$opt});
5055
5056         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5057             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5058             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5059             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5060
5061             # for non online change, we try to hot-unplug
5062             die "skip\n" if !$hotplug;
5063             vm_deviceunplug($vmid, $conf, $opt);
5064         } else {
5065
5066             die "internal error" if $opt !~ m/net(\d+)/;
5067             my $iface = "tap${vmid}i$1";
5068
5069             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5070                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5071                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5072                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5073                 PVE::Network::tap_unplug($iface);
5074
5075                 if ($have_sdn) {
5076                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5077                 } else {
5078                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5079                 }
5080             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5081                 # Rate can be applied on its own but any change above needs to
5082                 # include the rate in tap_plug since OVS resets everything.
5083                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5084             }
5085
5086             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5087                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5088             }
5089
5090             return 1;
5091         }
5092     }
5093
5094     if ($hotplug) {
5095         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5096     } else {
5097         die "skip\n";
5098     }
5099 }
5100
5101 sub vmconfig_update_agent {
5102     my ($conf, $opt, $value) = @_;
5103
5104     die "skip\n" if !$conf->{$opt};
5105
5106     my $hotplug_options = { fstrim_cloned_disks => 1 };
5107
5108     my $old_agent = parse_guest_agent($conf);
5109     my $agent = parse_guest_agent({$opt => $value});
5110
5111     for my $option (keys %$agent) { # added/changed options
5112         next if defined($hotplug_options->{$option});
5113         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5114     }
5115
5116     for my $option (keys %$old_agent) { # removed options
5117         next if defined($hotplug_options->{$option});
5118         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5119     }
5120
5121     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5122 }
5123
5124 sub vmconfig_update_disk {
5125     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5126
5127     my $drive = parse_drive($opt, $value);
5128
5129     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5130         my $media = $drive->{media} || 'disk';
5131         my $oldmedia = $old_drive->{media} || 'disk';
5132         die "unable to change media type\n" if $media ne $oldmedia;
5133
5134         if (!drive_is_cdrom($old_drive)) {
5135
5136             if ($drive->{file} ne $old_drive->{file}) {
5137
5138                 die "skip\n" if !$hotplug;
5139
5140                 # unplug and register as unused
5141                 vm_deviceunplug($vmid, $conf, $opt);
5142                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5143
5144             } else {
5145                 # update existing disk
5146
5147                 # skip non hotpluggable value
5148                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5149                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5150                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5151                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5152                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5153                     die "skip\n";
5154                 }
5155
5156                 # apply throttle
5157                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5158                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5159                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5160                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5161                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5162                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5163                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5164                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5165                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5166                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5167                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5168                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5169                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5170                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5171                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5172                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5173                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5174                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5175
5176                     qemu_block_set_io_throttle(
5177                         $vmid,"drive-$opt",
5178                         ($drive->{mbps} || 0)*1024*1024,
5179                         ($drive->{mbps_rd} || 0)*1024*1024,
5180                         ($drive->{mbps_wr} || 0)*1024*1024,
5181                          $drive->{iops} || 0,
5182                          $drive->{iops_rd} || 0,
5183                          $drive->{iops_wr} || 0,
5184                         ($drive->{mbps_max} || 0)*1024*1024,
5185                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5186                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5187                          $drive->{iops_max} || 0,
5188                          $drive->{iops_rd_max} || 0,
5189                          $drive->{iops_wr_max} || 0,
5190                          $drive->{bps_max_length} || 1,
5191                          $drive->{bps_rd_max_length} || 1,
5192                          $drive->{bps_wr_max_length} || 1,
5193                          $drive->{iops_max_length} || 1,
5194                          $drive->{iops_rd_max_length} || 1,
5195                          $drive->{iops_wr_max_length} || 1,
5196                     );
5197
5198                 }
5199
5200                 return 1;
5201             }
5202
5203         } else { # cdrom
5204
5205             if ($drive->{file} eq 'none') {
5206                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5207                 if (drive_is_cloudinit($old_drive)) {
5208                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5209                 }
5210             } else {
5211                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5212
5213                 # force eject if locked
5214                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5215
5216                 if ($path) {
5217                     mon_cmd($vmid, "blockdev-change-medium",
5218                         id => "$opt", filename => "$path");
5219                 }
5220             }
5221
5222             return 1;
5223         }
5224     }
5225
5226     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5227     # hotplug new disks
5228     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5229     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5230 }
5231
5232 # called in locked context by incoming migration
5233 sub vm_migrate_get_nbd_disks {
5234     my ($storecfg, $conf, $replicated_volumes) = @_;
5235
5236     my $local_volumes = {};
5237     PVE::QemuConfig->foreach_volume($conf, sub {
5238         my ($ds, $drive) = @_;
5239
5240         return if drive_is_cdrom($drive);
5241
5242         my $volid = $drive->{file};
5243
5244         return if !$volid;
5245
5246         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5247
5248         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5249         return if $scfg->{shared};
5250
5251         # replicated disks re-use existing state via bitmap
5252         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5253         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5254     });
5255     return $local_volumes;
5256 }
5257
5258 # called in locked context by incoming migration
5259 sub vm_migrate_alloc_nbd_disks {
5260     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5261
5262     my $format = undef;
5263
5264     my $nbd = {};
5265     foreach my $opt (sort keys %$source_volumes) {
5266         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5267
5268         if ($use_existing) {
5269             $nbd->{$opt}->{drivestr} = print_drive($drive);
5270             $nbd->{$opt}->{volid} = $volid;
5271             $nbd->{$opt}->{replicated} = 1;
5272             next;
5273         }
5274
5275         # If a remote storage is specified and the format of the original
5276         # volume is not available there, fall back to the default format.
5277         # Otherwise use the same format as the original.
5278         if (!$storagemap->{identity}) {
5279             $storeid = map_storage($storagemap, $storeid);
5280             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5281             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5282             my $fileFormat = qemu_img_format($scfg, $volname);
5283             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5284         } else {
5285             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5286             $format = qemu_img_format($scfg, $volname);
5287         }
5288
5289         my $size = $drive->{size} / 1024;
5290         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5291         my $newdrive = $drive;
5292         $newdrive->{format} = $format;
5293         $newdrive->{file} = $newvolid;
5294         my $drivestr = print_drive($newdrive);
5295         $nbd->{$opt}->{drivestr} = $drivestr;
5296         $nbd->{$opt}->{volid} = $newvolid;
5297     }
5298
5299     return $nbd;
5300 }
5301
5302 # see vm_start_nolock for parameters, additionally:
5303 # migrate_opts:
5304 #   storagemap = parsed storage map for allocating NBD disks
5305 sub vm_start {
5306     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5307
5308     return PVE::QemuConfig->lock_config($vmid, sub {
5309         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5310
5311         die "you can't start a vm if it's a template\n"
5312             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5313
5314         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5315         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5316
5317         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5318
5319         if ($has_backup_lock && $running) {
5320             # a backup is currently running, attempt to start the guest in the
5321             # existing QEMU instance
5322             return vm_resume($vmid);
5323         }
5324
5325         PVE::QemuConfig->check_lock($conf)
5326             if !($params->{skiplock} || $has_suspended_lock);
5327
5328         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5329
5330         die "VM $vmid already running\n" if $running;
5331
5332         if (my $storagemap = $migrate_opts->{storagemap}) {
5333             my $replicated = $migrate_opts->{replicated_volumes};
5334             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5335             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5336
5337             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5338                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5339             }
5340         }
5341
5342         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5343     });
5344 }
5345
5346
5347 # params:
5348 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5349 #   skiplock => 0/1, skip checking for config lock
5350 #   skiptemplate => 0/1, skip checking whether VM is template
5351 #   forcemachine => to force Qemu machine (rollback/migration)
5352 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5353 #   timeout => in seconds
5354 #   paused => start VM in paused state (backup)
5355 #   resume => resume from hibernation
5356 #   pbs-backing => {
5357 #      sata0 => {
5358 #         repository
5359 #         snapshot
5360 #         keyfile
5361 #         archive
5362 #      },
5363 #      virtio2 => ...
5364 #   }
5365 # migrate_opts:
5366 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5367 #   migratedfrom => source node
5368 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5369 #   network => CIDR of migration network
5370 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5371 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5372 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5373 sub vm_start_nolock {
5374     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5375
5376     my $statefile = $params->{statefile};
5377     my $resume = $params->{resume};
5378
5379     my $migratedfrom = $migrate_opts->{migratedfrom};
5380     my $migration_type = $migrate_opts->{type};
5381
5382     my $res = {};
5383
5384     # clean up leftover reboot request files
5385     eval { clear_reboot_request($vmid); };
5386     warn $@ if $@;
5387
5388     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5389         vmconfig_apply_pending($vmid, $conf, $storecfg);
5390         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5391     }
5392
5393     # don't regenerate the ISO if the VM is started as part of a live migration
5394     # this way we can reuse the old ISO with the correct config
5395     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5396
5397     my $defaults = load_defaults();
5398
5399     # set environment variable useful inside network script
5400     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5401
5402     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5403
5404     my $forcemachine = $params->{forcemachine};
5405     my $forcecpu = $params->{forcecpu};
5406     if ($resume) {
5407         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5408         $forcemachine = $conf->{runningmachine};
5409         $forcecpu = $conf->{runningcpu};
5410         print "Resuming suspended VM\n";
5411     }
5412
5413     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5414         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5415
5416     my $migration_ip;
5417     my $get_migration_ip = sub {
5418         my ($nodename) = @_;
5419
5420         return $migration_ip if defined($migration_ip);
5421
5422         my $cidr = $migrate_opts->{network};
5423
5424         if (!defined($cidr)) {
5425             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5426             $cidr = $dc_conf->{migration}->{network};
5427         }
5428
5429         if (defined($cidr)) {
5430             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5431
5432             die "could not get IP: no address configured on local " .
5433                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5434
5435             die "could not get IP: multiple addresses configured on local " .
5436                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5437
5438             $migration_ip = @$ips[0];
5439         }
5440
5441         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5442             if !defined($migration_ip);
5443
5444         return $migration_ip;
5445     };
5446
5447     my $migrate_uri;
5448     if ($statefile) {
5449         if ($statefile eq 'tcp') {
5450             my $localip = "localhost";
5451             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5452             my $nodename = nodename();
5453
5454             if (!defined($migration_type)) {
5455                 if (defined($datacenterconf->{migration}->{type})) {
5456                     $migration_type = $datacenterconf->{migration}->{type};
5457                 } else {
5458                     $migration_type = 'secure';
5459                 }
5460             }
5461
5462             if ($migration_type eq 'insecure') {
5463                 $localip = $get_migration_ip->($nodename);
5464                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5465             }
5466
5467             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5468             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5469             $migrate_uri = "tcp:${localip}:${migrate_port}";
5470             push @$cmd, '-incoming', $migrate_uri;
5471             push @$cmd, '-S';
5472
5473         } elsif ($statefile eq 'unix') {
5474             # should be default for secure migrations as a ssh TCP forward
5475             # tunnel is not deterministic reliable ready and fails regurarly
5476             # to set up in time, so use UNIX socket forwards
5477             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5478             unlink $socket_addr;
5479
5480             $migrate_uri = "unix:$socket_addr";
5481
5482             push @$cmd, '-incoming', $migrate_uri;
5483             push @$cmd, '-S';
5484
5485         } elsif (-e $statefile) {
5486             push @$cmd, '-loadstate', $statefile;
5487         } else {
5488             my $statepath = PVE::Storage::path($storecfg, $statefile);
5489             push @$vollist, $statefile;
5490             push @$cmd, '-loadstate', $statepath;
5491         }
5492     } elsif ($params->{paused}) {
5493         push @$cmd, '-S';
5494     }
5495
5496     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5497
5498     my $pci_devices = {}; # host pci devices
5499     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5500         my $dev = $conf->{"hostpci$i"} or next;
5501         $pci_devices->{$i} = parse_hostpci($dev);
5502     }
5503
5504     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5505     # reserve all PCI IDs before actually doing anything with them
5506     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5507
5508     eval {
5509         for my $id (sort keys %$pci_devices) {
5510             my $d = $pci_devices->{$id};
5511             for my $dev ($d->{pciid}->@*) {
5512                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5513             }
5514         }
5515     };
5516     if (my $err = $@) {
5517         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5518         warn $@ if $@;
5519         die $err;
5520     }
5521
5522     PVE::Storage::activate_volumes($storecfg, $vollist);
5523
5524     eval {
5525         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5526     };
5527     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5528     # timeout should be more than enough here...
5529     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5530
5531     my $cpuunits = get_cpuunits($conf);
5532
5533     my %run_params = (
5534         timeout => $statefile ? undef : $start_timeout,
5535         umask => 0077,
5536         noerr => 1,
5537     );
5538
5539     # when migrating, prefix QEMU output so other side can pick up any
5540     # errors that might occur and show the user
5541     if ($migratedfrom) {
5542         $run_params{quiet} = 1;
5543         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5544     }
5545
5546     my %systemd_properties = (
5547         Slice => 'qemu.slice',
5548         KillMode => 'process',
5549         SendSIGKILL => 0,
5550         TimeoutStopUSec => ULONG_MAX, # infinity
5551     );
5552
5553     if (PVE::CGroup::cgroup_mode() == 2) {
5554         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5555         $systemd_properties{CPUWeight} = $cpuunits;
5556     } else {
5557         $systemd_properties{CPUShares} = $cpuunits;
5558     }
5559
5560     if (my $cpulimit = $conf->{cpulimit}) {
5561         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5562     }
5563     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5564
5565     my $run_qemu = sub {
5566         PVE::Tools::run_fork sub {
5567             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5568
5569             my $tpmpid;
5570             if (my $tpm = $conf->{tpmstate0}) {
5571                 # start the TPM emulator so QEMU can connect on start
5572                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5573             }
5574
5575             my $exitcode = run_command($cmd, %run_params);
5576             if ($exitcode) {
5577                 if ($tpmpid) {
5578                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5579                     kill 'TERM', $tpmpid;
5580                 }
5581                 die "QEMU exited with code $exitcode\n";
5582             }
5583         };
5584     };
5585
5586     if ($conf->{hugepages}) {
5587
5588         my $code = sub {
5589             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5590             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5591
5592             PVE::QemuServer::Memory::hugepages_mount();
5593             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5594
5595             eval { $run_qemu->() };
5596             if (my $err = $@) {
5597                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5598                     if !$conf->{keephugepages};
5599                 die $err;
5600             }
5601
5602             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5603                 if !$conf->{keephugepages};
5604         };
5605         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5606
5607     } else {
5608         eval { $run_qemu->() };
5609     }
5610
5611     if (my $err = $@) {
5612         # deactivate volumes if start fails
5613         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5614         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5615
5616         die "start failed: $err";
5617     }
5618
5619     # re-reserve all PCI IDs now that we can know the actual VM PID
5620     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5621     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5622     warn $@ if $@;
5623
5624     print "migration listens on $migrate_uri\n" if $migrate_uri;
5625     $res->{migrate_uri} = $migrate_uri;
5626
5627     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5628         eval { mon_cmd($vmid, "cont"); };
5629         warn $@ if $@;
5630     }
5631
5632     #start nbd server for storage migration
5633     if (my $nbd = $migrate_opts->{nbd}) {
5634         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5635
5636         my $migrate_storage_uri;
5637         # nbd_protocol_version > 0 for unix socket support
5638         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5639             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5640             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5641             $migrate_storage_uri = "nbd:unix:$socket_path";
5642         } else {
5643             my $nodename = nodename();
5644             my $localip = $get_migration_ip->($nodename);
5645             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5646             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5647
5648             mon_cmd($vmid, "nbd-server-start", addr => {
5649                 type => 'inet',
5650                 data => {
5651                     host => "${localip}",
5652                     port => "${storage_migrate_port}",
5653                 },
5654             });
5655             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5656             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5657         }
5658
5659         $res->{migrate_storage_uri} = $migrate_storage_uri;
5660
5661         foreach my $opt (sort keys %$nbd) {
5662             my $drivestr = $nbd->{$opt}->{drivestr};
5663             my $volid = $nbd->{$opt}->{volid};
5664             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5665             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5666             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5667             print "re-using replicated volume: $opt - $volid\n"
5668                 if $nbd->{$opt}->{replicated};
5669
5670             $res->{drives}->{$opt} = $nbd->{$opt};
5671             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5672         }
5673     }
5674
5675     if ($migratedfrom) {
5676         eval {
5677             set_migration_caps($vmid);
5678         };
5679         warn $@ if $@;
5680
5681         if ($spice_port) {
5682             print "spice listens on port $spice_port\n";
5683             $res->{spice_port} = $spice_port;
5684             if ($migrate_opts->{spice_ticket}) {
5685                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5686                     $migrate_opts->{spice_ticket});
5687                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5688             }
5689         }
5690
5691     } else {
5692         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5693             if !$statefile && $conf->{balloon};
5694
5695         foreach my $opt (keys %$conf) {
5696             next if $opt !~  m/^net\d+$/;
5697             my $nicconf = parse_net($conf->{$opt});
5698             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5699         }
5700     }
5701
5702     mon_cmd($vmid, 'qom-set',
5703                 path => "machine/peripheral/balloon0",
5704                 property => "guest-stats-polling-interval",
5705                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5706
5707     if ($resume) {
5708         print "Resumed VM, removing state\n";
5709         if (my $vmstate = $conf->{vmstate}) {
5710             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5711             PVE::Storage::vdisk_free($storecfg, $vmstate);
5712         }
5713         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5714         PVE::QemuConfig->write_config($vmid, $conf);
5715     }
5716
5717     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5718
5719     return $res;
5720 }
5721
5722 sub vm_commandline {
5723     my ($storecfg, $vmid, $snapname) = @_;
5724
5725     my $conf = PVE::QemuConfig->load_config($vmid);
5726
5727     my ($forcemachine, $forcecpu);
5728     if ($snapname) {
5729         my $snapshot = $conf->{snapshots}->{$snapname};
5730         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5731
5732         # check for machine or CPU overrides in snapshot
5733         $forcemachine = $snapshot->{runningmachine};
5734         $forcecpu = $snapshot->{runningcpu};
5735
5736         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5737
5738         $conf = $snapshot;
5739     }
5740
5741     my $defaults = load_defaults();
5742
5743     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5744
5745     return PVE::Tools::cmd2string($cmd);
5746 }
5747
5748 sub vm_reset {
5749     my ($vmid, $skiplock) = @_;
5750
5751     PVE::QemuConfig->lock_config($vmid, sub {
5752
5753         my $conf = PVE::QemuConfig->load_config($vmid);
5754
5755         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5756
5757         mon_cmd($vmid, "system_reset");
5758     });
5759 }
5760
5761 sub get_vm_volumes {
5762     my ($conf) = @_;
5763
5764     my $vollist = [];
5765     foreach_volid($conf, sub {
5766         my ($volid, $attr) = @_;
5767
5768         return if $volid =~ m|^/|;
5769
5770         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5771         return if !$sid;
5772
5773         push @$vollist, $volid;
5774     });
5775
5776     return $vollist;
5777 }
5778
5779 sub vm_stop_cleanup {
5780     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5781
5782     eval {
5783
5784         if (!$keepActive) {
5785             my $vollist = get_vm_volumes($conf);
5786             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5787
5788             if (my $tpmdrive = $conf->{tpmstate0}) {
5789                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5790                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5791                 if ($storeid) {
5792                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5793                 }
5794             }
5795         }
5796
5797         foreach my $ext (qw(mon qmp pid vnc qga)) {
5798             unlink "/var/run/qemu-server/${vmid}.$ext";
5799         }
5800
5801         if ($conf->{ivshmem}) {
5802             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5803             # just delete it for now, VMs which have this already open do not
5804             # are affected, but new VMs will get a separated one. If this
5805             # becomes an issue we either add some sort of ref-counting or just
5806             # add a "don't delete on stop" flag to the ivshmem format.
5807             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5808         }
5809
5810         my $ids = [];
5811         foreach my $key (keys %$conf) {
5812             next if $key !~ m/^hostpci(\d+)$/;
5813             my $hostpciindex = $1;
5814             my $d = parse_hostpci($conf->{$key});
5815             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5816
5817             foreach my $pci (@{$d->{pciid}}) {
5818                 my $pciid = $pci->{id};
5819                 push @$ids, $pci->{id};
5820                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5821             }
5822         }
5823         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5824
5825         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5826     };
5827     warn $@ if $@; # avoid errors - just warn
5828 }
5829
5830 # call only in locked context
5831 sub _do_vm_stop {
5832     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5833
5834     my $pid = check_running($vmid, $nocheck);
5835     return if !$pid;
5836
5837     my $conf;
5838     if (!$nocheck) {
5839         $conf = PVE::QemuConfig->load_config($vmid);
5840         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5841         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5842             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5843             $timeout = $opts->{down} if $opts->{down};
5844         }
5845         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5846     }
5847
5848     eval {
5849         if ($shutdown) {
5850             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5851                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5852             } else {
5853                 mon_cmd($vmid, "system_powerdown");
5854             }
5855         } else {
5856             mon_cmd($vmid, "quit");
5857         }
5858     };
5859     my $err = $@;
5860
5861     if (!$err) {
5862         $timeout = 60 if !defined($timeout);
5863
5864         my $count = 0;
5865         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5866             $count++;
5867             sleep 1;
5868         }
5869
5870         if ($count >= $timeout) {
5871             if ($force) {
5872                 warn "VM still running - terminating now with SIGTERM\n";
5873                 kill 15, $pid;
5874             } else {
5875                 die "VM quit/powerdown failed - got timeout\n";
5876             }
5877         } else {
5878             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5879             return;
5880         }
5881     } else {
5882        if (!check_running($vmid, $nocheck)) {
5883             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5884             return;
5885        }
5886        if ($force) {
5887             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5888             kill 15, $pid;
5889         } else {
5890             die "VM quit/powerdown failed\n";
5891         }
5892     }
5893
5894     # wait again
5895     $timeout = 10;
5896
5897     my $count = 0;
5898     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5899         $count++;
5900         sleep 1;
5901     }
5902
5903     if ($count >= $timeout) {
5904         warn "VM still running - terminating now with SIGKILL\n";
5905         kill 9, $pid;
5906         sleep 1;
5907     }
5908
5909     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5910 }
5911
5912 # Note: use $nocheck to skip tests if VM configuration file exists.
5913 # We need that when migration VMs to other nodes (files already moved)
5914 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5915 sub vm_stop {
5916     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5917
5918     $force = 1 if !defined($force) && !$shutdown;
5919
5920     if ($migratedfrom){
5921         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5922         kill 15, $pid if $pid;
5923         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5924         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5925         return;
5926     }
5927
5928     PVE::QemuConfig->lock_config($vmid, sub {
5929         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5930    });
5931 }
5932
5933 sub vm_reboot {
5934     my ($vmid, $timeout) = @_;
5935
5936     PVE::QemuConfig->lock_config($vmid, sub {
5937         eval {
5938
5939             # only reboot if running, as qmeventd starts it again on a stop event
5940             return if !check_running($vmid);
5941
5942             create_reboot_request($vmid);
5943
5944             my $storecfg = PVE::Storage::config();
5945             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5946
5947         };
5948         if (my $err = $@) {
5949             # avoid that the next normal shutdown will be confused for a reboot
5950             clear_reboot_request($vmid);
5951             die $err;
5952         }
5953    });
5954 }
5955
5956 # note: if using the statestorage parameter, the caller has to check privileges
5957 sub vm_suspend {
5958     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5959
5960     my $conf;
5961     my $path;
5962     my $storecfg;
5963     my $vmstate;
5964
5965     PVE::QemuConfig->lock_config($vmid, sub {
5966
5967         $conf = PVE::QemuConfig->load_config($vmid);
5968
5969         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5970         PVE::QemuConfig->check_lock($conf)
5971             if !($skiplock || $is_backing_up);
5972
5973         die "cannot suspend to disk during backup\n"
5974             if $is_backing_up && $includestate;
5975
5976         if ($includestate) {
5977             $conf->{lock} = 'suspending';
5978             my $date = strftime("%Y-%m-%d", localtime(time()));
5979             $storecfg = PVE::Storage::config();
5980             if (!$statestorage) {
5981                 $statestorage = find_vmstate_storage($conf, $storecfg);
5982                 # check permissions for the storage
5983                 my $rpcenv = PVE::RPCEnvironment::get();
5984                 if ($rpcenv->{type} ne 'cli') {
5985                     my $authuser = $rpcenv->get_user();
5986                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5987                 }
5988             }
5989
5990
5991             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5992                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5993             $path = PVE::Storage::path($storecfg, $vmstate);
5994             PVE::QemuConfig->write_config($vmid, $conf);
5995         } else {
5996             mon_cmd($vmid, "stop");
5997         }
5998     });
5999
6000     if ($includestate) {
6001         # save vm state
6002         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6003
6004         eval {
6005             set_migration_caps($vmid, 1);
6006             mon_cmd($vmid, "savevm-start", statefile => $path);
6007             for(;;) {
6008                 my $state = mon_cmd($vmid, "query-savevm");
6009                 if (!$state->{status}) {
6010                     die "savevm not active\n";
6011                 } elsif ($state->{status} eq 'active') {
6012                     sleep(1);
6013                     next;
6014                 } elsif ($state->{status} eq 'completed') {
6015                     print "State saved, quitting\n";
6016                     last;
6017                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6018                     die "query-savevm failed with error '$state->{error}'\n"
6019                 } else {
6020                     die "query-savevm returned status '$state->{status}'\n";
6021                 }
6022             }
6023         };
6024         my $err = $@;
6025
6026         PVE::QemuConfig->lock_config($vmid, sub {
6027             $conf = PVE::QemuConfig->load_config($vmid);
6028             if ($err) {
6029                 # cleanup, but leave suspending lock, to indicate something went wrong
6030                 eval {
6031                     mon_cmd($vmid, "savevm-end");
6032                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6033                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6034                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6035                     PVE::QemuConfig->write_config($vmid, $conf);
6036                 };
6037                 warn $@ if $@;
6038                 die $err;
6039             }
6040
6041             die "lock changed unexpectedly\n"
6042                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6043
6044             mon_cmd($vmid, "quit");
6045             $conf->{lock} = 'suspended';
6046             PVE::QemuConfig->write_config($vmid, $conf);
6047         });
6048     }
6049 }
6050
6051 sub vm_resume {
6052     my ($vmid, $skiplock, $nocheck) = @_;
6053
6054     PVE::QemuConfig->lock_config($vmid, sub {
6055         my $res = mon_cmd($vmid, 'query-status');
6056         my $resume_cmd = 'cont';
6057         my $reset = 0;
6058
6059         if ($res->{status}) {
6060             return if $res->{status} eq 'running'; # job done, go home
6061             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6062             $reset = 1 if $res->{status} eq 'shutdown';
6063         }
6064
6065         if (!$nocheck) {
6066
6067             my $conf = PVE::QemuConfig->load_config($vmid);
6068
6069             PVE::QemuConfig->check_lock($conf)
6070                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6071         }
6072
6073         if ($reset) {
6074             # required if a VM shuts down during a backup and we get a resume
6075             # request before the backup finishes for example
6076             mon_cmd($vmid, "system_reset");
6077         }
6078         mon_cmd($vmid, $resume_cmd);
6079     });
6080 }
6081
6082 sub vm_sendkey {
6083     my ($vmid, $skiplock, $key) = @_;
6084
6085     PVE::QemuConfig->lock_config($vmid, sub {
6086
6087         my $conf = PVE::QemuConfig->load_config($vmid);
6088
6089         # there is no qmp command, so we use the human monitor command
6090         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6091         die $res if $res ne '';
6092     });
6093 }
6094
6095 # vzdump restore implementaion
6096
6097 sub tar_archive_read_firstfile {
6098     my $archive = shift;
6099
6100     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6101
6102     # try to detect archive type first
6103     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6104         die "unable to open file '$archive'\n";
6105     my $firstfile = <$fh>;
6106     kill 15, $pid;
6107     close $fh;
6108
6109     die "ERROR: archive contaions no data\n" if !$firstfile;
6110     chomp $firstfile;
6111
6112     return $firstfile;
6113 }
6114
6115 sub tar_restore_cleanup {
6116     my ($storecfg, $statfile) = @_;
6117
6118     print STDERR "starting cleanup\n";
6119
6120     if (my $fd = IO::File->new($statfile, "r")) {
6121         while (defined(my $line = <$fd>)) {
6122             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6123                 my $volid = $2;
6124                 eval {
6125                     if ($volid =~ m|^/|) {
6126                         unlink $volid || die 'unlink failed\n';
6127                     } else {
6128                         PVE::Storage::vdisk_free($storecfg, $volid);
6129                     }
6130                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6131                 };
6132                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6133             } else {
6134                 print STDERR "unable to parse line in statfile - $line";
6135             }
6136         }
6137         $fd->close();
6138     }
6139 }
6140
6141 sub restore_file_archive {
6142     my ($archive, $vmid, $user, $opts) = @_;
6143
6144     return restore_vma_archive($archive, $vmid, $user, $opts)
6145         if $archive eq '-';
6146
6147     my $info = PVE::Storage::archive_info($archive);
6148     my $format = $opts->{format} // $info->{format};
6149     my $comp = $info->{compression};
6150
6151     # try to detect archive format
6152     if ($format eq 'tar') {
6153         return restore_tar_archive($archive, $vmid, $user, $opts);
6154     } else {
6155         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6156     }
6157 }
6158
6159 # hepler to remove disks that will not be used after restore
6160 my $restore_cleanup_oldconf = sub {
6161     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6162
6163     PVE::QemuConfig->foreach_volume($oldconf, sub {
6164         my ($ds, $drive) = @_;
6165
6166         return if drive_is_cdrom($drive, 1);
6167
6168         my $volid = $drive->{file};
6169         return if !$volid || $volid =~ m|^/|;
6170
6171         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6172         return if !$path || !$owner || ($owner != $vmid);
6173
6174         # Note: only delete disk we want to restore
6175         # other volumes will become unused
6176         if ($virtdev_hash->{$ds}) {
6177             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6178             if (my $err = $@) {
6179                 warn $err;
6180             }
6181         }
6182     });
6183
6184     # delete vmstate files, after the restore we have no snapshots anymore
6185     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6186         my $snap = $oldconf->{snapshots}->{$snapname};
6187         if ($snap->{vmstate}) {
6188             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6189             if (my $err = $@) {
6190                 warn $err;
6191             }
6192         }
6193     }
6194 };
6195
6196 # Helper to parse vzdump backup device hints
6197 #
6198 # $rpcenv: Environment, used to ckeck storage permissions
6199 # $user: User ID, to check storage permissions
6200 # $storecfg: Storage configuration
6201 # $fh: the file handle for reading the configuration
6202 # $devinfo: should contain device sizes for all backu-up'ed devices
6203 # $options: backup options (pool, default storage)
6204 #
6205 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6206 my $parse_backup_hints = sub {
6207     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6208
6209     my $virtdev_hash = {};
6210
6211     while (defined(my $line = <$fh>)) {
6212         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6213             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6214             die "archive does not contain data for drive '$virtdev'\n"
6215                 if !$devinfo->{$devname};
6216
6217             if (defined($options->{storage})) {
6218                 $storeid = $options->{storage} || 'local';
6219             } elsif (!$storeid) {
6220                 $storeid = 'local';
6221             }
6222             $format = 'raw' if !$format;
6223             $devinfo->{$devname}->{devname} = $devname;
6224             $devinfo->{$devname}->{virtdev} = $virtdev;
6225             $devinfo->{$devname}->{format} = $format;
6226             $devinfo->{$devname}->{storeid} = $storeid;
6227
6228             # check permission on storage
6229             my $pool = $options->{pool}; # todo: do we need that?
6230             if ($user ne 'root@pam') {
6231                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6232             }
6233
6234             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6235         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6236             my $virtdev = $1;
6237             my $drive = parse_drive($virtdev, $2);
6238             if (drive_is_cloudinit($drive)) {
6239                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6240                 $storeid = $options->{storage} if defined ($options->{storage});
6241                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6242                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6243
6244                 $virtdev_hash->{$virtdev} = {
6245                     format => $format,
6246                     storeid => $storeid,
6247                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6248                     is_cloudinit => 1,
6249                 };
6250             }
6251         }
6252     }
6253
6254     return $virtdev_hash;
6255 };
6256
6257 # Helper to allocate and activate all volumes required for a restore
6258 #
6259 # $storecfg: Storage configuration
6260 # $virtdev_hash: as returned by parse_backup_hints()
6261 #
6262 # Returns: { $virtdev => $volid }
6263 my $restore_allocate_devices = sub {
6264     my ($storecfg, $virtdev_hash, $vmid) = @_;
6265
6266     my $map = {};
6267     foreach my $virtdev (sort keys %$virtdev_hash) {
6268         my $d = $virtdev_hash->{$virtdev};
6269         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6270         my $storeid = $d->{storeid};
6271         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6272
6273         # test if requested format is supported
6274         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6275         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6276         $d->{format} = $defFormat if !$supported;
6277
6278         my $name;
6279         if ($d->{is_cloudinit}) {
6280             $name = "vm-$vmid-cloudinit";
6281             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6282             if ($scfg->{path}) {
6283                 $name .= ".$d->{format}";
6284             }
6285         }
6286
6287         my $volid = PVE::Storage::vdisk_alloc(
6288             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6289
6290         print STDERR "new volume ID is '$volid'\n";
6291         $d->{volid} = $volid;
6292
6293         PVE::Storage::activate_volumes($storecfg, [$volid]);
6294
6295         $map->{$virtdev} = $volid;
6296     }
6297
6298     return $map;
6299 };
6300
6301 sub restore_update_config_line {
6302     my ($cookie, $map, $line, $unique) = @_;
6303
6304     return '' if $line =~ m/^\#qmdump\#/;
6305     return '' if $line =~ m/^\#vzdump\#/;
6306     return '' if $line =~ m/^lock:/;
6307     return '' if $line =~ m/^unused\d+:/;
6308     return '' if $line =~ m/^parent:/;
6309
6310     my $res = '';
6311
6312     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6313     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6314         # try to convert old 1.X settings
6315         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6316         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6317             my ($model, $macaddr) = split(/\=/, $devconfig);
6318             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6319             my $net = {
6320                 model => $model,
6321                 bridge => "vmbr$ind",
6322                 macaddr => $macaddr,
6323             };
6324             my $netstr = print_net($net);
6325
6326             $res .= "net$cookie->{netcount}: $netstr\n";
6327             $cookie->{netcount}++;
6328         }
6329     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6330         my ($id, $netstr) = ($1, $2);
6331         my $net = parse_net($netstr);
6332         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6333         $netstr = print_net($net);
6334         $res .= "$id: $netstr\n";
6335     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6336         my $virtdev = $1;
6337         my $value = $3;
6338         my $di = parse_drive($virtdev, $value);
6339         if (defined($di->{backup}) && !$di->{backup}) {
6340             $res .= "#$line";
6341         } elsif ($map->{$virtdev}) {
6342             delete $di->{format}; # format can change on restore
6343             $di->{file} = $map->{$virtdev};
6344             $value = print_drive($di);
6345             $res .= "$virtdev: $value\n";
6346         } else {
6347             $res .= $line;
6348         }
6349     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6350         my $vmgenid = $1;
6351         if ($vmgenid ne '0') {
6352             # always generate a new vmgenid if there was a valid one setup
6353             $vmgenid = generate_uuid();
6354         }
6355         $res .= "vmgenid: $vmgenid\n";
6356     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6357         my ($uuid, $uuid_str);
6358         UUID::generate($uuid);
6359         UUID::unparse($uuid, $uuid_str);
6360         my $smbios1 = parse_smbios1($2);
6361         $smbios1->{uuid} = $uuid_str;
6362         $res .= $1.print_smbios1($smbios1)."\n";
6363     } else {
6364         $res .= $line;
6365     }
6366
6367     return $res;
6368 }
6369
6370 my $restore_deactivate_volumes = sub {
6371     my ($storecfg, $devinfo) = @_;
6372
6373     my $vollist = [];
6374     foreach my $devname (keys %$devinfo) {
6375         my $volid = $devinfo->{$devname}->{volid};
6376         push @$vollist, $volid if $volid;
6377     }
6378
6379     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6380 };
6381
6382 my $restore_destroy_volumes = sub {
6383     my ($storecfg, $devinfo) = @_;
6384
6385     foreach my $devname (keys %$devinfo) {
6386         my $volid = $devinfo->{$devname}->{volid};
6387         next if !$volid;
6388         eval {
6389             if ($volid =~ m|^/|) {
6390                 unlink $volid || die 'unlink failed\n';
6391             } else {
6392                 PVE::Storage::vdisk_free($storecfg, $volid);
6393             }
6394             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6395         };
6396         print STDERR "unable to cleanup '$volid' - $@" if $@;
6397     }
6398 };
6399
6400 sub scan_volids {
6401     my ($cfg, $vmid) = @_;
6402
6403     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6404
6405     my $volid_hash = {};
6406     foreach my $storeid (keys %$info) {
6407         foreach my $item (@{$info->{$storeid}}) {
6408             next if !($item->{volid} && $item->{size});
6409             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6410             $volid_hash->{$item->{volid}} = $item;
6411         }
6412     }
6413
6414     return $volid_hash;
6415 }
6416
6417 sub update_disk_config {
6418     my ($vmid, $conf, $volid_hash) = @_;
6419
6420     my $changes;
6421     my $prefix = "VM $vmid";
6422
6423     # used and unused disks
6424     my $referenced = {};
6425
6426     # Note: it is allowed to define multiple storages with same path (alias), so
6427     # we need to check both 'volid' and real 'path' (two different volid can point
6428     # to the same path).
6429
6430     my $referencedpath = {};
6431
6432     # update size info
6433     PVE::QemuConfig->foreach_volume($conf, sub {
6434         my ($opt, $drive) = @_;
6435
6436         my $volid = $drive->{file};
6437         return if !$volid;
6438         my $volume = $volid_hash->{$volid};
6439
6440         # mark volid as "in-use" for next step
6441         $referenced->{$volid} = 1;
6442         if ($volume && (my $path = $volume->{path})) {
6443             $referencedpath->{$path} = 1;
6444         }
6445
6446         return if drive_is_cdrom($drive);
6447         return if !$volume;
6448
6449         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6450         if (defined($updated)) {
6451             $changes = 1;
6452             $conf->{$opt} = print_drive($updated);
6453             print "$prefix ($opt): $msg\n";
6454         }
6455     });
6456
6457     # remove 'unusedX' entry if volume is used
6458     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6459         my ($opt, $drive) = @_;
6460
6461         my $volid = $drive->{file};
6462         return if !$volid;
6463
6464         my $path;
6465         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6466         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6467             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6468             $changes = 1;
6469             delete $conf->{$opt};
6470         }
6471
6472         $referenced->{$volid} = 1;
6473         $referencedpath->{$path} = 1 if $path;
6474     });
6475
6476     foreach my $volid (sort keys %$volid_hash) {
6477         next if $volid =~ m/vm-$vmid-state-/;
6478         next if $referenced->{$volid};
6479         my $path = $volid_hash->{$volid}->{path};
6480         next if !$path; # just to be sure
6481         next if $referencedpath->{$path};
6482         $changes = 1;
6483         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6484         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6485         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6486     }
6487
6488     return $changes;
6489 }
6490
6491 sub rescan {
6492     my ($vmid, $nolock, $dryrun) = @_;
6493
6494     my $cfg = PVE::Storage::config();
6495
6496     print "rescan volumes...\n";
6497     my $volid_hash = scan_volids($cfg, $vmid);
6498
6499     my $updatefn =  sub {
6500         my ($vmid) = @_;
6501
6502         my $conf = PVE::QemuConfig->load_config($vmid);
6503
6504         PVE::QemuConfig->check_lock($conf);
6505
6506         my $vm_volids = {};
6507         foreach my $volid (keys %$volid_hash) {
6508             my $info = $volid_hash->{$volid};
6509             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6510         }
6511
6512         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6513
6514         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6515     };
6516
6517     if (defined($vmid)) {
6518         if ($nolock) {
6519             &$updatefn($vmid);
6520         } else {
6521             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6522         }
6523     } else {
6524         my $vmlist = config_list();
6525         foreach my $vmid (keys %$vmlist) {
6526             if ($nolock) {
6527                 &$updatefn($vmid);
6528             } else {
6529                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6530             }
6531         }
6532     }
6533 }
6534
6535 sub restore_proxmox_backup_archive {
6536     my ($archive, $vmid, $user, $options) = @_;
6537
6538     my $storecfg = PVE::Storage::config();
6539
6540     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6541     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6542
6543     my $fingerprint = $scfg->{fingerprint};
6544     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6545
6546     my $repo = PVE::PBSClient::get_repository($scfg);
6547
6548     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6549     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6550     local $ENV{PBS_PASSWORD} = $password;
6551     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6552
6553     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6554         PVE::Storage::parse_volname($storecfg, $archive);
6555
6556     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6557
6558     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6559
6560     my $tmpdir = "/var/tmp/vzdumptmp$$";
6561     rmtree $tmpdir;
6562     mkpath $tmpdir;
6563
6564     my $conffile = PVE::QemuConfig->config_file($vmid);
6565      # disable interrupts (always do cleanups)
6566     local $SIG{INT} =
6567         local $SIG{TERM} =
6568         local $SIG{QUIT} =
6569         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6570
6571     # Note: $oldconf is undef if VM does not exists
6572     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6573     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6574     my $new_conf_raw = '';
6575
6576     my $rpcenv = PVE::RPCEnvironment::get();
6577     my $devinfo = {};
6578
6579     eval {
6580         # enable interrupts
6581         local $SIG{INT} =
6582             local $SIG{TERM} =
6583             local $SIG{QUIT} =
6584             local $SIG{HUP} =
6585             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6586
6587         my $cfgfn = "$tmpdir/qemu-server.conf";
6588         my $firewall_config_fn = "$tmpdir/fw.conf";
6589         my $index_fn = "$tmpdir/index.json";
6590
6591         my $cmd = "restore";
6592
6593         my $param = [$pbs_backup_name, "index.json", $index_fn];
6594         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6595         my $index = PVE::Tools::file_get_contents($index_fn);
6596         $index = decode_json($index);
6597
6598         # print Dumper($index);
6599         foreach my $info (@{$index->{files}}) {
6600             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6601                 my $devname = $1;
6602                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6603                     $devinfo->{$devname}->{size} = $1;
6604                 } else {
6605                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6606                 }
6607             }
6608         }
6609
6610         my $is_qemu_server_backup = scalar(
6611             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6612         );
6613         if (!$is_qemu_server_backup) {
6614             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6615         }
6616         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6617
6618         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6619         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6620
6621         if ($has_firewall_config) {
6622             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6623             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6624
6625             my $pve_firewall_dir = '/etc/pve/firewall';
6626             mkdir $pve_firewall_dir; # make sure the dir exists
6627             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6628         }
6629
6630         my $fh = IO::File->new($cfgfn, "r") ||
6631             die "unable to read qemu-server.conf - $!\n";
6632
6633         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6634
6635         # fixme: rate limit?
6636
6637         # create empty/temp config
6638         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6639
6640         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6641
6642         # allocate volumes
6643         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6644
6645         foreach my $virtdev (sort keys %$virtdev_hash) {
6646             my $d = $virtdev_hash->{$virtdev};
6647             next if $d->{is_cloudinit}; # no need to restore cloudinit
6648
6649             # this fails if storage is unavailable
6650             my $volid = $d->{volid};
6651             my $path = PVE::Storage::path($storecfg, $volid);
6652
6653             # for live-restore we only want to preload the efidisk and TPM state
6654             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6655
6656             my $pbs_restore_cmd = [
6657                 '/usr/bin/pbs-restore',
6658                 '--repository', $repo,
6659                 $pbs_backup_name,
6660                 "$d->{devname}.img.fidx",
6661                 $path,
6662                 '--verbose',
6663                 ];
6664
6665             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6666             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6667
6668             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6669                 push @$pbs_restore_cmd, '--skip-zero';
6670             }
6671
6672             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6673             print "restore proxmox backup image: $dbg_cmdstring\n";
6674             run_command($pbs_restore_cmd);
6675         }
6676
6677         $fh->seek(0, 0) || die "seek failed - $!\n";
6678
6679         my $cookie = { netcount => 0 };
6680         while (defined(my $line = <$fh>)) {
6681             $new_conf_raw .= restore_update_config_line(
6682                 $cookie,
6683                 $map,
6684                 $line,
6685                 $options->{unique},
6686             );
6687         }
6688
6689         $fh->close();
6690     };
6691     my $err = $@;
6692
6693     if ($err || !$options->{live}) {
6694         $restore_deactivate_volumes->($storecfg, $devinfo);
6695     }
6696
6697     rmtree $tmpdir;
6698
6699     if ($err) {
6700         $restore_destroy_volumes->($storecfg, $devinfo);
6701         die $err;
6702     }
6703
6704     if ($options->{live}) {
6705         # keep lock during live-restore
6706         $new_conf_raw .= "\nlock: create";
6707     }
6708
6709     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6710
6711     PVE::Cluster::cfs_update(); # make sure we read new file
6712
6713     eval { rescan($vmid, 1); };
6714     warn $@ if $@;
6715
6716     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6717
6718     if ($options->{live}) {
6719         # enable interrupts
6720         local $SIG{INT} =
6721             local $SIG{TERM} =
6722             local $SIG{QUIT} =
6723             local $SIG{HUP} =
6724             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6725
6726         my $conf = PVE::QemuConfig->load_config($vmid);
6727         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6728
6729         # these special drives are already restored before start
6730         delete $devinfo->{'drive-efidisk0'};
6731         delete $devinfo->{'drive-tpmstate0-backup'};
6732         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6733
6734         PVE::QemuConfig->remove_lock($vmid, "create");
6735     }
6736 }
6737
6738 sub pbs_live_restore {
6739     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6740
6741     print "starting VM for live-restore\n";
6742     print "repository: '$repo', snapshot: '$snap'\n";
6743
6744     my $pbs_backing = {};
6745     for my $ds (keys %$restored_disks) {
6746         $ds =~ m/^drive-(.*)$/;
6747         my $confname = $1;
6748         $pbs_backing->{$confname} = {
6749             repository => $repo,
6750             snapshot => $snap,
6751             archive => "$ds.img.fidx",
6752         };
6753         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6754
6755         my $drive = parse_drive($confname, $conf->{$confname});
6756         print "restoring '$ds' to '$drive->{file}'\n";
6757     }
6758
6759     my $drives_streamed = 0;
6760     eval {
6761         # make sure HA doesn't interrupt our restore by stopping the VM
6762         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6763             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6764         }
6765
6766         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6767         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6768         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6769
6770         my $qmeventd_fd = register_qmeventd_handle($vmid);
6771
6772         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6773         # this will effectively collapse the backing image chain consisting of
6774         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6775         # removes itself once all backing images vanish with 'auto-remove=on')
6776         my $jobs = {};
6777         for my $ds (sort keys %$restored_disks) {
6778             my $job_id = "restore-$ds";
6779             mon_cmd($vmid, 'block-stream',
6780                 'job-id' => $job_id,
6781                 device => "$ds",
6782             );
6783             $jobs->{$job_id} = {};
6784         }
6785
6786         mon_cmd($vmid, 'cont');
6787         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6788
6789         print "restore-drive jobs finished successfully, removing all tracking block devices"
6790             ." to disconnect from Proxmox Backup Server\n";
6791
6792         for my $ds (sort keys %$restored_disks) {
6793             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6794         }
6795
6796         close($qmeventd_fd);
6797     };
6798
6799     my $err = $@;
6800
6801     if ($err) {
6802         warn "An error occured during live-restore: $err\n";
6803         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6804         die "live-restore failed\n";
6805     }
6806 }
6807
6808 sub restore_vma_archive {
6809     my ($archive, $vmid, $user, $opts, $comp) = @_;
6810
6811     my $readfrom = $archive;
6812
6813     my $cfg = PVE::Storage::config();
6814     my $commands = [];
6815     my $bwlimit = $opts->{bwlimit};
6816
6817     my $dbg_cmdstring = '';
6818     my $add_pipe = sub {
6819         my ($cmd) = @_;
6820         push @$commands, $cmd;
6821         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6822         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6823         $readfrom = '-';
6824     };
6825
6826     my $input = undef;
6827     if ($archive eq '-') {
6828         $input = '<&STDIN';
6829     } else {
6830         # If we use a backup from a PVE defined storage we also consider that
6831         # storage's rate limit:
6832         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6833         if (defined($volid)) {
6834             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6835             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6836             if ($readlimit) {
6837                 print STDERR "applying read rate limit: $readlimit\n";
6838                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6839                 $add_pipe->($cstream);
6840             }
6841         }
6842     }
6843
6844     if ($comp) {
6845         my $info = PVE::Storage::decompressor_info('vma', $comp);
6846         my $cmd = $info->{decompressor};
6847         push @$cmd, $readfrom;
6848         $add_pipe->($cmd);
6849     }
6850
6851     my $tmpdir = "/var/tmp/vzdumptmp$$";
6852     rmtree $tmpdir;
6853
6854     # disable interrupts (always do cleanups)
6855     local $SIG{INT} =
6856         local $SIG{TERM} =
6857         local $SIG{QUIT} =
6858         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6859
6860     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6861     POSIX::mkfifo($mapfifo, 0600);
6862     my $fifofh;
6863     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6864
6865     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6866
6867     my $oldtimeout;
6868     my $timeout = 5;
6869
6870     my $devinfo = {};
6871
6872     my $rpcenv = PVE::RPCEnvironment::get();
6873
6874     my $conffile = PVE::QemuConfig->config_file($vmid);
6875
6876     # Note: $oldconf is undef if VM does not exist
6877     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6878     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6879     my $new_conf_raw = '';
6880
6881     my %storage_limits;
6882
6883     my $print_devmap = sub {
6884         my $cfgfn = "$tmpdir/qemu-server.conf";
6885
6886         # we can read the config - that is already extracted
6887         my $fh = IO::File->new($cfgfn, "r") ||
6888             die "unable to read qemu-server.conf - $!\n";
6889
6890         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6891         if (-f $fwcfgfn) {
6892             my $pve_firewall_dir = '/etc/pve/firewall';
6893             mkdir $pve_firewall_dir; # make sure the dir exists
6894             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6895         }
6896
6897         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6898
6899         foreach my $info (values %{$virtdev_hash}) {
6900             my $storeid = $info->{storeid};
6901             next if defined($storage_limits{$storeid});
6902
6903             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6904             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6905             $storage_limits{$storeid} = $limit * 1024;
6906         }
6907
6908         foreach my $devname (keys %$devinfo) {
6909             die "found no device mapping information for device '$devname'\n"
6910                 if !$devinfo->{$devname}->{virtdev};
6911         }
6912
6913         # create empty/temp config
6914         if ($oldconf) {
6915             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6916             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6917         }
6918
6919         # allocate volumes
6920         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6921
6922         # print restore information to $fifofh
6923         foreach my $virtdev (sort keys %$virtdev_hash) {
6924             my $d = $virtdev_hash->{$virtdev};
6925             next if $d->{is_cloudinit}; # no need to restore cloudinit
6926
6927             my $storeid = $d->{storeid};
6928             my $volid = $d->{volid};
6929
6930             my $map_opts = '';
6931             if (my $limit = $storage_limits{$storeid}) {
6932                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6933             }
6934
6935             my $write_zeros = 1;
6936             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6937                 $write_zeros = 0;
6938             }
6939
6940             my $path = PVE::Storage::path($cfg, $volid);
6941
6942             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6943
6944             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6945         }
6946
6947         $fh->seek(0, 0) || die "seek failed - $!\n";
6948
6949         my $cookie = { netcount => 0 };
6950         while (defined(my $line = <$fh>)) {
6951             $new_conf_raw .= restore_update_config_line(
6952                 $cookie,
6953                 $map,
6954                 $line,
6955                 $opts->{unique},
6956             );
6957         }
6958
6959         $fh->close();
6960     };
6961
6962     eval {
6963         # enable interrupts
6964         local $SIG{INT} =
6965             local $SIG{TERM} =
6966             local $SIG{QUIT} =
6967             local $SIG{HUP} =
6968             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6969         local $SIG{ALRM} = sub { die "got timeout\n"; };
6970
6971         $oldtimeout = alarm($timeout);
6972
6973         my $parser = sub {
6974             my $line = shift;
6975
6976             print "$line\n";
6977
6978             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6979                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6980                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6981             } elsif ($line =~ m/^CTIME: /) {
6982                 # we correctly received the vma config, so we can disable
6983                 # the timeout now for disk allocation (set to 10 minutes, so
6984                 # that we always timeout if something goes wrong)
6985                 alarm(600);
6986                 &$print_devmap();
6987                 print $fifofh "done\n";
6988                 my $tmp = $oldtimeout || 0;
6989                 $oldtimeout = undef;
6990                 alarm($tmp);
6991                 close($fifofh);
6992                 $fifofh = undef;
6993             }
6994         };
6995
6996         print "restore vma archive: $dbg_cmdstring\n";
6997         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6998     };
6999     my $err = $@;
7000
7001     alarm($oldtimeout) if $oldtimeout;
7002
7003     $restore_deactivate_volumes->($cfg, $devinfo);
7004
7005     close($fifofh) if $fifofh;
7006     unlink $mapfifo;
7007     rmtree $tmpdir;
7008
7009     if ($err) {
7010         $restore_destroy_volumes->($cfg, $devinfo);
7011         die $err;
7012     }
7013
7014     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7015
7016     PVE::Cluster::cfs_update(); # make sure we read new file
7017
7018     eval { rescan($vmid, 1); };
7019     warn $@ if $@;
7020
7021     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7022 }
7023
7024 sub restore_tar_archive {
7025     my ($archive, $vmid, $user, $opts) = @_;
7026
7027     if ($archive ne '-') {
7028         my $firstfile = tar_archive_read_firstfile($archive);
7029         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7030             if $firstfile ne 'qemu-server.conf';
7031     }
7032
7033     my $storecfg = PVE::Storage::config();
7034
7035     # avoid zombie disks when restoring over an existing VM -> cleanup first
7036     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7037     # skiplock=1 because qmrestore has set the 'create' lock itself already
7038     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7039     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7040
7041     my $tocmd = "/usr/lib/qemu-server/qmextract";
7042
7043     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7044     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7045     $tocmd .= ' --prealloc' if $opts->{prealloc};
7046     $tocmd .= ' --info' if $opts->{info};
7047
7048     # tar option "xf" does not autodetect compression when read from STDIN,
7049     # so we pipe to zcat
7050     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7051         PVE::Tools::shellquote("--to-command=$tocmd");
7052
7053     my $tmpdir = "/var/tmp/vzdumptmp$$";
7054     mkpath $tmpdir;
7055
7056     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7057     local $ENV{VZDUMP_VMID} = $vmid;
7058     local $ENV{VZDUMP_USER} = $user;
7059
7060     my $conffile = PVE::QemuConfig->config_file($vmid);
7061     my $new_conf_raw = '';
7062
7063     # disable interrupts (always do cleanups)
7064     local $SIG{INT} =
7065         local $SIG{TERM} =
7066         local $SIG{QUIT} =
7067         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7068
7069     eval {
7070         # enable interrupts
7071         local $SIG{INT} =
7072             local $SIG{TERM} =
7073             local $SIG{QUIT} =
7074             local $SIG{HUP} =
7075             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7076
7077         if ($archive eq '-') {
7078             print "extracting archive from STDIN\n";
7079             run_command($cmd, input => "<&STDIN");
7080         } else {
7081             print "extracting archive '$archive'\n";
7082             run_command($cmd);
7083         }
7084
7085         return if $opts->{info};
7086
7087         # read new mapping
7088         my $map = {};
7089         my $statfile = "$tmpdir/qmrestore.stat";
7090         if (my $fd = IO::File->new($statfile, "r")) {
7091             while (defined (my $line = <$fd>)) {
7092                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7093                     $map->{$1} = $2 if $1;
7094                 } else {
7095                     print STDERR "unable to parse line in statfile - $line\n";
7096                 }
7097             }
7098             $fd->close();
7099         }
7100
7101         my $confsrc = "$tmpdir/qemu-server.conf";
7102
7103         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7104
7105         my $cookie = { netcount => 0 };
7106         while (defined (my $line = <$srcfd>)) {
7107             $new_conf_raw .= restore_update_config_line(
7108                 $cookie,
7109                 $map,
7110                 $line,
7111                 $opts->{unique},
7112             );
7113         }
7114
7115         $srcfd->close();
7116     };
7117     if (my $err = $@) {
7118         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7119         die $err;
7120     }
7121
7122     rmtree $tmpdir;
7123
7124     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7125
7126     PVE::Cluster::cfs_update(); # make sure we read new file
7127
7128     eval { rescan($vmid, 1); };
7129     warn $@ if $@;
7130 };
7131
7132 sub foreach_storage_used_by_vm {
7133     my ($conf, $func) = @_;
7134
7135     my $sidhash = {};
7136
7137     PVE::QemuConfig->foreach_volume($conf, sub {
7138         my ($ds, $drive) = @_;
7139         return if drive_is_cdrom($drive);
7140
7141         my $volid = $drive->{file};
7142
7143         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7144         $sidhash->{$sid} = $sid if $sid;
7145     });
7146
7147     foreach my $sid (sort keys %$sidhash) {
7148         &$func($sid);
7149     }
7150 }
7151
7152 my $qemu_snap_storage = {
7153     rbd => 1,
7154 };
7155 sub do_snapshots_with_qemu {
7156     my ($storecfg, $volid, $deviceid) = @_;
7157
7158     return if $deviceid =~ m/tpmstate0/;
7159
7160     my $storage_name = PVE::Storage::parse_volume_id($volid);
7161     my $scfg = $storecfg->{ids}->{$storage_name};
7162     die "could not find storage '$storage_name'\n" if !defined($scfg);
7163
7164     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7165         return 1;
7166     }
7167
7168     if ($volid =~ m/\.(qcow2|qed)$/){
7169         return 1;
7170     }
7171
7172     return;
7173 }
7174
7175 sub qga_check_running {
7176     my ($vmid, $nowarn) = @_;
7177
7178     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7179     if ($@) {
7180         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7181         return 0;
7182     }
7183     return 1;
7184 }
7185
7186 sub template_create {
7187     my ($vmid, $conf, $disk) = @_;
7188
7189     my $storecfg = PVE::Storage::config();
7190
7191     PVE::QemuConfig->foreach_volume($conf, sub {
7192         my ($ds, $drive) = @_;
7193
7194         return if drive_is_cdrom($drive);
7195         return if $disk && $ds ne $disk;
7196
7197         my $volid = $drive->{file};
7198         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7199
7200         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7201         $drive->{file} = $voliddst;
7202         $conf->{$ds} = print_drive($drive);
7203         PVE::QemuConfig->write_config($vmid, $conf);
7204     });
7205 }
7206
7207 sub convert_iscsi_path {
7208     my ($path) = @_;
7209
7210     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7211         my $portal = $1;
7212         my $target = $2;
7213         my $lun = $3;
7214
7215         my $initiator_name = get_initiator_name();
7216
7217         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7218                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7219     }
7220
7221     die "cannot convert iscsi path '$path', unkown format\n";
7222 }
7223
7224 sub qemu_img_convert {
7225     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7226
7227     my $storecfg = PVE::Storage::config();
7228     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7229     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7230
7231     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7232
7233     my $cachemode;
7234     my $src_path;
7235     my $src_is_iscsi = 0;
7236     my $src_format;
7237
7238     if ($src_storeid) {
7239         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7240         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7241         $src_format = qemu_img_format($src_scfg, $src_volname);
7242         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7243         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7244         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7245     } elsif (-f $src_volid) {
7246         $src_path = $src_volid;
7247         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7248             $src_format = $1;
7249         }
7250     }
7251
7252     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7253
7254     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7255     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7256     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7257     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7258
7259     my $cmd = [];
7260     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7261     push @$cmd, '-l', "snapshot.name=$snapname"
7262         if $snapname && $src_format && $src_format eq "qcow2";
7263     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7264     push @$cmd, '-T', $cachemode if defined($cachemode);
7265
7266     if ($src_is_iscsi) {
7267         push @$cmd, '--image-opts';
7268         $src_path = convert_iscsi_path($src_path);
7269     } elsif ($src_format) {
7270         push @$cmd, '-f', $src_format;
7271     }
7272
7273     if ($dst_is_iscsi) {
7274         push @$cmd, '--target-image-opts';
7275         $dst_path = convert_iscsi_path($dst_path);
7276     } else {
7277         push @$cmd, '-O', $dst_format;
7278     }
7279
7280     push @$cmd, $src_path;
7281
7282     if (!$dst_is_iscsi && $is_zero_initialized) {
7283         push @$cmd, "zeroinit:$dst_path";
7284     } else {
7285         push @$cmd, $dst_path;
7286     }
7287
7288     my $parser = sub {
7289         my $line = shift;
7290         if($line =~ m/\((\S+)\/100\%\)/){
7291             my $percent = $1;
7292             my $transferred = int($size * $percent / 100);
7293             my $total_h = render_bytes($size, 1);
7294             my $transferred_h = render_bytes($transferred, 1);
7295
7296             print "transferred $transferred_h of $total_h ($percent%)\n";
7297         }
7298
7299     };
7300
7301     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7302     my $err = $@;
7303     die "copy failed: $err" if $err;
7304 }
7305
7306 sub qemu_img_format {
7307     my ($scfg, $volname) = @_;
7308
7309     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7310         return $1;
7311     } else {
7312         return "raw";
7313     }
7314 }
7315
7316 sub qemu_drive_mirror {
7317     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7318
7319     $jobs = {} if !$jobs;
7320
7321     my $qemu_target;
7322     my $format;
7323     $jobs->{"drive-$drive"} = {};
7324
7325     if ($dst_volid =~ /^nbd:/) {
7326         $qemu_target = $dst_volid;
7327         $format = "nbd";
7328     } else {
7329         my $storecfg = PVE::Storage::config();
7330         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7331
7332         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7333
7334         $format = qemu_img_format($dst_scfg, $dst_volname);
7335
7336         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7337
7338         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7339     }
7340
7341     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7342     $opts->{format} = $format if $format;
7343
7344     if (defined($src_bitmap)) {
7345         $opts->{sync} = 'incremental';
7346         $opts->{bitmap} = $src_bitmap;
7347         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7348     }
7349
7350     if (defined($bwlimit)) {
7351         $opts->{speed} = $bwlimit * 1024;
7352         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7353     } else {
7354         print "drive mirror is starting for drive-$drive\n";
7355     }
7356
7357     # if a job already runs for this device we get an error, catch it for cleanup
7358     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7359     if (my $err = $@) {
7360         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7361         warn "$@\n" if $@;
7362         die "mirroring error: $err\n";
7363     }
7364
7365     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7366 }
7367
7368 # $completion can be either
7369 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7370 # 'cancel': wait until all jobs are ready, block-job-cancel them
7371 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7372 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7373 sub qemu_drive_mirror_monitor {
7374     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7375
7376     $completion //= 'complete';
7377     $op //= "mirror";
7378
7379     eval {
7380         my $err_complete = 0;
7381
7382         my $starttime = time ();
7383         while (1) {
7384             die "block job ('$op') timed out\n" if $err_complete > 300;
7385
7386             my $stats = mon_cmd($vmid, "query-block-jobs");
7387             my $ctime = time();
7388
7389             my $running_jobs = {};
7390             for my $stat (@$stats) {
7391                 next if $stat->{type} ne $op;
7392                 $running_jobs->{$stat->{device}} = $stat;
7393             }
7394
7395             my $readycounter = 0;
7396
7397             for my $job_id (sort keys %$jobs) {
7398                 my $job = $running_jobs->{$job_id};
7399
7400                 my $vanished = !defined($job);
7401                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7402                 if($complete || ($vanished && $completion eq 'auto')) {
7403                     print "$job_id: $op-job finished\n";
7404                     delete $jobs->{$job_id};
7405                     next;
7406                 }
7407
7408                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7409
7410                 my $busy = $job->{busy};
7411                 my $ready = $job->{ready};
7412                 if (my $total = $job->{len}) {
7413                     my $transferred = $job->{offset} || 0;
7414                     my $remaining = $total - $transferred;
7415                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7416
7417                     my $duration = $ctime - $starttime;
7418                     my $total_h = render_bytes($total, 1);
7419                     my $transferred_h = render_bytes($transferred, 1);
7420
7421                     my $status = sprintf(
7422                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7423                         render_duration($duration),
7424                     );
7425
7426                     if ($ready) {
7427                         if ($busy) {
7428                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7429                         } else {
7430                             $status .= ", ready";
7431                         }
7432                     }
7433                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7434                     $jobs->{$job_id}->{ready} = $ready;
7435                 }
7436
7437                 $readycounter++ if $job->{ready};
7438             }
7439
7440             last if scalar(keys %$jobs) == 0;
7441
7442             if ($readycounter == scalar(keys %$jobs)) {
7443                 print "all '$op' jobs are ready\n";
7444
7445                 # do the complete later (or has already been done)
7446                 last if $completion eq 'skip' || $completion eq 'auto';
7447
7448                 if ($vmiddst && $vmiddst != $vmid) {
7449                     my $agent_running = $qga && qga_check_running($vmid);
7450                     if ($agent_running) {
7451                         print "freeze filesystem\n";
7452                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7453                     } else {
7454                         print "suspend vm\n";
7455                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7456                     }
7457
7458                     # if we clone a disk for a new target vm, we don't switch the disk
7459                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7460
7461                     if ($agent_running) {
7462                         print "unfreeze filesystem\n";
7463                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7464                     } else {
7465                         print "resume vm\n";
7466                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7467                     }
7468
7469                     last;
7470                 } else {
7471
7472                     for my $job_id (sort keys %$jobs) {
7473                         # try to switch the disk if source and destination are on the same guest
7474                         print "$job_id: Completing block job_id...\n";
7475
7476                         my $op;
7477                         if ($completion eq 'complete') {
7478                             $op = 'block-job-complete';
7479                         } elsif ($completion eq 'cancel') {
7480                             $op = 'block-job-cancel';
7481                         } else {
7482                             die "invalid completion value: $completion\n";
7483                         }
7484                         eval { mon_cmd($vmid, $op, device => $job_id) };
7485                         if ($@ =~ m/cannot be completed/) {
7486                             print "$job_id: block job cannot be completed, trying again.\n";
7487                             $err_complete++;
7488                         }else {
7489                             print "$job_id: Completed successfully.\n";
7490                             $jobs->{$job_id}->{complete} = 1;
7491                         }
7492                     }
7493                 }
7494             }
7495             sleep 1;
7496         }
7497     };
7498     my $err = $@;
7499
7500     if ($err) {
7501         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7502         die "block job ($op) error: $err";
7503     }
7504 }
7505
7506 sub qemu_blockjobs_cancel {
7507     my ($vmid, $jobs) = @_;
7508
7509     foreach my $job (keys %$jobs) {
7510         print "$job: Cancelling block job\n";
7511         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7512         $jobs->{$job}->{cancel} = 1;
7513     }
7514
7515     while (1) {
7516         my $stats = mon_cmd($vmid, "query-block-jobs");
7517
7518         my $running_jobs = {};
7519         foreach my $stat (@$stats) {
7520             $running_jobs->{$stat->{device}} = $stat;
7521         }
7522
7523         foreach my $job (keys %$jobs) {
7524
7525             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7526                 print "$job: Done.\n";
7527                 delete $jobs->{$job};
7528             }
7529         }
7530
7531         last if scalar(keys %$jobs) == 0;
7532
7533         sleep 1;
7534     }
7535 }
7536
7537 sub clone_disk {
7538     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7539         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7540
7541     my $newvolid;
7542
7543     if (!$full) {
7544         print "create linked clone of drive $drivename ($drive->{file})\n";
7545         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7546         push @$newvollist, $newvolid;
7547     } else {
7548
7549         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7550         $storeid = $storage if $storage;
7551
7552         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7553
7554         print "create full clone of drive $drivename ($drive->{file})\n";
7555         my $name = undef;
7556         my $size = undef;
7557         if (drive_is_cloudinit($drive)) {
7558             $name = "vm-$newvmid-cloudinit";
7559             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7560             if ($scfg->{path}) {
7561                 $name .= ".$dst_format";
7562             }
7563             $snapname = undef;
7564             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7565         } elsif ($drivename eq 'efidisk0') {
7566             $size = get_efivars_size($conf);
7567         } elsif ($drivename eq 'tpmstate0') {
7568             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7569         } else {
7570             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7571         }
7572         $newvolid = PVE::Storage::vdisk_alloc(
7573             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7574         );
7575         push @$newvollist, $newvolid;
7576
7577         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7578
7579         if (drive_is_cloudinit($drive)) {
7580             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7581             # if this is the case, we have to complete any block-jobs still there from
7582             # previous drive-mirrors
7583             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7584                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7585             }
7586             goto no_data_clone;
7587         }
7588
7589         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7590         if (!$running || $snapname) {
7591             # TODO: handle bwlimits
7592             if ($drivename eq 'efidisk0') {
7593                 # the relevant data on the efidisk may be smaller than the source
7594                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7595                 # that is given by the OVMF_VARS.fd
7596                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7597                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7598
7599                 # better for Ceph if block size is not too small, see bug #3324
7600                 my $bs = 1024*1024;
7601
7602                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7603                     "if=$src_path", "of=$dst_path"]);
7604             } else {
7605                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7606             }
7607         } else {
7608
7609             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7610
7611             my $kvmver = get_running_qemu_version ($vmid);
7612             if (!min_version($kvmver, 2, 7)) {
7613                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7614                     if $drive->{iothread};
7615             }
7616
7617             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7618                 $completion, $qga, $bwlimit);
7619         }
7620     }
7621
7622 no_data_clone:
7623     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7624
7625     my $disk = $drive;
7626     $disk->{format} = undef;
7627     $disk->{file} = $newvolid;
7628     $disk->{size} = $size if defined($size);
7629
7630     return $disk;
7631 }
7632
7633 sub get_running_qemu_version {
7634     my ($vmid) = @_;
7635     my $res = mon_cmd($vmid, "query-version");
7636     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7637 }
7638
7639 sub qemu_use_old_bios_files {
7640     my ($machine_type) = @_;
7641
7642     return if !$machine_type;
7643
7644     my $use_old_bios_files = undef;
7645
7646     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7647         $machine_type = $1;
7648         $use_old_bios_files = 1;
7649     } else {
7650         my $version = extract_version($machine_type, kvm_user_version());
7651         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7652         # load new efi bios files on migration. So this hack is required to allow
7653         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7654         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7655         $use_old_bios_files = !min_version($version, 2, 4);
7656     }
7657
7658     return ($use_old_bios_files, $machine_type);
7659 }
7660
7661 sub get_efivars_size {
7662     my ($conf) = @_;
7663     my $arch = get_vm_arch($conf);
7664     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7665     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7666     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7667     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7668     return -s $ovmf_vars;
7669 }
7670
7671 sub update_efidisk_size {
7672     my ($conf) = @_;
7673
7674     return if !defined($conf->{efidisk0});
7675
7676     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7677     $disk->{size} = get_efivars_size($conf);
7678     $conf->{efidisk0} = print_drive($disk);
7679
7680     return;
7681 }
7682
7683 sub update_tpmstate_size {
7684     my ($conf) = @_;
7685
7686     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7687     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7688     $conf->{tpmstate0} = print_drive($disk);
7689 }
7690
7691 sub create_efidisk($$$$$$$) {
7692     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7693
7694     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7695     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7696
7697     my $vars_size_b = -s $ovmf_vars;
7698     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7699     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7700     PVE::Storage::activate_volumes($storecfg, [$volid]);
7701
7702     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7703     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7704
7705     return ($volid, $size/1024);
7706 }
7707
7708 sub vm_iothreads_list {
7709     my ($vmid) = @_;
7710
7711     my $res = mon_cmd($vmid, 'query-iothreads');
7712
7713     my $iothreads = {};
7714     foreach my $iothread (@$res) {
7715         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7716     }
7717
7718     return $iothreads;
7719 }
7720
7721 sub scsihw_infos {
7722     my ($conf, $drive) = @_;
7723
7724     my $maxdev = 0;
7725
7726     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7727         $maxdev = 7;
7728     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7729         $maxdev = 1;
7730     } else {
7731         $maxdev = 256;
7732     }
7733
7734     my $controller = int($drive->{index} / $maxdev);
7735     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7736         ? "virtioscsi"
7737         : "scsihw";
7738
7739     return ($maxdev, $controller, $controller_prefix);
7740 }
7741
7742 sub windows_version {
7743     my ($ostype) = @_;
7744
7745     return 0 if !$ostype;
7746
7747     my $winversion = 0;
7748
7749     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7750         $winversion = 5;
7751     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7752         $winversion = 6;
7753     } elsif ($ostype =~ m/^win(\d+)$/) {
7754         $winversion = $1;
7755     }
7756
7757     return $winversion;
7758 }
7759
7760 sub resolve_dst_disk_format {
7761         my ($storecfg, $storeid, $src_volname, $format) = @_;
7762         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7763
7764         if (!$format) {
7765             # if no target format is specified, use the source disk format as hint
7766             if ($src_volname) {
7767                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7768                 $format = qemu_img_format($scfg, $src_volname);
7769             } else {
7770                 return $defFormat;
7771             }
7772         }
7773
7774         # test if requested format is supported - else use default
7775         my $supported = grep { $_ eq $format } @$validFormats;
7776         $format = $defFormat if !$supported;
7777         return $format;
7778 }
7779
7780 # NOTE: if this logic changes, please update docs & possibly gui logic
7781 sub find_vmstate_storage {
7782     my ($conf, $storecfg) = @_;
7783
7784     # first, return storage from conf if set
7785     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7786
7787     my ($target, $shared, $local);
7788
7789     foreach_storage_used_by_vm($conf, sub {
7790         my ($sid) = @_;
7791         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7792         my $dst = $scfg->{shared} ? \$shared : \$local;
7793         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7794     });
7795
7796     # second, use shared storage where VM has at least one disk
7797     # third, use local storage where VM has at least one disk
7798     # fall back to local storage
7799     $target = $shared // $local // 'local';
7800
7801     return $target;
7802 }
7803
7804 sub generate_uuid {
7805     my ($uuid, $uuid_str);
7806     UUID::generate($uuid);
7807     UUID::unparse($uuid, $uuid_str);
7808     return $uuid_str;
7809 }
7810
7811 sub generate_smbios1_uuid {
7812     return "uuid=".generate_uuid();
7813 }
7814
7815 sub nbd_stop {
7816     my ($vmid) = @_;
7817
7818     mon_cmd($vmid, 'nbd-server-stop');
7819 }
7820
7821 sub create_reboot_request {
7822     my ($vmid) = @_;
7823     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7824         or die "failed to create reboot trigger file: $!\n";
7825     close($fh);
7826 }
7827
7828 sub clear_reboot_request {
7829     my ($vmid) = @_;
7830     my $path = "/run/qemu-server/$vmid.reboot";
7831     my $res = 0;
7832
7833     $res = unlink($path);
7834     die "could not remove reboot request for $vmid: $!"
7835         if !$res && $! != POSIX::ENOENT;
7836
7837     return $res;
7838 }
7839
7840 sub bootorder_from_legacy {
7841     my ($conf, $bootcfg) = @_;
7842
7843     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7844     my $bootindex_hash = {};
7845     my $i = 1;
7846     foreach my $o (split(//, $boot)) {
7847         $bootindex_hash->{$o} = $i*100;
7848         $i++;
7849     }
7850
7851     my $bootorder = {};
7852
7853     PVE::QemuConfig->foreach_volume($conf, sub {
7854         my ($ds, $drive) = @_;
7855
7856         if (drive_is_cdrom ($drive, 1)) {
7857             if ($bootindex_hash->{d}) {
7858                 $bootorder->{$ds} = $bootindex_hash->{d};
7859                 $bootindex_hash->{d} += 1;
7860             }
7861         } elsif ($bootindex_hash->{c}) {
7862             $bootorder->{$ds} = $bootindex_hash->{c}
7863                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7864             $bootindex_hash->{c} += 1;
7865         }
7866     });
7867
7868     if ($bootindex_hash->{n}) {
7869         for (my $i = 0; $i < $MAX_NETS; $i++) {
7870             my $netname = "net$i";
7871             next if !$conf->{$netname};
7872             $bootorder->{$netname} = $bootindex_hash->{n};
7873             $bootindex_hash->{n} += 1;
7874         }
7875     }
7876
7877     return $bootorder;
7878 }
7879
7880 # Generate default device list for 'boot: order=' property. Matches legacy
7881 # default boot order, but with explicit device names. This is important, since
7882 # the fallback for when neither 'order' nor the old format is specified relies
7883 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7884 sub get_default_bootdevices {
7885     my ($conf) = @_;
7886
7887     my @ret = ();
7888
7889     # harddisk
7890     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7891     push @ret, $first if $first;
7892
7893     # cdrom
7894     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7895     push @ret, $first if $first;
7896
7897     # network
7898     for (my $i = 0; $i < $MAX_NETS; $i++) {
7899         my $netname = "net$i";
7900         next if !$conf->{$netname};
7901         push @ret, $netname;
7902         last;
7903     }
7904
7905     return \@ret;
7906 }
7907
7908 sub device_bootorder {
7909     my ($conf) = @_;
7910
7911     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7912
7913     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7914
7915     my $bootorder = {};
7916     if (!defined($boot) || $boot->{legacy}) {
7917         $bootorder = bootorder_from_legacy($conf, $boot);
7918     } elsif ($boot->{order}) {
7919         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7920         for my $dev (PVE::Tools::split_list($boot->{order})) {
7921             $bootorder->{$dev} = $i++;
7922         }
7923     }
7924
7925     return $bootorder;
7926 }
7927
7928 sub register_qmeventd_handle {
7929     my ($vmid) = @_;
7930
7931     my $fh;
7932     my $peer = "/var/run/qmeventd.sock";
7933     my $count = 0;
7934
7935     for (;;) {
7936         $count++;
7937         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7938         last if $fh;
7939         if ($! != EINTR && $! != EAGAIN) {
7940             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7941         }
7942         if ($count > 4) {
7943             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7944               . "after $count retries\n";
7945         }
7946         usleep(25000);
7947     }
7948
7949     # send handshake to mark VM as backing up
7950     print $fh to_json({vzdump => {vmid => "$vmid"}});
7951
7952     # return handle to be closed later when inhibit is no longer required
7953     return $fh;
7954 }
7955
7956 # bash completion helper
7957
7958 sub complete_backup_archives {
7959     my ($cmdname, $pname, $cvalue) = @_;
7960
7961     my $cfg = PVE::Storage::config();
7962
7963     my $storeid;
7964
7965     if ($cvalue =~ m/^([^:]+):/) {
7966         $storeid = $1;
7967     }
7968
7969     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7970
7971     my $res = [];
7972     foreach my $id (keys %$data) {
7973         foreach my $item (@{$data->{$id}}) {
7974             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7975             push @$res, $item->{volid} if defined($item->{volid});
7976         }
7977     }
7978
7979     return $res;
7980 }
7981
7982 my $complete_vmid_full = sub {
7983     my ($running) = @_;
7984
7985     my $idlist = vmstatus();
7986
7987     my $res = [];
7988
7989     foreach my $id (keys %$idlist) {
7990         my $d = $idlist->{$id};
7991         if (defined($running)) {
7992             next if $d->{template};
7993             next if $running && $d->{status} ne 'running';
7994             next if !$running && $d->{status} eq 'running';
7995         }
7996         push @$res, $id;
7997
7998     }
7999     return $res;
8000 };
8001
8002 sub complete_vmid {
8003     return &$complete_vmid_full();
8004 }
8005
8006 sub complete_vmid_stopped {
8007     return &$complete_vmid_full(0);
8008 }
8009
8010 sub complete_vmid_running {
8011     return &$complete_vmid_full(1);
8012 }
8013
8014 sub complete_storage {
8015
8016     my $cfg = PVE::Storage::config();
8017     my $ids = $cfg->{ids};
8018
8019     my $res = [];
8020     foreach my $sid (keys %$ids) {
8021         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8022         next if !$ids->{$sid}->{content}->{images};
8023         push @$res, $sid;
8024     }
8025
8026     return $res;
8027 }
8028
8029 sub complete_migration_storage {
8030     my ($cmd, $param, $current_value, $all_args) = @_;
8031
8032     my $targetnode = @$all_args[1];
8033
8034     my $cfg = PVE::Storage::config();
8035     my $ids = $cfg->{ids};
8036
8037     my $res = [];
8038     foreach my $sid (keys %$ids) {
8039         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8040         next if !$ids->{$sid}->{content}->{images};
8041         push @$res, $sid;
8042     }
8043
8044     return $res;
8045 }
8046
8047 sub vm_is_paused {
8048     my ($vmid) = @_;
8049     my $qmpstatus = eval {
8050         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8051         mon_cmd($vmid, "query-status");
8052     };
8053     warn "$@\n" if $@;
8054     return $qmpstatus && $qmpstatus->{status} eq "paused";
8055 }
8056
8057 sub check_volume_storage_type {
8058     my ($storecfg, $vol) = @_;
8059
8060     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8061     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8062     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8063
8064     die "storage '$storeid' does not support content-type '$vtype'\n"
8065         if !$scfg->{content}->{$vtype};
8066
8067     return 1;
8068 }
8069
8070 1;