PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2345
2346     if ($conf->{template}) {
2347         # check if any base image is still used by a linked clone
2348         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2349                 my ($ds, $drive) = @_;
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353                 return if !$volid || $volid =~ m|^/|;
2354
2355                 die "base volume '$volid' is still in use by linked cloned\n"
2356                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2357
2358         });
2359     }
2360
2361     my $volids = {};
2362     my $remove_owned_drive = sub {
2363         my ($ds, $drive) = @_;
2364         return if drive_is_cdrom($drive, 1);
2365
2366         my $volid = $drive->{file};
2367         return if !$volid || $volid =~ m|^/|;
2368         return if $volids->{$volid};
2369
2370         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2371         return if !$path || !$owner || ($owner != $vmid);
2372
2373         $volids->{$volid} = 1;
2374         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2375         warn "Could not remove disk '$volid', check manually: $@" if $@;
2376     };
2377
2378     # only remove disks owned by this VM (referenced in the config)
2379     my $include_opts = {
2380         include_unused => 1,
2381         extra_keys => ['vmstate'],
2382     };
2383     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2384
2385     for my $snap (values %{$conf->{snapshots}}) {
2386         next if !defined($snap->{vmstate});
2387         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2388         next if !defined($drive);
2389         $remove_owned_drive->('vmstate', $drive);
2390     }
2391
2392     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2393
2394     if ($purge_unreferenced) { # also remove unreferenced disk
2395         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2396         PVE::Storage::foreach_volid($vmdisks, sub {
2397             my ($volid, $sid, $volname, $d) = @_;
2398             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2399             warn $@ if $@;
2400         });
2401     }
2402
2403     if (defined $replacement_conf) {
2404         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2405     } else {
2406         PVE::QemuConfig->destroy_config($vmid);
2407     }
2408 }
2409
2410 sub parse_vm_config {
2411     my ($filename, $raw, $strict) = @_;
2412
2413     return if !defined($raw);
2414
2415     my $res = {
2416         digest => Digest::SHA::sha1_hex($raw),
2417         snapshots => {},
2418         pending => {},
2419         cloudinit => {},
2420     };
2421
2422     my $handle_error = sub {
2423         my ($msg) = @_;
2424
2425         if ($strict) {
2426             die $msg;
2427         } else {
2428             warn $msg;
2429         }
2430     };
2431
2432     $filename =~ m|/qemu-server/(\d+)\.conf$|
2433         || die "got strange filename '$filename'";
2434
2435     my $vmid = $1;
2436
2437     my $conf = $res;
2438     my $descr;
2439     my $finish_description = sub {
2440         if (defined($descr)) {
2441             $descr =~ s/\s+$//;
2442             $conf->{description} = $descr;
2443         }
2444         $descr = undef;
2445     };
2446     my $section = '';
2447
2448     my @lines = split(/\n/, $raw);
2449     foreach my $line (@lines) {
2450         next if $line =~ m/^\s*$/;
2451
2452         if ($line =~ m/^\[PENDING\]\s*$/i) {
2453             $section = 'pending';
2454             $finish_description->();
2455             $conf = $res->{$section} = {};
2456             next;
2457         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458             $section = 'cloudinit';
2459             $finish_description->();
2460             $conf = $res->{$section} = {};
2461             next;
2462
2463         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2464             $section = $1;
2465             $finish_description->();
2466             $conf = $res->{snapshots}->{$section} = {};
2467             next;
2468         }
2469
2470         if ($line =~ m/^\#(.*)$/) {
2471             $descr = '' if !defined($descr);
2472             $descr .= PVE::Tools::decode_text($1) . "\n";
2473             next;
2474         }
2475
2476         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477             $descr = '' if !defined($descr);
2478             $descr .= PVE::Tools::decode_text($2);
2479         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480             $conf->{snapstate} = $1;
2481         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2482             my $key = $1;
2483             my $value = $2;
2484             $conf->{$key} = $value;
2485         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2486             my $value = $1;
2487             if ($section eq 'pending') {
2488                 $conf->{delete} = $value; # we parse this later
2489             } else {
2490                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2491             }
2492         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             if ($section eq 'cloudinit') {
2496                 # ignore validation only used for informative purpose
2497                 $conf->{$key} = $value;
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             if ($@) {
2502                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2503             } else {
2504                 $key = 'ide2' if $key eq 'cdrom';
2505                 my $fmt = $confdesc->{$key}->{format};
2506                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507                     my $v = parse_drive($key, $value);
2508                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2509                         $v->{file} = $volid;
2510                         $value = print_drive($v);
2511                     } else {
2512                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513                         next;
2514                     }
2515                 }
2516
2517                 $conf->{$key} = $value;
2518             }
2519         } else {
2520             $handle_error->("vm $vmid - unable to parse config: $line\n");
2521         }
2522     }
2523
2524     $finish_description->();
2525     delete $res->{snapstate}; # just to be sure
2526
2527     return $res;
2528 }
2529
2530 sub write_vm_config {
2531     my ($filename, $conf) = @_;
2532
2533     delete $conf->{snapstate}; # just to be sure
2534
2535     if ($conf->{cdrom}) {
2536         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2537         $conf->{ide2} = $conf->{cdrom};
2538         delete $conf->{cdrom};
2539     }
2540
2541     # we do not use 'smp' any longer
2542     if ($conf->{sockets}) {
2543         delete $conf->{smp};
2544     } elsif ($conf->{smp}) {
2545         $conf->{sockets} = $conf->{smp};
2546         delete $conf->{cores};
2547         delete $conf->{smp};
2548     }
2549
2550     my $used_volids = {};
2551
2552     my $cleanup_config = sub {
2553         my ($cref, $pending, $snapname) = @_;
2554
2555         foreach my $key (keys %$cref) {
2556             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558             my $value = $cref->{$key};
2559             if ($key eq 'delete') {
2560                 die "propertry 'delete' is only allowed in [PENDING]\n"
2561                     if !$pending;
2562                 # fixme: check syntax?
2563                 next;
2564             }
2565             eval { $value = check_type($key, $value); };
2566             die "unable to parse value of '$key' - $@" if $@;
2567
2568             $cref->{$key} = $value;
2569
2570             if (!$snapname && is_valid_drivename($key)) {
2571                 my $drive = parse_drive($key, $value);
2572                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2573             }
2574         }
2575     };
2576
2577     &$cleanup_config($conf);
2578
2579     &$cleanup_config($conf->{pending}, 1);
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2584     }
2585
2586     # remove 'unusedX' settings if we re-add a volume
2587     foreach my $key (keys %$conf) {
2588         my $value = $conf->{$key};
2589         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590             delete $conf->{$key};
2591         }
2592     }
2593
2594     my $generate_raw_config = sub {
2595         my ($conf, $pending) = @_;
2596
2597         my $raw = '';
2598
2599         # add description as comment to top of file
2600         if (defined(my $descr = $conf->{description})) {
2601             if ($descr) {
2602                 foreach my $cl (split(/\n/, $descr)) {
2603                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2604                 }
2605             } else {
2606                 $raw .= "#\n" if $pending;
2607             }
2608         }
2609
2610         foreach my $key (sort keys %$conf) {
2611             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612             $raw .= "$key: $conf->{$key}\n";
2613         }
2614         return $raw;
2615     };
2616
2617     my $raw = &$generate_raw_config($conf);
2618
2619     if (scalar(keys %{$conf->{pending}})){
2620         $raw .= "\n[PENDING]\n";
2621         $raw .= &$generate_raw_config($conf->{pending}, 1);
2622     }
2623
2624     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2625         $raw .= "\n[special:cloudinit]\n";
2626         $raw .= &$generate_raw_config($conf->{cloudinit});
2627     }
2628
2629     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2630         $raw .= "\n[$snapname]\n";
2631         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2632     }
2633
2634     return $raw;
2635 }
2636
2637 sub load_defaults {
2638
2639     my $res = {};
2640
2641     # we use static defaults from our JSON schema configuration
2642     foreach my $key (keys %$confdesc) {
2643         if (defined(my $default = $confdesc->{$key}->{default})) {
2644             $res->{$key} = $default;
2645         }
2646     }
2647
2648     return $res;
2649 }
2650
2651 sub config_list {
2652     my $vmlist = PVE::Cluster::get_vmlist();
2653     my $res = {};
2654     return $res if !$vmlist || !$vmlist->{ids};
2655     my $ids = $vmlist->{ids};
2656     my $nodename = nodename();
2657
2658     foreach my $vmid (keys %$ids) {
2659         my $d = $ids->{$vmid};
2660         next if !$d->{node} || $d->{node} ne $nodename;
2661         next if !$d->{type} || $d->{type} ne 'qemu';
2662         $res->{$vmid}->{exists} = 1;
2663     }
2664     return $res;
2665 }
2666
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources {
2669     my ($conf, $noerr) = @_;
2670
2671     my @loc_res = ();
2672
2673     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2674     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2675
2676     push @loc_res, "ivshmem" if $conf->{ivshmem};
2677
2678     foreach my $k (keys %$conf) {
2679         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680         # sockets are safe: they will recreated be on the target side post-migrate
2681         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2683     }
2684
2685     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2686
2687     return \@loc_res;
2688 }
2689
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability {
2692     my ($storecfg, $conf, $node) = @_;
2693
2694     PVE::QemuConfig->foreach_volume($conf, sub {
2695         my ($ds, $drive) = @_;
2696
2697         my $volid = $drive->{file};
2698         return if !$volid;
2699
2700         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2701         return if !$sid;
2702
2703         # check if storage is available on both nodes
2704         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2705         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2706
2707         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2708
2709         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710             if !$scfg->{content}->{$vtype};
2711    });
2712 }
2713
2714 # list nodes where all VM images are available (used by has_feature API)
2715 sub shared_nodes {
2716     my ($conf, $storecfg) = @_;
2717
2718     my $nodelist = PVE::Cluster::get_nodelist();
2719     my $nodehash = { map { $_ => 1 } @$nodelist };
2720     my $nodename = nodename();
2721
2722     PVE::QemuConfig->foreach_volume($conf, sub {
2723         my ($ds, $drive) = @_;
2724
2725         my $volid = $drive->{file};
2726         return if !$volid;
2727
2728         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2729         if ($storeid) {
2730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2731             if ($scfg->{disable}) {
2732                 $nodehash = {};
2733             } elsif (my $avail = $scfg->{nodes}) {
2734                 foreach my $node (keys %$nodehash) {
2735                     delete $nodehash->{$node} if !$avail->{$node};
2736                 }
2737             } elsif (!$scfg->{shared}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     delete $nodehash->{$node} if $node ne $nodename
2740                 }
2741             }
2742         }
2743     });
2744
2745     return $nodehash
2746 }
2747
2748 sub check_local_storage_availability {
2749     my ($conf, $storecfg) = @_;
2750
2751     my $nodelist = PVE::Cluster::get_nodelist();
2752     my $nodehash = { map { $_ => {} } @$nodelist };
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763
2764             if ($scfg->{disable}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2767                 }
2768             } elsif (my $avail = $scfg->{nodes}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     if (!$avail->{$node}) {
2771                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2772                     }
2773                 }
2774             }
2775         }
2776     });
2777
2778     foreach my $node (values %$nodehash) {
2779         if (my $unavail = $node->{unavailable_storages}) {
2780             $node->{unavailable_storages} = [ sort keys %$unavail ];
2781         }
2782     }
2783
2784     return $nodehash
2785 }
2786
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2788 sub check_running {
2789     my ($vmid, $nocheck, $node) = @_;
2790
2791     # $nocheck is set when called during a migration, in which case the config
2792     # file might still or already reside on the *other* node
2793     # - because rename has already happened, and current node is source
2794     # - because rename hasn't happened yet, and current node is target
2795     # - because rename has happened, current node is target, but hasn't yet
2796     # processed it yet
2797     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2798     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2799 }
2800
2801 sub vzlist {
2802
2803     my $vzlist = config_list();
2804
2805     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2806
2807     while (defined(my $de = $fd->read)) {
2808         next if $de !~ m/^(\d+)\.pid$/;
2809         my $vmid = $1;
2810         next if !defined($vzlist->{$vmid});
2811         if (my $pid = check_running($vmid)) {
2812             $vzlist->{$vmid}->{pid} = $pid;
2813         }
2814     }
2815
2816     return $vzlist;
2817 }
2818
2819 our $vmstatus_return_properties = {
2820     vmid => get_standard_option('pve-vmid'),
2821     status => {
2822         description => "Qemu process status.",
2823         type => 'string',
2824         enum => ['stopped', 'running'],
2825     },
2826     maxmem => {
2827         description => "Maximum memory in bytes.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'bytes',
2831     },
2832     maxdisk => {
2833         description => "Root disk size in bytes.",
2834         type => 'integer',
2835         optional => 1,
2836         renderer => 'bytes',
2837     },
2838     name => {
2839         description => "VM name.",
2840         type => 'string',
2841         optional => 1,
2842     },
2843     qmpstatus => {
2844         description => "Qemu QMP agent status.",
2845         type => 'string',
2846         optional => 1,
2847     },
2848     pid => {
2849         description => "PID of running qemu process.",
2850         type => 'integer',
2851         optional => 1,
2852     },
2853     uptime => {
2854         description => "Uptime.",
2855         type => 'integer',
2856         optional => 1,
2857         renderer => 'duration',
2858     },
2859     cpus => {
2860         description => "Maximum usable CPUs.",
2861         type => 'number',
2862         optional => 1,
2863     },
2864     lock => {
2865         description => "The current config lock, if any.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     tags => {
2870         description  => "The current configured tags, if any",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     'running-machine' => {
2875         description => "The currently running machine type (if running).",
2876         type => 'string',
2877         optional => 1,
2878     },
2879     'running-qemu' => {
2880         description => "The currently running QEMU version (if running).",
2881         type => 'string',
2882         optional => 1,
2883     },
2884 };
2885
2886 my $last_proc_pid_stat;
2887
2888 # get VM status information
2889 # This must be fast and should not block ($full == false)
2890 # We only query KVM using QMP if $full == true (this can be slow)
2891 sub vmstatus {
2892     my ($opt_vmid, $full) = @_;
2893
2894     my $res = {};
2895
2896     my $storecfg = PVE::Storage::config();
2897
2898     my $list = vzlist();
2899     my $defaults = load_defaults();
2900
2901     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2902
2903     my $cpucount = $cpuinfo->{cpus} || 1;
2904
2905     foreach my $vmid (keys %$list) {
2906         next if $opt_vmid && ($vmid ne $opt_vmid);
2907
2908         my $conf = PVE::QemuConfig->load_config($vmid);
2909
2910         my $d = { vmid => int($vmid) };
2911         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2912
2913         # fixme: better status?
2914         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2915
2916         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2917         if (defined($size)) {
2918             $d->{disk} = 0; # no info available
2919             $d->{maxdisk} = $size;
2920         } else {
2921             $d->{disk} = 0;
2922             $d->{maxdisk} = 0;
2923         }
2924
2925         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2926             * ($conf->{cores} || $defaults->{cores});
2927         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2928         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2929
2930         $d->{name} = $conf->{name} || "VM $vmid";
2931         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2932             : $defaults->{memory}*(1024*1024);
2933
2934         if ($conf->{balloon}) {
2935             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2936             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2937                 : $defaults->{shares};
2938         }
2939
2940         $d->{uptime} = 0;
2941         $d->{cpu} = 0;
2942         $d->{mem} = 0;
2943
2944         $d->{netout} = 0;
2945         $d->{netin} = 0;
2946
2947         $d->{diskread} = 0;
2948         $d->{diskwrite} = 0;
2949
2950         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2951
2952         $d->{serial} = 1 if conf_has_serial($conf);
2953         $d->{lock} = $conf->{lock} if $conf->{lock};
2954         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2955
2956         $res->{$vmid} = $d;
2957     }
2958
2959     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2960     foreach my $dev (keys %$netdev) {
2961         next if $dev !~ m/^tap([1-9]\d*)i/;
2962         my $vmid = $1;
2963         my $d = $res->{$vmid};
2964         next if !$d;
2965
2966         $d->{netout} += $netdev->{$dev}->{receive};
2967         $d->{netin} += $netdev->{$dev}->{transmit};
2968
2969         if ($full) {
2970             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2971             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2972         }
2973
2974     }
2975
2976     my $ctime = gettimeofday;
2977
2978     foreach my $vmid (keys %$list) {
2979
2980         my $d = $res->{$vmid};
2981         my $pid = $d->{pid};
2982         next if !$pid;
2983
2984         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2985         next if !$pstat; # not running
2986
2987         my $used = $pstat->{utime} + $pstat->{stime};
2988
2989         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2990
2991         if ($pstat->{vsize}) {
2992             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2993         }
2994
2995         my $old = $last_proc_pid_stat->{$pid};
2996         if (!$old) {
2997             $last_proc_pid_stat->{$pid} = {
2998                 time => $ctime,
2999                 used => $used,
3000                 cpu => 0,
3001             };
3002             next;
3003         }
3004
3005         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3006
3007         if ($dtime > 1000) {
3008             my $dutime = $used -  $old->{used};
3009
3010             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3011             $last_proc_pid_stat->{$pid} = {
3012                 time => $ctime,
3013                 used => $used,
3014                 cpu => $d->{cpu},
3015             };
3016         } else {
3017             $d->{cpu} = $old->{cpu};
3018         }
3019     }
3020
3021     return $res if !$full;
3022
3023     my $qmpclient = PVE::QMPClient->new();
3024
3025     my $ballooncb = sub {
3026         my ($vmid, $resp) = @_;
3027
3028         my $info = $resp->{'return'};
3029         return if !$info->{max_mem};
3030
3031         my $d = $res->{$vmid};
3032
3033         # use memory assigned to VM
3034         $d->{maxmem} = $info->{max_mem};
3035         $d->{balloon} = $info->{actual};
3036
3037         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3038             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3039             $d->{freemem} = $info->{free_mem};
3040         }
3041
3042         $d->{ballooninfo} = $info;
3043     };
3044
3045     my $blockstatscb = sub {
3046         my ($vmid, $resp) = @_;
3047         my $data = $resp->{'return'} || [];
3048         my $totalrdbytes = 0;
3049         my $totalwrbytes = 0;
3050
3051         for my $blockstat (@$data) {
3052             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3053             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3054
3055             $blockstat->{device} =~ s/drive-//;
3056             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3057         }
3058         $res->{$vmid}->{diskread} = $totalrdbytes;
3059         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3060     };
3061
3062     my $machinecb = sub {
3063         my ($vmid, $resp) = @_;
3064         my $data = $resp->{'return'} || [];
3065
3066         $res->{$vmid}->{'running-machine'} =
3067             PVE::QemuServer::Machine::current_from_query_machines($data);
3068     };
3069
3070     my $versioncb = sub {
3071         my ($vmid, $resp) = @_;
3072         my $data = $resp->{'return'} // {};
3073         my $version = 'unknown';
3074
3075         if (my $v = $data->{qemu}) {
3076             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3077         }
3078
3079         $res->{$vmid}->{'running-qemu'} = $version;
3080     };
3081
3082     my $statuscb = sub {
3083         my ($vmid, $resp) = @_;
3084
3085         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3086         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3087         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3088         # this fails if ballon driver is not loaded, so this must be
3089         # the last commnand (following command are aborted if this fails).
3090         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3091
3092         my $status = 'unknown';
3093         if (!defined($status = $resp->{'return'}->{status})) {
3094             warn "unable to get VM status\n";
3095             return;
3096         }
3097
3098         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3099     };
3100
3101     foreach my $vmid (keys %$list) {
3102         next if $opt_vmid && ($vmid ne $opt_vmid);
3103         next if !$res->{$vmid}->{pid}; # not running
3104         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3105     }
3106
3107     $qmpclient->queue_execute(undef, 2);
3108
3109     foreach my $vmid (keys %$list) {
3110         next if $opt_vmid && ($vmid ne $opt_vmid);
3111         next if !$res->{$vmid}->{pid}; #not running
3112
3113         # we can't use the $qmpclient since it might have already aborted on
3114         # 'query-balloon', but this might also fail for older versions...
3115         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3116         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3117     }
3118
3119     foreach my $vmid (keys %$list) {
3120         next if $opt_vmid && ($vmid ne $opt_vmid);
3121         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3122     }
3123
3124     return $res;
3125 }
3126
3127 sub conf_has_serial {
3128     my ($conf) = @_;
3129
3130     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3131         if ($conf->{"serial$i"}) {
3132             return 1;
3133         }
3134     }
3135
3136     return 0;
3137 }
3138
3139 sub conf_has_audio {
3140     my ($conf, $id) = @_;
3141
3142     $id //= 0;
3143     my $audio = $conf->{"audio$id"};
3144     return if !defined($audio);
3145
3146     my $audioproperties = parse_property_string($audio_fmt, $audio);
3147     my $audiodriver = $audioproperties->{driver} // 'spice';
3148
3149     return {
3150         dev => $audioproperties->{device},
3151         dev_id => "audiodev$id",
3152         backend => $audiodriver,
3153         backend_id => "$audiodriver-backend${id}",
3154     };
3155 }
3156
3157 sub audio_devs {
3158     my ($audio, $audiopciaddr, $machine_version) = @_;
3159
3160     my $devs = [];
3161
3162     my $id = $audio->{dev_id};
3163     my $audiodev = "";
3164     if (min_version($machine_version, 4, 2)) {
3165         $audiodev = ",audiodev=$audio->{backend_id}";
3166     }
3167
3168     if ($audio->{dev} eq 'AC97') {
3169         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3170     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3171         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3172         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3173         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3174     } else {
3175         die "unkown audio device '$audio->{dev}', implement me!";
3176     }
3177
3178     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3179
3180     return $devs;
3181 }
3182
3183 sub get_tpm_paths {
3184     my ($vmid) = @_;
3185     return {
3186         socket => "/var/run/qemu-server/$vmid.swtpm",
3187         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3188     };
3189 }
3190
3191 sub add_tpm_device {
3192     my ($vmid, $devices, $conf) = @_;
3193
3194     return if !$conf->{tpmstate0};
3195
3196     my $paths = get_tpm_paths($vmid);
3197
3198     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3199     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3200     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3201 }
3202
3203 sub start_swtpm {
3204     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3205
3206     return if !$tpmdrive;
3207
3208     my $state;
3209     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3210     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3211     if ($storeid) {
3212         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3213     } else {
3214         $state = $tpm->{file};
3215     }
3216
3217     my $paths = get_tpm_paths($vmid);
3218
3219     # during migration, we will get state from remote
3220     #
3221     if (!$migration) {
3222         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3223         my $setup_cmd = [
3224             "swtpm_setup",
3225             "--tpmstate",
3226             "file://$state",
3227             "--createek",
3228             "--create-ek-cert",
3229             "--create-platform-cert",
3230             "--lock-nvram",
3231             "--config",
3232             "/etc/swtpm_setup.conf", # do not use XDG configs
3233             "--runas",
3234             "0", # force creation as root, error if not possible
3235             "--not-overwrite", # ignore existing state, do not modify
3236         ];
3237
3238         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3239         # TPM 2.0 supports ECC crypto, use if possible
3240         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3241
3242         run_command($setup_cmd, outfunc => sub {
3243             print "swtpm_setup: $1\n";
3244         });
3245     }
3246
3247     my $emulator_cmd = [
3248         "swtpm",
3249         "socket",
3250         "--tpmstate",
3251         "backend-uri=file://$state,mode=0600",
3252         "--ctrl",
3253         "type=unixio,path=$paths->{socket},mode=0600",
3254         "--pid",
3255         "file=$paths->{pid}",
3256         "--terminate", # terminate on QEMU disconnect
3257         "--daemon",
3258     ];
3259     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3260     run_command($emulator_cmd, outfunc => sub { print $1; });
3261
3262     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3263     while (! -e $paths->{pid}) {
3264         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3265         usleep(50_000);
3266     }
3267
3268     # return untainted PID of swtpm daemon so it can be killed on error
3269     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3270     return $1;
3271 }
3272
3273 sub vga_conf_has_spice {
3274     my ($vga) = @_;
3275
3276     my $vgaconf = parse_vga($vga);
3277     my $vgatype = $vgaconf->{type};
3278     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3279
3280     return $1 || 1;
3281 }
3282
3283 sub is_native($) {
3284     my ($arch) = @_;
3285     return get_host_arch() eq $arch;
3286 }
3287
3288 sub get_vm_arch {
3289     my ($conf) = @_;
3290     return $conf->{arch} // get_host_arch();
3291 }
3292
3293 my $default_machines = {
3294     x86_64 => 'pc',
3295     aarch64 => 'virt',
3296 };
3297
3298 sub get_installed_machine_version {
3299     my ($kvmversion) = @_;
3300     $kvmversion = kvm_user_version() if !defined($kvmversion);
3301     $kvmversion =~ m/^(\d+\.\d+)/;
3302     return $1;
3303 }
3304
3305 sub windows_get_pinned_machine_version {
3306     my ($machine, $base_version, $kvmversion) = @_;
3307
3308     my $pin_version = $base_version;
3309     if (!defined($base_version) ||
3310         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3311     ) {
3312         $pin_version = get_installed_machine_version($kvmversion);
3313     }
3314     if (!$machine || $machine eq 'pc') {
3315         $machine = "pc-i440fx-$pin_version";
3316     } elsif ($machine eq 'q35') {
3317         $machine = "pc-q35-$pin_version";
3318     } elsif ($machine eq 'virt') {
3319         $machine = "virt-$pin_version";
3320     } else {
3321         warn "unknown machine type '$machine', not touching that!\n";
3322     }
3323
3324     return $machine;
3325 }
3326
3327 sub get_vm_machine {
3328     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3329
3330     my $machine = $forcemachine || $conf->{machine};
3331
3332     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3333         $kvmversion //= kvm_user_version();
3334         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3335         # layout which confuses windows quite a bit and may result in various regressions..
3336         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3337         if (windows_version($conf->{ostype})) {
3338             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3339         }
3340         $arch //= 'x86_64';
3341         $machine ||= $default_machines->{$arch};
3342         if ($add_pve_version) {
3343             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3344             $machine .= "+pve$pvever";
3345         }
3346     }
3347
3348     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3349         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3350         $machine = $1 if $is_pxe;
3351
3352         # for version-pinned machines that do not include a pve-version (e.g.
3353         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3354         $machine .= '+pve0';
3355
3356         $machine .= '.pxe' if $is_pxe;
3357     }
3358
3359     return $machine;
3360 }
3361
3362 sub get_ovmf_files($$$) {
3363     my ($arch, $efidisk, $smm) = @_;
3364
3365     my $types = $OVMF->{$arch}
3366         or die "no OVMF images known for architecture '$arch'\n";
3367
3368     my $type = 'default';
3369     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3370         $type = $smm ? "4m" : "4m-no-smm";
3371         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3372     }
3373
3374     return $types->{$type}->@*;
3375 }
3376
3377 my $Arch2Qemu = {
3378     aarch64 => '/usr/bin/qemu-system-aarch64',
3379     x86_64 => '/usr/bin/qemu-system-x86_64',
3380 };
3381 sub get_command_for_arch($) {
3382     my ($arch) = @_;
3383     return '/usr/bin/kvm' if is_native($arch);
3384
3385     my $cmd = $Arch2Qemu->{$arch}
3386         or die "don't know how to emulate architecture '$arch'\n";
3387     return $cmd;
3388 }
3389
3390 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3391 # to use in a QEMU command line (-cpu element), first array_intersect the result
3392 # of query_supported_ with query_understood_. This is necessary because:
3393 #
3394 # a) query_understood_ returns flags the host cannot use and
3395 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3396 #    flags, but CPU settings - with most of them being flags. Those settings
3397 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3398 #
3399 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3400 # expensive. If you need the value returned from this, you can get it much
3401 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3402 # $accel being 'kvm' or 'tcg'.
3403 #
3404 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3405 # changes, automatically populating pmxcfs.
3406 #
3407 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3408 # since kvm and tcg machines support different flags
3409 #
3410 sub query_supported_cpu_flags {
3411     my ($arch) = @_;
3412
3413     $arch //= get_host_arch();
3414     my $default_machine = $default_machines->{$arch};
3415
3416     my $flags = {};
3417
3418     # FIXME: Once this is merged, the code below should work for ARM as well:
3419     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3420     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3421         $arch eq "aarch64";
3422
3423     my $kvm_supported = defined(kvm_version());
3424     my $qemu_cmd = get_command_for_arch($arch);
3425     my $fakevmid = -1;
3426     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3427
3428     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3429     my $query_supported_run_qemu = sub {
3430         my ($kvm) = @_;
3431
3432         my $flags = {};
3433         my $cmd = [
3434             $qemu_cmd,
3435             '-machine', $default_machine,
3436             '-display', 'none',
3437             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3438             '-mon', 'chardev=qmp,mode=control',
3439             '-pidfile', $pidfile,
3440             '-S', '-daemonize'
3441         ];
3442
3443         if (!$kvm) {
3444             push @$cmd, '-accel', 'tcg';
3445         }
3446
3447         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3448         die "QEMU flag querying VM exited with code " . $rc if $rc;
3449
3450         eval {
3451             my $cmd_result = mon_cmd(
3452                 $fakevmid,
3453                 'query-cpu-model-expansion',
3454                 type => 'full',
3455                 model => { name => 'host' }
3456             );
3457
3458             my $props = $cmd_result->{model}->{props};
3459             foreach my $prop (keys %$props) {
3460                 next if $props->{$prop} ne '1';
3461                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3462                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3463                 # We only keep those with underscores, to match /proc/cpuinfo
3464                 $prop =~ s/\.|-/_/g;
3465                 $flags->{$prop} = 1;
3466             }
3467         };
3468         my $err = $@;
3469
3470         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3471         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3472
3473         die $err if $err;
3474
3475         return [ sort keys %$flags ];
3476     };
3477
3478     # We need to query QEMU twice, since KVM and TCG have different supported flags
3479     PVE::QemuConfig->lock_config($fakevmid, sub {
3480         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3481         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3482
3483         if ($kvm_supported) {
3484             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3485             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3486         }
3487     });
3488
3489     return $flags;
3490 }
3491
3492 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3493 my $understood_cpu_flag_dir = "/usr/share/kvm";
3494 sub query_understood_cpu_flags {
3495     my $arch = get_host_arch();
3496     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3497
3498     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3499         if ! -e $filepath;
3500
3501     my $raw = file_get_contents($filepath);
3502     $raw =~ s/^\s+|\s+$//g;
3503     my @flags = split(/\s+/, $raw);
3504
3505     return \@flags;
3506 }
3507
3508 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3509 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3510 my sub should_disable_smm {
3511     my ($conf, $vga) = @_;
3512
3513     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3514         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3515 }
3516
3517 sub config_to_command {
3518     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3519         $pbs_backing) = @_;
3520
3521     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3522     my $devices = [];
3523     my $bridges = {};
3524     my $ostype = $conf->{ostype};
3525     my $winversion = windows_version($ostype);
3526     my $kvm = $conf->{kvm};
3527     my $nodename = nodename();
3528
3529     my $arch = get_vm_arch($conf);
3530     my $kvm_binary = get_command_for_arch($arch);
3531     my $kvmver = kvm_user_version($kvm_binary);
3532
3533     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3534         $kvmver //= "undefined";
3535         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3536     }
3537
3538     my $add_pve_version = min_version($kvmver, 4, 1);
3539
3540     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3541     my $machine_version = extract_version($machine_type, $kvmver);
3542     $kvm //= 1 if is_native($arch);
3543
3544     $machine_version =~ m/(\d+)\.(\d+)/;
3545     my ($machine_major, $machine_minor) = ($1, $2);
3546
3547     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3548         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3549     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3550         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3551             ." please upgrade node '$nodename'\n"
3552     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3553         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3554         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3555             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3556             ." node '$nodename'\n";
3557     }
3558
3559     # if a specific +pve version is required for a feature, use $version_guard
3560     # instead of min_version to allow machines to be run with the minimum
3561     # required version
3562     my $required_pve_version = 0;
3563     my $version_guard = sub {
3564         my ($major, $minor, $pve) = @_;
3565         return 0 if !min_version($machine_version, $major, $minor, $pve);
3566         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3567         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3568         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3569         return 1;
3570     };
3571
3572     if ($kvm && !defined kvm_version()) {
3573         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3574             ." or enable in BIOS.\n";
3575     }
3576
3577     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3578     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3579     my $use_old_bios_files = undef;
3580     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3581
3582     my $cmd = [];
3583     if ($conf->{affinity}) {
3584         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3585     }
3586
3587     push @$cmd, $kvm_binary;
3588
3589     push @$cmd, '-id', $vmid;
3590
3591     my $vmname = $conf->{name} || "vm$vmid";
3592
3593     push @$cmd, '-name', "$vmname,debug-threads=on";
3594
3595     push @$cmd, '-no-shutdown';
3596
3597     my $use_virtio = 0;
3598
3599     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3600     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3601     push @$cmd, '-mon', "chardev=qmp,mode=control";
3602
3603     if (min_version($machine_version, 2, 12)) {
3604         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3605         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3606     }
3607
3608     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3609
3610     push @$cmd, '-daemonize';
3611
3612     if ($conf->{smbios1}) {
3613         my $smbios_conf = parse_smbios1($conf->{smbios1});
3614         if ($smbios_conf->{base64}) {
3615             # Do not pass base64 flag to qemu
3616             delete $smbios_conf->{base64};
3617             my $smbios_string = "";
3618             foreach my $key (keys %$smbios_conf) {
3619                 my $value;
3620                 if ($key eq "uuid") {
3621                     $value = $smbios_conf->{uuid}
3622                 } else {
3623                     $value = decode_base64($smbios_conf->{$key});
3624                 }
3625                 # qemu accepts any binary data, only commas need escaping by double comma
3626                 $value =~ s/,/,,/g;
3627                 $smbios_string .= "," . $key . "=" . $value if $value;
3628             }
3629             push @$cmd, '-smbios', "type=1" . $smbios_string;
3630         } else {
3631             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3632         }
3633     }
3634
3635     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3636         my $d;
3637         if (my $efidisk = $conf->{efidisk0}) {
3638             $d = parse_drive('efidisk0', $efidisk);
3639         }
3640
3641         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3642         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3643
3644         my ($path, $format);
3645         my $read_only_str = '';
3646         if ($d) {
3647             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3648             $format = $d->{format};
3649             if ($storeid) {
3650                 $path = PVE::Storage::path($storecfg, $d->{file});
3651                 if (!defined($format)) {
3652                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3653                     $format = qemu_img_format($scfg, $volname);
3654                 }
3655             } else {
3656                 $path = $d->{file};
3657                 die "efidisk format must be specified\n"
3658                     if !defined($format);
3659             }
3660
3661             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3662         } else {
3663             log_warn("no efidisk configured! Using temporary efivars disk.");
3664             $path = "/tmp/$vmid-ovmf.fd";
3665             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3666             $format = 'raw';
3667         }
3668
3669         my $size_str = "";
3670
3671         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3672             $size_str = ",size=" . (-s $ovmf_vars);
3673         }
3674
3675         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3676         my $cache = "";
3677         if ($path =~ m/^rbd:/) {
3678                 $cache = ',cache=writeback';
3679                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3680         }
3681
3682         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3683         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3684     }
3685
3686     if ($q35) { # tell QEMU to load q35 config early
3687         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3688         if (min_version($machine_version, 4, 0)) {
3689             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3690         } else {
3691             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3692         }
3693     }
3694
3695     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3696         push @$cmd, $fixups->@*;
3697     }
3698
3699     if ($conf->{vmgenid}) {
3700         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3701     }
3702
3703     # add usb controllers
3704     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3705         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3706     push @$devices, @usbcontrollers if @usbcontrollers;
3707     my $vga = parse_vga($conf->{vga});
3708
3709     my $qxlnum = vga_conf_has_spice($conf->{vga});
3710     $vga->{type} = 'qxl' if $qxlnum;
3711
3712     if (!$vga->{type}) {
3713         if ($arch eq 'aarch64') {
3714             $vga->{type} = 'virtio';
3715         } elsif (min_version($machine_version, 2, 9)) {
3716             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3717         } else {
3718             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3719         }
3720     }
3721
3722     # enable absolute mouse coordinates (needed by vnc)
3723     my $tablet = $conf->{tablet};
3724     if (!defined($tablet)) {
3725         $tablet = $defaults->{tablet};
3726         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3727         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3728     }
3729
3730     if ($tablet) {
3731         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3732         my $kbd = print_keyboarddevice_full($conf, $arch);
3733         push @$devices, '-device', $kbd if defined($kbd);
3734     }
3735
3736     my $bootorder = device_bootorder($conf);
3737
3738     # host pci device passthrough
3739     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3740         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3741
3742     # usb devices
3743     my $usb_dev_features = {};
3744     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3745
3746     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3747         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3748     push @$devices, @usbdevices if @usbdevices;
3749
3750     # serial devices
3751     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3752         my $path = $conf->{"serial$i"} or next;
3753         if ($path eq 'socket') {
3754             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3755             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3756             # On aarch64, serial0 is the UART device. Qemu only allows
3757             # connecting UART devices via the '-serial' command line, as
3758             # the device has a fixed slot on the hardware...
3759             if ($arch eq 'aarch64' && $i == 0) {
3760                 push @$devices, '-serial', "chardev:serial$i";
3761             } else {
3762                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3763             }
3764         } else {
3765             die "no such serial device\n" if ! -c $path;
3766             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3767             push @$devices, '-device', "isa-serial,chardev=serial$i";
3768         }
3769     }
3770
3771     # parallel devices
3772     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3773         if (my $path = $conf->{"parallel$i"}) {
3774             die "no such parallel device\n" if ! -c $path;
3775             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3776             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3777             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3778         }
3779     }
3780
3781     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3782         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3783         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3784         push @$devices, @$audio_devs;
3785     }
3786
3787     add_tpm_device($vmid, $devices, $conf);
3788
3789     my $sockets = 1;
3790     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3791     $sockets = $conf->{sockets} if  $conf->{sockets};
3792
3793     my $cores = $conf->{cores} || 1;
3794
3795     my $maxcpus = $sockets * $cores;
3796
3797     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3798
3799     my $allowed_vcpus = $cpuinfo->{cpus};
3800
3801     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3802
3803     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3804         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3805         for (my $i = 2; $i <= $vcpus; $i++)  {
3806             my $cpustr = print_cpu_device($conf,$i);
3807             push @$cmd, '-device', $cpustr;
3808         }
3809
3810     } else {
3811
3812         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3813     }
3814     push @$cmd, '-nodefaults';
3815
3816     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3817
3818     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3819
3820     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3821
3822     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3823         push @$devices, '-device', print_vga_device(
3824             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3825
3826         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3827
3828         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3829         push @$cmd,  '-vnc', "unix:$socket,password=on";
3830     } else {
3831         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3832         push @$cmd, '-nographic';
3833     }
3834
3835     # time drift fix
3836     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3837     my $useLocaltime = $conf->{localtime};
3838
3839     if ($winversion >= 5) { # windows
3840         $useLocaltime = 1 if !defined($conf->{localtime});
3841
3842         # use time drift fix when acpi is enabled
3843         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3844             $tdf = 1 if !defined($conf->{tdf});
3845         }
3846     }
3847
3848     if ($winversion >= 6) {
3849         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3850         push @$cmd, '-no-hpet';
3851     }
3852
3853     push @$rtcFlags, 'driftfix=slew' if $tdf;
3854
3855     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3856         push @$rtcFlags, "base=$conf->{startdate}";
3857     } elsif ($useLocaltime) {
3858         push @$rtcFlags, 'base=localtime';
3859     }
3860
3861     if ($forcecpu) {
3862         push @$cmd, '-cpu', $forcecpu;
3863     } else {
3864         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3865     }
3866
3867     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3868
3869     push @$cmd, '-S' if $conf->{freeze};
3870
3871     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3872
3873     my $guest_agent = parse_guest_agent($conf);
3874
3875     if ($guest_agent->{enabled}) {
3876         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3877         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3878
3879         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3880             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3881             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3882             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3883         } elsif ($guest_agent->{type} eq 'isa') {
3884             push @$devices, '-device', "isa-serial,chardev=qga0";
3885         }
3886     }
3887
3888     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3889     if ($rng && $version_guard->(4, 1, 2)) {
3890         check_rng_source($rng->{source});
3891
3892         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3893         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3894         my $limiter_str = "";
3895         if ($max_bytes) {
3896             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3897         }
3898
3899         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3900         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3901         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3902     }
3903
3904     my $spice_port;
3905
3906     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3907         if ($qxlnum > 1) {
3908             if ($winversion){
3909                 for (my $i = 1; $i < $qxlnum; $i++){
3910                     push @$devices, '-device', print_vga_device(
3911                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3912                 }
3913             } else {
3914                 # assume other OS works like Linux
3915                 my ($ram, $vram) = ("134217728", "67108864");
3916                 if ($vga->{memory}) {
3917                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3918                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3919                 }
3920                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3921                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3922             }
3923         }
3924
3925         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3926
3927         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3928         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3929         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3930
3931         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3932         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3933         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3934
3935         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3936         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3937
3938         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3939         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3940         if ($spice_enhancement->{foldersharing}) {
3941             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3942             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3943         }
3944
3945         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3946         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3947             if $spice_enhancement->{videostreaming};
3948
3949         push @$devices, '-spice', "$spice_opts";
3950     }
3951
3952     # enable balloon by default, unless explicitly disabled
3953     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3954         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3955         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3956         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3957         push @$devices, '-device', $ballooncmd;
3958     }
3959
3960     if ($conf->{watchdog}) {
3961         my $wdopts = parse_watchdog($conf->{watchdog});
3962         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3963         my $watchdog = $wdopts->{model} || 'i6300esb';
3964         push @$devices, '-device', "$watchdog$pciaddr";
3965         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3966     }
3967
3968     my $vollist = [];
3969     my $scsicontroller = {};
3970     my $ahcicontroller = {};
3971     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3972
3973     # Add iscsi initiator name if available
3974     if (my $initiator = get_initiator_name()) {
3975         push @$devices, '-iscsi', "initiator-name=$initiator";
3976     }
3977
3978     PVE::QemuConfig->foreach_volume($conf, sub {
3979         my ($ds, $drive) = @_;
3980
3981         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3982             check_volume_storage_type($storecfg, $drive->{file});
3983             push @$vollist, $drive->{file};
3984         }
3985
3986         # ignore efidisk here, already added in bios/fw handling code above
3987         return if $drive->{interface} eq 'efidisk';
3988         # similar for TPM
3989         return if $drive->{interface} eq 'tpmstate';
3990
3991         $use_virtio = 1 if $ds =~ m/^virtio/;
3992
3993         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3994
3995         if ($drive->{interface} eq 'virtio'){
3996            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3997         }
3998
3999         if ($drive->{interface} eq 'scsi') {
4000
4001             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4002
4003             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4004                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4005
4006             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4007             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4008
4009             my $iothread = '';
4010             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4011                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4012                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4013             } elsif ($drive->{iothread}) {
4014                 log_warn(
4015                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4016                 );
4017             }
4018
4019             my $queues = '';
4020             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4021                 $queues = ",num_queues=$drive->{queues}";
4022             }
4023
4024             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4025                 if !$scsicontroller->{$controller};
4026             $scsicontroller->{$controller}=1;
4027         }
4028
4029         if ($drive->{interface} eq 'sata') {
4030             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4031             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4032             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4033                 if !$ahcicontroller->{$controller};
4034             $ahcicontroller->{$controller}=1;
4035         }
4036
4037         my $pbs_conf = $pbs_backing->{$ds};
4038         my $pbs_name = undef;
4039         if ($pbs_conf) {
4040             $pbs_name = "drive-$ds-pbs";
4041             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4042         }
4043
4044         my $drive_cmd = print_drive_commandline_full(
4045             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4046
4047         # extra protection for templates, but SATA and IDE don't support it..
4048         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4049
4050         push @$devices, '-drive',$drive_cmd;
4051         push @$devices, '-device', print_drivedevice_full(
4052             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4053     });
4054
4055     for (my $i = 0; $i < $MAX_NETS; $i++) {
4056         my $netname = "net$i";
4057
4058         next if !$conf->{$netname};
4059         my $d = parse_net($conf->{$netname});
4060         next if !$d;
4061         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4062
4063         $use_virtio = 1 if $d->{model} eq 'virtio';
4064
4065         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4066
4067         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4068         push @$devices, '-netdev', $netdevfull;
4069
4070         my $netdevicefull = print_netdevice_full(
4071             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4072
4073         push @$devices, '-device', $netdevicefull;
4074     }
4075
4076     if ($conf->{ivshmem}) {
4077         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4078
4079         my $bus;
4080         if ($q35) {
4081             $bus = print_pcie_addr("ivshmem");
4082         } else {
4083             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4084         }
4085
4086         my $ivshmem_name = $ivshmem->{name} // $vmid;
4087         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4088
4089         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4090         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4091             .",size=$ivshmem->{size}M";
4092     }
4093
4094     # pci.4 is nested in pci.1
4095     $bridges->{1} = 1 if $bridges->{4};
4096
4097     if (!$q35) { # add pci bridges
4098         if (min_version($machine_version, 2, 3)) {
4099            $bridges->{1} = 1;
4100            $bridges->{2} = 1;
4101         }
4102         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4103     }
4104
4105     for my $k (sort {$b cmp $a} keys %$bridges) {
4106         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4107
4108         my $k_name = $k;
4109         if ($k == 2 && $legacy_igd) {
4110             $k_name = "$k-igd";
4111         }
4112         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4113         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4114
4115         if ($q35) { # add after -readconfig pve-q35.cfg
4116             splice @$devices, 2, 0, '-device', $devstr;
4117         } else {
4118             unshift @$devices, '-device', $devstr if $k > 0;
4119         }
4120     }
4121
4122     if (!$kvm) {
4123         push @$machineFlags, 'accel=tcg';
4124     }
4125
4126     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4127
4128     my $machine_type_min = $machine_type;
4129     if ($add_pve_version) {
4130         $machine_type_min =~ s/\+pve\d+$//;
4131         $machine_type_min .= "+pve$required_pve_version";
4132     }
4133     push @$machineFlags, "type=${machine_type_min}";
4134
4135     push @$cmd, @$devices;
4136     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4137     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4138     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4139
4140     if (my $vmstate = $conf->{vmstate}) {
4141         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4142         push @$vollist, $vmstate;
4143         push @$cmd, '-loadstate', $statepath;
4144         print "activating and using '$vmstate' as vmstate\n";
4145     }
4146
4147     if (PVE::QemuConfig->is_template($conf)) {
4148         # needed to workaround base volumes being read-only
4149         push @$cmd, '-snapshot';
4150     }
4151
4152     # add custom args
4153     if ($conf->{args}) {
4154         my $aa = PVE::Tools::split_args($conf->{args});
4155         push @$cmd, @$aa;
4156     }
4157
4158     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4159 }
4160
4161 sub check_rng_source {
4162     my ($source) = @_;
4163
4164     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4165     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4166         if ! -e $source;
4167
4168     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4169     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4170         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4171         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4172         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4173             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4174             ." to the host.\n";
4175     }
4176 }
4177
4178 sub spice_port {
4179     my ($vmid) = @_;
4180
4181     my $res = mon_cmd($vmid, 'query-spice');
4182
4183     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4184 }
4185
4186 sub vm_devices_list {
4187     my ($vmid) = @_;
4188
4189     my $res = mon_cmd($vmid, 'query-pci');
4190     my $devices_to_check = [];
4191     my $devices = {};
4192     foreach my $pcibus (@$res) {
4193         push @$devices_to_check, @{$pcibus->{devices}},
4194     }
4195
4196     while (@$devices_to_check) {
4197         my $to_check = [];
4198         for my $d (@$devices_to_check) {
4199             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4200             next if !$d->{'pci_bridge'};
4201
4202             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4203             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4204         }
4205         $devices_to_check = $to_check;
4206     }
4207
4208     my $resblock = mon_cmd($vmid, 'query-block');
4209     foreach my $block (@$resblock) {
4210         if($block->{device} =~ m/^drive-(\S+)/){
4211                 $devices->{$1} = 1;
4212         }
4213     }
4214
4215     my $resmice = mon_cmd($vmid, 'query-mice');
4216     foreach my $mice (@$resmice) {
4217         if ($mice->{name} eq 'QEMU HID Tablet') {
4218             $devices->{tablet} = 1;
4219             last;
4220         }
4221     }
4222
4223     # for usb devices there is no query-usb
4224     # but we can iterate over the entries in
4225     # qom-list path=/machine/peripheral
4226     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4227     foreach my $per (@$resperipheral) {
4228         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4229             $devices->{$per->{name}} = 1;
4230         }
4231     }
4232
4233     return $devices;
4234 }
4235
4236 sub vm_deviceplug {
4237     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4238
4239     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4240
4241     my $devices_list = vm_devices_list($vmid);
4242     return 1 if defined($devices_list->{$deviceid});
4243
4244     # add PCI bridge if we need it for the device
4245     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4246
4247     if ($deviceid eq 'tablet') {
4248         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4249     } elsif ($deviceid eq 'keyboard') {
4250         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4251     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4252         my $id = $1;
4253         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4254         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4255     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4256         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4257     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4258         qemu_iothread_add($vmid, $deviceid, $device);
4259
4260         qemu_driveadd($storecfg, $vmid, $device);
4261         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4262
4263         qemu_deviceadd($vmid, $devicefull);
4264         eval { qemu_deviceaddverify($vmid, $deviceid); };
4265         if (my $err = $@) {
4266             eval { qemu_drivedel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4271         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4272         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4273         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4274
4275         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4276
4277         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4278             qemu_iothread_add($vmid, $deviceid, $device);
4279             $devicefull .= ",iothread=iothread-$deviceid";
4280         }
4281
4282         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4283             $devicefull .= ",num_queues=$device->{queues}";
4284         }
4285
4286         qemu_deviceadd($vmid, $devicefull);
4287         qemu_deviceaddverify($vmid, $deviceid);
4288     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4289         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4290         qemu_driveadd($storecfg, $vmid, $device);
4291
4292         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4293         eval { qemu_deviceadd($vmid, $devicefull); };
4294         if (my $err = $@) {
4295             eval { qemu_drivedel($vmid, $deviceid); };
4296             warn $@ if $@;
4297             die $err;
4298         }
4299     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4300         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4301
4302         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4303         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4304         my $use_old_bios_files = undef;
4305         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4306
4307         my $netdevicefull = print_netdevice_full(
4308             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4309         qemu_deviceadd($vmid, $netdevicefull);
4310         eval {
4311             qemu_deviceaddverify($vmid, $deviceid);
4312             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4313         };
4314         if (my $err = $@) {
4315             eval { qemu_netdevdel($vmid, $deviceid); };
4316             warn $@ if $@;
4317             die $err;
4318         }
4319     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4320         my $bridgeid = $2;
4321         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4322         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4323
4324         qemu_deviceadd($vmid, $devicefull);
4325         qemu_deviceaddverify($vmid, $deviceid);
4326     } else {
4327         die "can't hotplug device '$deviceid'\n";
4328     }
4329
4330     return 1;
4331 }
4332
4333 # fixme: this should raise exceptions on error!
4334 sub vm_deviceunplug {
4335     my ($vmid, $conf, $deviceid) = @_;
4336
4337     my $devices_list = vm_devices_list($vmid);
4338     return 1 if !defined($devices_list->{$deviceid});
4339
4340     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4341     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4342
4343     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4344         qemu_devicedel($vmid, $deviceid);
4345     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348     } elsif ($deviceid =~ m/^usb\d+$/) {
4349         qemu_devicedel($vmid, $deviceid);
4350         qemu_devicedelverify($vmid, $deviceid);
4351     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4352         my $device = parse_drive($deviceid, $conf->{$deviceid});
4353
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356         qemu_drivedel($vmid, $deviceid);
4357         qemu_iothread_del($vmid, $deviceid, $device);
4358     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4359         qemu_devicedel($vmid, $deviceid);
4360         qemu_devicedelverify($vmid, $deviceid);
4361     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4362         my $device = parse_drive($deviceid, $conf->{$deviceid});
4363
4364         qemu_devicedel($vmid, $deviceid);
4365         qemu_devicedelverify($vmid, $deviceid);
4366         qemu_drivedel($vmid, $deviceid);
4367         qemu_deletescsihw($conf, $vmid, $deviceid);
4368
4369         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4370             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4371     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4372         qemu_devicedel($vmid, $deviceid);
4373         qemu_devicedelverify($vmid, $deviceid);
4374         qemu_netdevdel($vmid, $deviceid);
4375     } else {
4376         die "can't unplug device '$deviceid'\n";
4377     }
4378
4379     return 1;
4380 }
4381
4382 sub qemu_spice_usbredir_chardev_add {
4383     my ($vmid, $id) = @_;
4384
4385     mon_cmd($vmid, "chardev-add" , (
4386         id => $id,
4387         backend => {
4388             type => 'spicevmc',
4389             data => {
4390                 type => "usbredir",
4391             },
4392         },
4393     ));
4394 }
4395
4396 sub qemu_deviceadd {
4397     my ($vmid, $devicefull) = @_;
4398
4399     $devicefull = "driver=".$devicefull;
4400     my %options =  split(/[=,]/, $devicefull);
4401
4402     mon_cmd($vmid, "device_add" , %options);
4403 }
4404
4405 sub qemu_devicedel {
4406     my ($vmid, $deviceid) = @_;
4407
4408     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4409 }
4410
4411 sub qemu_iothread_add {
4412     my ($vmid, $deviceid, $device) = @_;
4413
4414     if ($device->{iothread}) {
4415         my $iothreads = vm_iothreads_list($vmid);
4416         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4417     }
4418 }
4419
4420 sub qemu_iothread_del {
4421     my ($vmid, $deviceid, $device) = @_;
4422
4423     if ($device->{iothread}) {
4424         my $iothreads = vm_iothreads_list($vmid);
4425         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4426     }
4427 }
4428
4429 sub qemu_objectadd {
4430     my ($vmid, $objectid, $qomtype) = @_;
4431
4432     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4433
4434     return 1;
4435 }
4436
4437 sub qemu_objectdel {
4438     my ($vmid, $objectid) = @_;
4439
4440     mon_cmd($vmid, "object-del", id => $objectid);
4441
4442     return 1;
4443 }
4444
4445 sub qemu_driveadd {
4446     my ($storecfg, $vmid, $device) = @_;
4447
4448     my $kvmver = get_running_qemu_version($vmid);
4449     my $io_uring = min_version($kvmver, 6, 0);
4450     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4451     $drive =~ s/\\/\\\\/g;
4452     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4453
4454     # If the command succeeds qemu prints: "OK"
4455     return 1 if $ret =~ m/OK/s;
4456
4457     die "adding drive failed: $ret\n";
4458 }
4459
4460 sub qemu_drivedel {
4461     my ($vmid, $deviceid) = @_;
4462
4463     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4464     $ret =~ s/^\s+//;
4465
4466     return 1 if $ret eq "";
4467
4468     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4469     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4470
4471     die "deleting drive $deviceid failed : $ret\n";
4472 }
4473
4474 sub qemu_deviceaddverify {
4475     my ($vmid, $deviceid) = @_;
4476
4477     for (my $i = 0; $i <= 5; $i++) {
4478          my $devices_list = vm_devices_list($vmid);
4479          return 1 if defined($devices_list->{$deviceid});
4480          sleep 1;
4481     }
4482
4483     die "error on hotplug device '$deviceid'\n";
4484 }
4485
4486
4487 sub qemu_devicedelverify {
4488     my ($vmid, $deviceid) = @_;
4489
4490     # need to verify that the device is correctly removed as device_del
4491     # is async and empty return is not reliable
4492
4493     for (my $i = 0; $i <= 5; $i++) {
4494          my $devices_list = vm_devices_list($vmid);
4495          return 1 if !defined($devices_list->{$deviceid});
4496          sleep 1;
4497     }
4498
4499     die "error on hot-unplugging device '$deviceid'\n";
4500 }
4501
4502 sub qemu_findorcreatescsihw {
4503     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4504
4505     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4506
4507     my $scsihwid="$controller_prefix$controller";
4508     my $devices_list = vm_devices_list($vmid);
4509
4510     if (!defined($devices_list->{$scsihwid})) {
4511         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4512     }
4513
4514     return 1;
4515 }
4516
4517 sub qemu_deletescsihw {
4518     my ($conf, $vmid, $opt) = @_;
4519
4520     my $device = parse_drive($opt, $conf->{$opt});
4521
4522     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4523         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4524         return 1;
4525     }
4526
4527     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4528
4529     my $devices_list = vm_devices_list($vmid);
4530     foreach my $opt (keys %{$devices_list}) {
4531         if (is_valid_drivename($opt)) {
4532             my $drive = parse_drive($opt, $conf->{$opt});
4533             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4534                 return 1;
4535             }
4536         }
4537     }
4538
4539     my $scsihwid="scsihw$controller";
4540
4541     vm_deviceunplug($vmid, $conf, $scsihwid);
4542
4543     return 1;
4544 }
4545
4546 sub qemu_add_pci_bridge {
4547     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4548
4549     my $bridges = {};
4550
4551     my $bridgeid;
4552
4553     print_pci_addr($device, $bridges, $arch, $machine_type);
4554
4555     while (my ($k, $v) = each %$bridges) {
4556         $bridgeid = $k;
4557     }
4558     return 1 if !defined($bridgeid) || $bridgeid < 1;
4559
4560     my $bridge = "pci.$bridgeid";
4561     my $devices_list = vm_devices_list($vmid);
4562
4563     if (!defined($devices_list->{$bridge})) {
4564         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4565     }
4566
4567     return 1;
4568 }
4569
4570 sub qemu_set_link_status {
4571     my ($vmid, $device, $up) = @_;
4572
4573     mon_cmd($vmid, "set_link", name => $device,
4574                up => $up ? JSON::true : JSON::false);
4575 }
4576
4577 sub qemu_netdevadd {
4578     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4579
4580     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4581     my %options =  split(/[=,]/, $netdev);
4582
4583     if (defined(my $vhost = $options{vhost})) {
4584         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4585     }
4586
4587     if (defined(my $queues = $options{queues})) {
4588         $options{queues} = $queues + 0;
4589     }
4590
4591     mon_cmd($vmid, "netdev_add",  %options);
4592     return 1;
4593 }
4594
4595 sub qemu_netdevdel {
4596     my ($vmid, $deviceid) = @_;
4597
4598     mon_cmd($vmid, "netdev_del", id => $deviceid);
4599 }
4600
4601 sub qemu_usb_hotplug {
4602     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4603
4604     return if !$device;
4605
4606     # remove the old one first
4607     vm_deviceunplug($vmid, $conf, $deviceid);
4608
4609     # check if xhci controller is necessary and available
4610     my $devicelist = vm_devices_list($vmid);
4611
4612     if (!$devicelist->{xhci}) {
4613         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4614         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4615     }
4616
4617     # print_usbdevice_full expects the parsed device
4618     my $d = parse_usb_device($device->{host});
4619     $d->{usb3} = $device->{usb3};
4620
4621     # add the new one
4622     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4623 }
4624
4625 sub qemu_cpu_hotplug {
4626     my ($vmid, $conf, $vcpus) = @_;
4627
4628     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4629
4630     my $sockets = 1;
4631     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4632     $sockets = $conf->{sockets} if  $conf->{sockets};
4633     my $cores = $conf->{cores} || 1;
4634     my $maxcpus = $sockets * $cores;
4635
4636     $vcpus = $maxcpus if !$vcpus;
4637
4638     die "you can't add more vcpus than maxcpus\n"
4639         if $vcpus > $maxcpus;
4640
4641     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4642
4643     if ($vcpus < $currentvcpus) {
4644
4645         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4646
4647             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4648                 qemu_devicedel($vmid, "cpu$i");
4649                 my $retry = 0;
4650                 my $currentrunningvcpus = undef;
4651                 while (1) {
4652                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4653                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4654                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4655                     $retry++;
4656                     sleep 1;
4657                 }
4658                 #update conf after each succesfull cpu unplug
4659                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4660                 PVE::QemuConfig->write_config($vmid, $conf);
4661             }
4662         } else {
4663             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4664         }
4665
4666         return;
4667     }
4668
4669     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4670     die "vcpus in running vm does not match its configuration\n"
4671         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4672
4673     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4674
4675         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4676             my $cpustr = print_cpu_device($conf, $i);
4677             qemu_deviceadd($vmid, $cpustr);
4678
4679             my $retry = 0;
4680             my $currentrunningvcpus = undef;
4681             while (1) {
4682                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4683                 last if scalar(@{$currentrunningvcpus}) == $i;
4684                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4685                 sleep 1;
4686                 $retry++;
4687             }
4688             #update conf after each succesfull cpu hotplug
4689             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4690             PVE::QemuConfig->write_config($vmid, $conf);
4691         }
4692     } else {
4693
4694         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4695             mon_cmd($vmid, "cpu-add", id => int($i));
4696         }
4697     }
4698 }
4699
4700 sub qemu_block_set_io_throttle {
4701     my ($vmid, $deviceid,
4702         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4703         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4704         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4705         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4706
4707     return if !check_running($vmid) ;
4708
4709     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4710         bps => int($bps),
4711         bps_rd => int($bps_rd),
4712         bps_wr => int($bps_wr),
4713         iops => int($iops),
4714         iops_rd => int($iops_rd),
4715         iops_wr => int($iops_wr),
4716         bps_max => int($bps_max),
4717         bps_rd_max => int($bps_rd_max),
4718         bps_wr_max => int($bps_wr_max),
4719         iops_max => int($iops_max),
4720         iops_rd_max => int($iops_rd_max),
4721         iops_wr_max => int($iops_wr_max),
4722         bps_max_length => int($bps_max_length),
4723         bps_rd_max_length => int($bps_rd_max_length),
4724         bps_wr_max_length => int($bps_wr_max_length),
4725         iops_max_length => int($iops_max_length),
4726         iops_rd_max_length => int($iops_rd_max_length),
4727         iops_wr_max_length => int($iops_wr_max_length),
4728     );
4729
4730 }
4731
4732 sub qemu_block_resize {
4733     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4734
4735     my $running = check_running($vmid);
4736
4737     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4738
4739     return if !$running;
4740
4741     my $padding = (1024 - $size % 1024) % 1024;
4742     $size = $size + $padding;
4743
4744     mon_cmd(
4745         $vmid,
4746         "block_resize",
4747         device => $deviceid,
4748         size => int($size),
4749         timeout => 60,
4750     );
4751 }
4752
4753 sub qemu_volume_snapshot {
4754     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4759         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4760     } else {
4761         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4762     }
4763 }
4764
4765 sub qemu_volume_snapshot_delete {
4766     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4767
4768     my $running = check_running($vmid);
4769
4770     if($running) {
4771
4772         $running = undef;
4773         my $conf = PVE::QemuConfig->load_config($vmid);
4774         PVE::QemuConfig->foreach_volume($conf, sub {
4775             my ($ds, $drive) = @_;
4776             $running = 1 if $drive->{file} eq $volid;
4777         });
4778     }
4779
4780     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4781         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4782     } else {
4783         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4784     }
4785 }
4786
4787 sub set_migration_caps {
4788     my ($vmid, $savevm) = @_;
4789
4790     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4791
4792     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4793     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4794
4795     my $cap_ref = [];
4796
4797     my $enabled_cap = {
4798         "auto-converge" => 1,
4799         "xbzrle" => 1,
4800         "x-rdma-pin-all" => 0,
4801         "zero-blocks" => 0,
4802         "compress" => 0,
4803         "dirty-bitmaps" => $dirty_bitmaps,
4804     };
4805
4806     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4807
4808     for my $supported_capability (@$supported_capabilities) {
4809         push @$cap_ref, {
4810             capability => $supported_capability->{capability},
4811             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4812         };
4813     }
4814
4815     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4816 }
4817
4818 sub foreach_volid {
4819     my ($conf, $func, @param) = @_;
4820
4821     my $volhash = {};
4822
4823     my $test_volid = sub {
4824         my ($key, $drive, $snapname) = @_;
4825
4826         my $volid = $drive->{file};
4827         return if !$volid;
4828
4829         $volhash->{$volid}->{cdrom} //= 1;
4830         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4831
4832         my $replicate = $drive->{replicate} // 1;
4833         $volhash->{$volid}->{replicate} //= 0;
4834         $volhash->{$volid}->{replicate} = 1 if $replicate;
4835
4836         $volhash->{$volid}->{shared} //= 0;
4837         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4838
4839         $volhash->{$volid}->{referenced_in_config} //= 0;
4840         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4841
4842         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4843             if defined($snapname);
4844
4845         my $size = $drive->{size};
4846         $volhash->{$volid}->{size} //= $size if $size;
4847
4848         $volhash->{$volid}->{is_vmstate} //= 0;
4849         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4850
4851         $volhash->{$volid}->{is_tpmstate} //= 0;
4852         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4853
4854         $volhash->{$volid}->{is_unused} //= 0;
4855         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4856
4857         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4858     };
4859
4860     my $include_opts = {
4861         extra_keys => ['vmstate'],
4862         include_unused => 1,
4863     };
4864
4865     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4866     foreach my $snapname (keys %{$conf->{snapshots}}) {
4867         my $snap = $conf->{snapshots}->{$snapname};
4868         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4869     }
4870
4871     foreach my $volid (keys %$volhash) {
4872         &$func($volid, $volhash->{$volid}, @param);
4873     }
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886     'tags' => 1,
4887 };
4888
4889 for my $opt (keys %$confdesc_cloudinit) {
4890     $fast_plug_option->{$opt} = 1;
4891 };
4892
4893 # hotplug changes in [PENDING]
4894 # $selection hash can be used to only apply specified options, for
4895 # example: { cores => 1 } (only apply changed 'cores')
4896 # $errors ref is used to return error messages
4897 sub vmconfig_hotplug_pending {
4898     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4899
4900     my $defaults = load_defaults();
4901     my $arch = get_vm_arch($conf);
4902     my $machine_type = get_vm_machine($conf, undef, $arch);
4903
4904     # commit values which do not have any impact on running VM first
4905     # Note: those option cannot raise errors, we we do not care about
4906     # $selection and always apply them.
4907
4908     my $add_error = sub {
4909         my ($opt, $msg) = @_;
4910         $errors->{$opt} = "hotplug problem - $msg";
4911     };
4912
4913     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4914
4915     my $cloudinit_record_changed = sub {
4916         my ($conf, $opt, $old, $new) = @_;
4917         return if !$cloudinit_pending_properties->{$opt};
4918
4919         my $ci = ($conf->{cloudinit} //= {});
4920
4921         my $recorded = $ci->{$opt};
4922         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4923
4924         if (defined($new)) {
4925             if (defined($old)) {
4926                 # an existing value is being modified
4927                 if (defined($recorded)) {
4928                     # the value was already not in sync
4929                     if ($new eq $recorded) {
4930                         # a value is being reverted to the cloud-init state:
4931                         delete $ci->{$opt};
4932                         delete $added{$opt};
4933                     } else {
4934                         # the value was changed multiple times, do nothing
4935                     }
4936                 } elsif ($added{$opt}) {
4937                     # the value had been marked as added and is being changed, do nothing
4938                 } else {
4939                     # the value is new, record it:
4940                     $ci->{$opt} = $old;
4941                 }
4942             } else {
4943                 # a new value is being added
4944                 if (defined($recorded)) {
4945                     # it was already not in sync
4946                     if ($new eq $recorded) {
4947                         # a value is being reverted to the cloud-init state:
4948                         delete $ci->{$opt};
4949                         delete $added{$opt};
4950                     } else {
4951                         # the value had temporarily been removed, do nothing
4952                     }
4953                 } elsif ($added{$opt}) {
4954                     # the value had been marked as added already, do nothing
4955                 } else {
4956                     # the value is new, add it
4957                     $added{$opt} = 1;
4958                 }
4959             }
4960         } elsif (!defined($old)) {
4961             # a non-existent value is being removed? ignore...
4962         } else {
4963             # a value is being deleted
4964             if (defined($recorded)) {
4965                 # a value was already recorded, just keep it
4966             } elsif ($added{$opt}) {
4967                 # the value was marked as added, remove it
4968                 delete $added{$opt};
4969             } else {
4970                 # a previously unrecorded value is being removed, record the old value:
4971                 $ci->{$opt} = $old;
4972             }
4973         }
4974
4975         my $added = join(',', sort keys %added);
4976         $ci->{added} = $added if length($added);
4977     };
4978
4979     my $changes = 0;
4980     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4981         if ($fast_plug_option->{$opt}) {
4982             my $new = delete $conf->{pending}->{$opt};
4983             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4984             $conf->{$opt} = $new;
4985             $changes = 1;
4986         }
4987     }
4988
4989     if ($changes) {
4990         PVE::QemuConfig->write_config($vmid, $conf);
4991     }
4992
4993     my $ostype = $conf->{ostype};
4994     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4995     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4996     my $usb_hotplug = $hotplug_features->{usb}
4997         && min_version($version, 7, 1)
4998         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4999
5000     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5001     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5002
5003     foreach my $opt (sort keys %$pending_delete_hash) {
5004         next if $selection && !$selection->{$opt};
5005         my $force = $pending_delete_hash->{$opt}->{force};
5006         eval {
5007             if ($opt eq 'hotplug') {
5008                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5009             } elsif ($opt eq 'tablet') {
5010                 die "skip\n" if !$hotplug_features->{usb};
5011                 if ($defaults->{tablet}) {
5012                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5013                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5014                         if $arch eq 'aarch64';
5015                 } else {
5016                     vm_deviceunplug($vmid, $conf, 'tablet');
5017                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5018                 }
5019             } elsif ($opt =~ m/^usb(\d+)$/) {
5020                 my $index = $1;
5021                 die "skip\n" if !$usb_hotplug;
5022                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5023                 vm_deviceunplug($vmid, $conf, $opt);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, undef);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable balloon device is not hotpluggable
5029                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5030                 # here we reset the ballooning value to memory
5031                 my $balloon = $conf->{memory} || $defaults->{memory};
5032                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5033             } elsif ($fast_plug_option->{$opt}) {
5034                 # do nothing
5035             } elsif ($opt =~ m/^net(\d+)$/) {
5036                 die "skip\n" if !$hotplug_features->{network};
5037                 vm_deviceunplug($vmid, $conf, $opt);
5038             } elsif (is_valid_drivename($opt)) {
5039                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5040                 vm_deviceunplug($vmid, $conf, $opt);
5041                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5042             } elsif ($opt =~ m/^memory$/) {
5043                 die "skip\n" if !$hotplug_features->{memory};
5044                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5045             } elsif ($opt eq 'cpuunits') {
5046                 $cgroup->change_cpu_shares(undef);
5047             } elsif ($opt eq 'cpulimit') {
5048                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5049             } else {
5050                 die "skip\n";
5051             }
5052         };
5053         if (my $err = $@) {
5054             &$add_error($opt, $err) if $err ne "skip\n";
5055         } else {
5056             my $old = delete $conf->{$opt};
5057             $cloudinit_record_changed->($conf, $opt, $old, undef);
5058             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5059         }
5060     }
5061
5062     my $cloudinit_opt;
5063     foreach my $opt (keys %{$conf->{pending}}) {
5064         next if $selection && !$selection->{$opt};
5065         my $value = $conf->{pending}->{$opt};
5066         eval {
5067             if ($opt eq 'hotplug') {
5068                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5069             } elsif ($opt eq 'tablet') {
5070                 die "skip\n" if !$hotplug_features->{usb};
5071                 if ($value == 1) {
5072                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5073                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5074                         if $arch eq 'aarch64';
5075                 } elsif ($value == 0) {
5076                     vm_deviceunplug($vmid, $conf, 'tablet');
5077                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5078                 }
5079             } elsif ($opt =~ m/^usb(\d+)$/) {
5080                 my $index = $1;
5081                 die "skip\n" if !$usb_hotplug;
5082                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5083                 my $id = $opt;
5084                 if ($d->{host} eq 'spice')  {
5085                     $id = "usbredirdev$index";
5086                 }
5087                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5088             } elsif ($opt eq 'vcpus') {
5089                 die "skip\n" if !$hotplug_features->{cpu};
5090                 qemu_cpu_hotplug($vmid, $conf, $value);
5091             } elsif ($opt eq 'balloon') {
5092                 # enable/disable balloning device is not hotpluggable
5093                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5094                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5095                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5096
5097                 # allow manual ballooning if shares is set to zero
5098                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5099                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5100                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5101                 }
5102             } elsif ($opt =~ m/^net(\d+)$/) {
5103                 # some changes can be done without hotplug
5104                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5105                                     $vmid, $opt, $value, $arch, $machine_type);
5106             } elsif (is_valid_drivename($opt)) {
5107                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5108                 # some changes can be done without hotplug
5109                 my $drive = parse_drive($opt, $value);
5110                 if (drive_is_cloudinit($drive)) {
5111                     $cloudinit_opt = [$opt, $drive];
5112                     # apply all the other changes first, then generate the cloudinit disk
5113                     die "skip\n";
5114                 }
5115                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5116                                      $vmid, $opt, $value, $arch, $machine_type);
5117             } elsif ($opt =~ m/^memory$/) { #dimms
5118                 die "skip\n" if !$hotplug_features->{memory};
5119                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5120             } elsif ($opt eq 'cpuunits') {
5121                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5122                 $cgroup->change_cpu_shares($new_cpuunits);
5123             } elsif ($opt eq 'cpulimit') {
5124                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5125                 $cgroup->change_cpu_quota($cpulimit, 100000);
5126             } elsif ($opt eq 'agent') {
5127                 vmconfig_update_agent($conf, $opt, $value);
5128             } else {
5129                 die "skip\n";  # skip non-hot-pluggable options
5130             }
5131         };
5132         if (my $err = $@) {
5133             &$add_error($opt, $err) if $err ne "skip\n";
5134         } else {
5135             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5136             $conf->{$opt} = $value;
5137             delete $conf->{pending}->{$opt};
5138         }
5139     }
5140
5141     if (defined($cloudinit_opt)) {
5142         my ($opt, $drive) = @$cloudinit_opt;
5143         my $value = $conf->{pending}->{$opt};
5144         eval {
5145             my $temp = {%$conf, $opt => $value};
5146             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5147             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5148                                  $vmid, $opt, $value, $arch, $machine_type);
5149         };
5150         if (my $err = $@) {
5151             &$add_error($opt, $err) if $err ne "skip\n";
5152         } else {
5153             $conf->{$opt} = $value;
5154             delete $conf->{pending}->{$opt};
5155         }
5156     }
5157
5158     # unplug xhci controller if no usb device is left
5159     if ($usb_hotplug) {
5160         my $has_usb = 0;
5161         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5162             next if !defined($conf->{"usb$i"});
5163             $has_usb = 1;
5164             last;
5165         }
5166         if (!$has_usb) {
5167             vm_deviceunplug($vmid, $conf, 'xhci');
5168         }
5169     }
5170
5171     PVE::QemuConfig->write_config($vmid, $conf);
5172
5173     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5174         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5175     }
5176 }
5177
5178 sub try_deallocate_drive {
5179     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5180
5181     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5182         my $volid = $drive->{file};
5183         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5184             my $sid = PVE::Storage::parse_volume_id($volid);
5185             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5186
5187             # check if the disk is really unused
5188             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5189                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5190             PVE::Storage::vdisk_free($storecfg, $volid);
5191             return 1;
5192         } else {
5193             # If vm is not owner of this disk remove from config
5194             return 1;
5195         }
5196     }
5197
5198     return;
5199 }
5200
5201 sub vmconfig_delete_or_detach_drive {
5202     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5203
5204     my $drive = parse_drive($opt, $conf->{$opt});
5205
5206     my $rpcenv = PVE::RPCEnvironment::get();
5207     my $authuser = $rpcenv->get_user();
5208
5209     if ($force) {
5210         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5211         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5212     } else {
5213         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5214     }
5215 }
5216
5217
5218
5219 sub vmconfig_apply_pending {
5220     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5221
5222     return if !scalar(keys %{$conf->{pending}});
5223
5224     my $add_apply_error = sub {
5225         my ($opt, $msg) = @_;
5226         my $err_msg = "unable to apply pending change $opt : $msg";
5227         $errors->{$opt} = $err_msg;
5228         warn $err_msg;
5229     };
5230
5231     # cold plug
5232
5233     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5234     foreach my $opt (sort keys %$pending_delete_hash) {
5235         my $force = $pending_delete_hash->{$opt}->{force};
5236         eval {
5237             if ($opt =~ m/^unused/) {
5238                 die "internal error";
5239             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5240                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5241             }
5242         };
5243         if (my $err = $@) {
5244             $add_apply_error->($opt, $err);
5245         } else {
5246             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5247             delete $conf->{$opt};
5248         }
5249     }
5250
5251     PVE::QemuConfig->cleanup_pending($conf);
5252
5253     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5254
5255     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5256         next if $opt eq 'delete'; # just to be sure
5257         eval {
5258             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5259                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5260             }
5261         };
5262         if (my $err = $@) {
5263             $add_apply_error->($opt, $err);
5264         } else {
5265
5266             if (is_valid_drivename($opt)) {
5267                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5268                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5269             }
5270
5271             $conf->{$opt} = delete $conf->{pending}->{$opt};
5272         }
5273     }
5274
5275     # write all changes at once to avoid unnecessary i/o
5276     PVE::QemuConfig->write_config($vmid, $conf);
5277     if ($generate_cloudinit) {
5278         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5279             # After successful generation and if there were changes to be applied, update the
5280             # config to drop the {cloudinit} entry.
5281             PVE::QemuConfig->write_config($vmid, $conf);
5282         }
5283     }
5284 }
5285
5286 sub vmconfig_update_net {
5287     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5288
5289     my $newnet = parse_net($value);
5290
5291     if ($conf->{$opt}) {
5292         my $oldnet = parse_net($conf->{$opt});
5293
5294         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5295             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5296             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5297             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5298
5299             # for non online change, we try to hot-unplug
5300             die "skip\n" if !$hotplug;
5301             vm_deviceunplug($vmid, $conf, $opt);
5302         } else {
5303
5304             die "internal error" if $opt !~ m/net(\d+)/;
5305             my $iface = "tap${vmid}i$1";
5306
5307             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5308                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5309                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5310                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5311                 PVE::Network::tap_unplug($iface);
5312
5313                 if ($have_sdn) {
5314                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5315                 } else {
5316                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5317                 }
5318             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5319                 # Rate can be applied on its own but any change above needs to
5320                 # include the rate in tap_plug since OVS resets everything.
5321                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5322             }
5323
5324             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5325                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5326             }
5327
5328             return 1;
5329         }
5330     }
5331
5332     if ($hotplug) {
5333         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5334     } else {
5335         die "skip\n";
5336     }
5337 }
5338
5339 sub vmconfig_update_agent {
5340     my ($conf, $opt, $value) = @_;
5341
5342     die "skip\n" if !$conf->{$opt};
5343
5344     my $hotplug_options = { fstrim_cloned_disks => 1 };
5345
5346     my $old_agent = parse_guest_agent($conf);
5347     my $agent = parse_guest_agent({$opt => $value});
5348
5349     for my $option (keys %$agent) { # added/changed options
5350         next if defined($hotplug_options->{$option});
5351         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5352     }
5353
5354     for my $option (keys %$old_agent) { # removed options
5355         next if defined($hotplug_options->{$option});
5356         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5357     }
5358
5359     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5360 }
5361
5362 sub vmconfig_update_disk {
5363     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5364
5365     my $drive = parse_drive($opt, $value);
5366
5367     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5368         my $media = $drive->{media} || 'disk';
5369         my $oldmedia = $old_drive->{media} || 'disk';
5370         die "unable to change media type\n" if $media ne $oldmedia;
5371
5372         if (!drive_is_cdrom($old_drive)) {
5373
5374             if ($drive->{file} ne $old_drive->{file}) {
5375
5376                 die "skip\n" if !$hotplug;
5377
5378                 # unplug and register as unused
5379                 vm_deviceunplug($vmid, $conf, $opt);
5380                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5381
5382             } else {
5383                 # update existing disk
5384
5385                 # skip non hotpluggable value
5386                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5387                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5388                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5389                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5390                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5391                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5392                     die "skip\n";
5393                 }
5394
5395                 # apply throttle
5396                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5397                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5398                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5399                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5400                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5401                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5402                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5403                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5404                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5405                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5406                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5407                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5408                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5409                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5410                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5411                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5412                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5413                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5414
5415                     qemu_block_set_io_throttle(
5416                         $vmid,"drive-$opt",
5417                         ($drive->{mbps} || 0)*1024*1024,
5418                         ($drive->{mbps_rd} || 0)*1024*1024,
5419                         ($drive->{mbps_wr} || 0)*1024*1024,
5420                          $drive->{iops} || 0,
5421                          $drive->{iops_rd} || 0,
5422                          $drive->{iops_wr} || 0,
5423                         ($drive->{mbps_max} || 0)*1024*1024,
5424                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5425                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5426                          $drive->{iops_max} || 0,
5427                          $drive->{iops_rd_max} || 0,
5428                          $drive->{iops_wr_max} || 0,
5429                          $drive->{bps_max_length} || 1,
5430                          $drive->{bps_rd_max_length} || 1,
5431                          $drive->{bps_wr_max_length} || 1,
5432                          $drive->{iops_max_length} || 1,
5433                          $drive->{iops_rd_max_length} || 1,
5434                          $drive->{iops_wr_max_length} || 1,
5435                     );
5436
5437                 }
5438
5439                 return 1;
5440             }
5441
5442         } else { # cdrom
5443
5444             if ($drive->{file} eq 'none') {
5445                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5446                 if (drive_is_cloudinit($old_drive)) {
5447                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5448                 }
5449             } else {
5450                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5451
5452                 # force eject if locked
5453                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5454
5455                 if ($path) {
5456                     mon_cmd($vmid, "blockdev-change-medium",
5457                         id => "$opt", filename => "$path");
5458                 }
5459             }
5460
5461             return 1;
5462         }
5463     }
5464
5465     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5466     # hotplug new disks
5467     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5468     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5469 }
5470
5471 sub vmconfig_update_cloudinit_drive {
5472     my ($storecfg, $conf, $vmid) = @_;
5473
5474     my $cloudinit_ds = undef;
5475     my $cloudinit_drive = undef;
5476
5477     PVE::QemuConfig->foreach_volume($conf, sub {
5478         my ($ds, $drive) = @_;
5479         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5480             $cloudinit_ds = $ds;
5481             $cloudinit_drive = $drive;
5482         }
5483     });
5484
5485     return if !$cloudinit_drive;
5486
5487     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5488         PVE::QemuConfig->write_config($vmid, $conf);
5489     }
5490
5491     my $running = PVE::QemuServer::check_running($vmid);
5492
5493     if ($running) {
5494         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5495         if ($path) {
5496             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5497             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5498         }
5499     }
5500 }
5501
5502 # called in locked context by incoming migration
5503 sub vm_migrate_get_nbd_disks {
5504     my ($storecfg, $conf, $replicated_volumes) = @_;
5505
5506     my $local_volumes = {};
5507     PVE::QemuConfig->foreach_volume($conf, sub {
5508         my ($ds, $drive) = @_;
5509
5510         return if drive_is_cdrom($drive);
5511         return if $ds eq 'tpmstate0';
5512
5513         my $volid = $drive->{file};
5514
5515         return if !$volid;
5516
5517         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5518
5519         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5520         return if $scfg->{shared};
5521
5522         # replicated disks re-use existing state via bitmap
5523         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5524         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5525     });
5526     return $local_volumes;
5527 }
5528
5529 # called in locked context by incoming migration
5530 sub vm_migrate_alloc_nbd_disks {
5531     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5532
5533     my $nbd = {};
5534     foreach my $opt (sort keys %$source_volumes) {
5535         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5536
5537         if ($use_existing) {
5538             $nbd->{$opt}->{drivestr} = print_drive($drive);
5539             $nbd->{$opt}->{volid} = $volid;
5540             $nbd->{$opt}->{replicated} = 1;
5541             next;
5542         }
5543
5544         # storage mapping + volname = regular migration
5545         # storage mapping + format = remote migration
5546         # order of precedence, filtered by whether storage supports it:
5547         # 1. explicit requested format
5548         # 2. format of current volume
5549         # 3. default format of storage
5550         if (!$storagemap->{identity}) {
5551             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5552             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5553             if (!$format || !grep { $format eq $_ } @$validFormats) {
5554                 if ($volname) {
5555                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5556                     my $fileFormat = qemu_img_format($scfg, $volname);
5557                     $format = $fileFormat
5558                         if grep { $fileFormat eq $_ } @$validFormats;
5559                 }
5560                 $format //= $defFormat;
5561             }
5562         } else {
5563             # can't happen for remote migration, so $volname is always defined
5564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5565             $format = qemu_img_format($scfg, $volname);
5566         }
5567
5568         my $size = $drive->{size} / 1024;
5569         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5570         my $newdrive = $drive;
5571         $newdrive->{format} = $format;
5572         $newdrive->{file} = $newvolid;
5573         my $drivestr = print_drive($newdrive);
5574         $nbd->{$opt}->{drivestr} = $drivestr;
5575         $nbd->{$opt}->{volid} = $newvolid;
5576     }
5577
5578     return $nbd;
5579 }
5580
5581 # see vm_start_nolock for parameters, additionally:
5582 # migrate_opts:
5583 #   storagemap = parsed storage map for allocating NBD disks
5584 sub vm_start {
5585     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5586
5587     return PVE::QemuConfig->lock_config($vmid, sub {
5588         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5589
5590         die "you can't start a vm if it's a template\n"
5591             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5592
5593         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5594         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5595
5596         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5597
5598         if ($has_backup_lock && $running) {
5599             # a backup is currently running, attempt to start the guest in the
5600             # existing QEMU instance
5601             return vm_resume($vmid);
5602         }
5603
5604         PVE::QemuConfig->check_lock($conf)
5605             if !($params->{skiplock} || $has_suspended_lock);
5606
5607         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5608
5609         die "VM $vmid already running\n" if $running;
5610
5611         if (my $storagemap = $migrate_opts->{storagemap}) {
5612             my $replicated = $migrate_opts->{replicated_volumes};
5613             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5614             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5615
5616             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5617                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5618             }
5619         }
5620
5621         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5622     });
5623 }
5624
5625
5626 # params:
5627 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5628 #   skiplock => 0/1, skip checking for config lock
5629 #   skiptemplate => 0/1, skip checking whether VM is template
5630 #   forcemachine => to force Qemu machine (rollback/migration)
5631 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5632 #   timeout => in seconds
5633 #   paused => start VM in paused state (backup)
5634 #   resume => resume from hibernation
5635 #   pbs-backing => {
5636 #      sata0 => {
5637 #         repository
5638 #         snapshot
5639 #         keyfile
5640 #         archive
5641 #      },
5642 #      virtio2 => ...
5643 #   }
5644 # migrate_opts:
5645 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5646 #   migratedfrom => source node
5647 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5648 #   network => CIDR of migration network
5649 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5650 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5651 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5652 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5653 #       contained in config
5654 sub vm_start_nolock {
5655     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5656
5657     my $statefile = $params->{statefile};
5658     my $resume = $params->{resume};
5659
5660     my $migratedfrom = $migrate_opts->{migratedfrom};
5661     my $migration_type = $migrate_opts->{type};
5662
5663     my $res = {};
5664
5665     # clean up leftover reboot request files
5666     eval { clear_reboot_request($vmid); };
5667     warn $@ if $@;
5668
5669     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5670         vmconfig_apply_pending($vmid, $conf, $storecfg);
5671         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5672     }
5673
5674     # don't regenerate the ISO if the VM is started as part of a live migration
5675     # this way we can reuse the old ISO with the correct config
5676     if (!$migratedfrom) {
5677         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5678             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5679             # $conf->{cloudinit}, so we could just not do this?
5680             # But we do it above, so for now let's be consistent.
5681             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5682         }
5683     }
5684
5685     # override offline migrated volumes, conf is out of date still
5686     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5687         for my $key (sort keys $offline_volumes->%*) {
5688             my $parsed = parse_drive($key, $conf->{$key});
5689             $parsed->{file} = $offline_volumes->{$key};
5690             $conf->{$key} = print_drive($parsed);
5691         }
5692     }
5693
5694     my $defaults = load_defaults();
5695
5696     # set environment variable useful inside network script
5697     # for remote migration the config is available on the target node!
5698     if (!$migrate_opts->{remote_node}) {
5699         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5700     }
5701
5702     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5703
5704     my $forcemachine = $params->{forcemachine};
5705     my $forcecpu = $params->{forcecpu};
5706     if ($resume) {
5707         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5708         $forcemachine = $conf->{runningmachine};
5709         $forcecpu = $conf->{runningcpu};
5710         print "Resuming suspended VM\n";
5711     }
5712
5713     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5714         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5715
5716     my $migration_ip;
5717     my $get_migration_ip = sub {
5718         my ($nodename) = @_;
5719
5720         return $migration_ip if defined($migration_ip);
5721
5722         my $cidr = $migrate_opts->{network};
5723
5724         if (!defined($cidr)) {
5725             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5726             $cidr = $dc_conf->{migration}->{network};
5727         }
5728
5729         if (defined($cidr)) {
5730             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5731
5732             die "could not get IP: no address configured on local " .
5733                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5734
5735             die "could not get IP: multiple addresses configured on local " .
5736                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5737
5738             $migration_ip = @$ips[0];
5739         }
5740
5741         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5742             if !defined($migration_ip);
5743
5744         return $migration_ip;
5745     };
5746
5747     if ($statefile) {
5748         if ($statefile eq 'tcp') {
5749             my $migrate = $res->{migrate} = { proto => 'tcp' };
5750             $migrate->{addr} = "localhost";
5751             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5752             my $nodename = nodename();
5753
5754             if (!defined($migration_type)) {
5755                 if (defined($datacenterconf->{migration}->{type})) {
5756                     $migration_type = $datacenterconf->{migration}->{type};
5757                 } else {
5758                     $migration_type = 'secure';
5759                 }
5760             }
5761
5762             if ($migration_type eq 'insecure') {
5763                 $migrate->{addr} = $get_migration_ip->($nodename);
5764                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5765             }
5766
5767             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5768             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5769             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5770             push @$cmd, '-incoming', $migrate->{uri};
5771             push @$cmd, '-S';
5772
5773         } elsif ($statefile eq 'unix') {
5774             # should be default for secure migrations as a ssh TCP forward
5775             # tunnel is not deterministic reliable ready and fails regurarly
5776             # to set up in time, so use UNIX socket forwards
5777             my $migrate = $res->{migrate} = { proto => 'unix' };
5778             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5779             unlink $migrate->{addr};
5780
5781             $migrate->{uri} = "unix:$migrate->{addr}";
5782             push @$cmd, '-incoming', $migrate->{uri};
5783             push @$cmd, '-S';
5784
5785         } elsif (-e $statefile) {
5786             push @$cmd, '-loadstate', $statefile;
5787         } else {
5788             my $statepath = PVE::Storage::path($storecfg, $statefile);
5789             push @$vollist, $statefile;
5790             push @$cmd, '-loadstate', $statepath;
5791         }
5792     } elsif ($params->{paused}) {
5793         push @$cmd, '-S';
5794     }
5795
5796     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5797
5798     my $pci_devices = {}; # host pci devices
5799     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5800         my $dev = $conf->{"hostpci$i"} or next;
5801         $pci_devices->{$i} = parse_hostpci($dev);
5802     }
5803
5804     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5805     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5806
5807     # map to a flat list of pci ids
5808     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5809
5810     # reserve all PCI IDs before actually doing anything with them
5811     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5812
5813     eval {
5814         my $uuid;
5815         for my $id (sort keys %$pci_devices) {
5816             my $d = $pci_devices->{$id};
5817             for my $dev ($d->{pciid}->@*) {
5818                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5819
5820                 # nvidia grid needs the uuid of the mdev as qemu parameter
5821                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5822                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5823                 }
5824             }
5825         }
5826         push @$cmd, '-uuid', $uuid if defined($uuid);
5827     };
5828     if (my $err = $@) {
5829         eval { cleanup_pci_devices($vmid, $conf) };
5830         warn $@ if $@;
5831         die $err;
5832     }
5833
5834     PVE::Storage::activate_volumes($storecfg, $vollist);
5835
5836     eval {
5837         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5838     };
5839     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5840     # timeout should be more than enough here...
5841     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5842
5843     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5844
5845     my %run_params = (
5846         timeout => $statefile ? undef : $start_timeout,
5847         umask => 0077,
5848         noerr => 1,
5849     );
5850
5851     # when migrating, prefix QEMU output so other side can pick up any
5852     # errors that might occur and show the user
5853     if ($migratedfrom) {
5854         $run_params{quiet} = 1;
5855         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5856     }
5857
5858     my %systemd_properties = (
5859         Slice => 'qemu.slice',
5860         KillMode => 'process',
5861         SendSIGKILL => 0,
5862         TimeoutStopUSec => ULONG_MAX, # infinity
5863     );
5864
5865     if (PVE::CGroup::cgroup_mode() == 2) {
5866         $systemd_properties{CPUWeight} = $cpuunits;
5867     } else {
5868         $systemd_properties{CPUShares} = $cpuunits;
5869     }
5870
5871     if (my $cpulimit = $conf->{cpulimit}) {
5872         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5873     }
5874     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5875
5876     my $run_qemu = sub {
5877         PVE::Tools::run_fork sub {
5878             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5879
5880             my $tpmpid;
5881             if (my $tpm = $conf->{tpmstate0}) {
5882                 # start the TPM emulator so QEMU can connect on start
5883                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5884             }
5885
5886             my $exitcode = run_command($cmd, %run_params);
5887             if ($exitcode) {
5888                 if ($tpmpid) {
5889                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5890                     kill 'TERM', $tpmpid;
5891                 }
5892                 die "QEMU exited with code $exitcode\n";
5893             }
5894         };
5895     };
5896
5897     if ($conf->{hugepages}) {
5898
5899         my $code = sub {
5900             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5901             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5902
5903             PVE::QemuServer::Memory::hugepages_mount();
5904             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5905
5906             eval { $run_qemu->() };
5907             if (my $err = $@) {
5908                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5909                     if !$conf->{keephugepages};
5910                 die $err;
5911             }
5912
5913             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5914                 if !$conf->{keephugepages};
5915         };
5916         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5917
5918     } else {
5919         eval { $run_qemu->() };
5920     }
5921
5922     if (my $err = $@) {
5923         # deactivate volumes if start fails
5924         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5925         warn $@ if $@;
5926         eval { cleanup_pci_devices($vmid, $conf) };
5927         warn $@ if $@;
5928
5929         die "start failed: $err";
5930     }
5931
5932     # re-reserve all PCI IDs now that we can know the actual VM PID
5933     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5934     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5935     warn $@ if $@;
5936
5937     if (defined($res->{migrate})) {
5938         print "migration listens on $res->{migrate}->{uri}\n";
5939     } elsif ($statefile) {
5940         eval { mon_cmd($vmid, "cont"); };
5941         warn $@ if $@;
5942     }
5943
5944     #start nbd server for storage migration
5945     if (my $nbd = $migrate_opts->{nbd}) {
5946         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5947
5948         my $migrate_storage_uri;
5949         # nbd_protocol_version > 0 for unix socket support
5950         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5951             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5952             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5953             $migrate_storage_uri = "nbd:unix:$socket_path";
5954             $res->{migrate}->{unix_sockets} = [$socket_path];
5955         } else {
5956             my $nodename = nodename();
5957             my $localip = $get_migration_ip->($nodename);
5958             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5959             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5960
5961             mon_cmd($vmid, "nbd-server-start", addr => {
5962                 type => 'inet',
5963                 data => {
5964                     host => "${localip}",
5965                     port => "${storage_migrate_port}",
5966                 },
5967             });
5968             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5969             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5970         }
5971
5972         foreach my $opt (sort keys %$nbd) {
5973             my $drivestr = $nbd->{$opt}->{drivestr};
5974             my $volid = $nbd->{$opt}->{volid};
5975             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5976             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5977             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5978             print "re-using replicated volume: $opt - $volid\n"
5979                 if $nbd->{$opt}->{replicated};
5980
5981             $res->{drives}->{$opt} = $nbd->{$opt};
5982             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5983         }
5984     }
5985
5986     if ($migratedfrom) {
5987         eval {
5988             set_migration_caps($vmid);
5989         };
5990         warn $@ if $@;
5991
5992         if ($spice_port) {
5993             print "spice listens on port $spice_port\n";
5994             $res->{spice_port} = $spice_port;
5995             if ($migrate_opts->{spice_ticket}) {
5996                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5997                     $migrate_opts->{spice_ticket});
5998                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5999             }
6000         }
6001
6002     } else {
6003         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6004             if !$statefile && $conf->{balloon};
6005
6006         foreach my $opt (keys %$conf) {
6007             next if $opt !~  m/^net\d+$/;
6008             my $nicconf = parse_net($conf->{$opt});
6009             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6010         }
6011         add_nets_bridge_fdb($conf, $vmid);
6012     }
6013
6014     mon_cmd($vmid, 'qom-set',
6015                 path => "machine/peripheral/balloon0",
6016                 property => "guest-stats-polling-interval",
6017                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6018
6019     if ($resume) {
6020         print "Resumed VM, removing state\n";
6021         if (my $vmstate = $conf->{vmstate}) {
6022             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6023             PVE::Storage::vdisk_free($storecfg, $vmstate);
6024         }
6025         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6026         PVE::QemuConfig->write_config($vmid, $conf);
6027     }
6028
6029     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6030
6031     return $res;
6032 }
6033
6034 sub vm_commandline {
6035     my ($storecfg, $vmid, $snapname) = @_;
6036
6037     my $conf = PVE::QemuConfig->load_config($vmid);
6038
6039     my ($forcemachine, $forcecpu);
6040     if ($snapname) {
6041         my $snapshot = $conf->{snapshots}->{$snapname};
6042         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6043
6044         # check for machine or CPU overrides in snapshot
6045         $forcemachine = $snapshot->{runningmachine};
6046         $forcecpu = $snapshot->{runningcpu};
6047
6048         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6049
6050         $conf = $snapshot;
6051     }
6052
6053     my $defaults = load_defaults();
6054
6055     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6056
6057     return PVE::Tools::cmd2string($cmd);
6058 }
6059
6060 sub vm_reset {
6061     my ($vmid, $skiplock) = @_;
6062
6063     PVE::QemuConfig->lock_config($vmid, sub {
6064
6065         my $conf = PVE::QemuConfig->load_config($vmid);
6066
6067         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6068
6069         mon_cmd($vmid, "system_reset");
6070     });
6071 }
6072
6073 sub get_vm_volumes {
6074     my ($conf) = @_;
6075
6076     my $vollist = [];
6077     foreach_volid($conf, sub {
6078         my ($volid, $attr) = @_;
6079
6080         return if $volid =~ m|^/|;
6081
6082         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6083         return if !$sid;
6084
6085         push @$vollist, $volid;
6086     });
6087
6088     return $vollist;
6089 }
6090
6091 sub cleanup_pci_devices {
6092     my ($vmid, $conf) = @_;
6093
6094     foreach my $key (keys %$conf) {
6095         next if $key !~ m/^hostpci(\d+)$/;
6096         my $hostpciindex = $1;
6097         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6098         my $d = parse_hostpci($conf->{$key});
6099         if ($d->{mdev}) {
6100             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6101             # don't want to break ABI just for this two liner
6102             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6103             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6104         }
6105     }
6106     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6107 }
6108
6109 sub vm_stop_cleanup {
6110     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6111
6112     eval {
6113
6114         if (!$keepActive) {
6115             my $vollist = get_vm_volumes($conf);
6116             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6117
6118             if (my $tpmdrive = $conf->{tpmstate0}) {
6119                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6120                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6121                 if ($storeid) {
6122                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6123                 }
6124             }
6125         }
6126
6127         foreach my $ext (qw(mon qmp pid vnc qga)) {
6128             unlink "/var/run/qemu-server/${vmid}.$ext";
6129         }
6130
6131         if ($conf->{ivshmem}) {
6132             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6133             # just delete it for now, VMs which have this already open do not
6134             # are affected, but new VMs will get a separated one. If this
6135             # becomes an issue we either add some sort of ref-counting or just
6136             # add a "don't delete on stop" flag to the ivshmem format.
6137             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6138         }
6139
6140         cleanup_pci_devices($vmid, $conf);
6141
6142         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6143     };
6144     warn $@ if $@; # avoid errors - just warn
6145 }
6146
6147 # call only in locked context
6148 sub _do_vm_stop {
6149     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6150
6151     my $pid = check_running($vmid, $nocheck);
6152     return if !$pid;
6153
6154     my $conf;
6155     if (!$nocheck) {
6156         $conf = PVE::QemuConfig->load_config($vmid);
6157         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6158         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6159             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6160             $timeout = $opts->{down} if $opts->{down};
6161         }
6162         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6163     }
6164
6165     eval {
6166         if ($shutdown) {
6167             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6168                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6169             } else {
6170                 mon_cmd($vmid, "system_powerdown");
6171             }
6172         } else {
6173             mon_cmd($vmid, "quit");
6174         }
6175     };
6176     my $err = $@;
6177
6178     if (!$err) {
6179         $timeout = 60 if !defined($timeout);
6180
6181         my $count = 0;
6182         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6183             $count++;
6184             sleep 1;
6185         }
6186
6187         if ($count >= $timeout) {
6188             if ($force) {
6189                 warn "VM still running - terminating now with SIGTERM\n";
6190                 kill 15, $pid;
6191             } else {
6192                 die "VM quit/powerdown failed - got timeout\n";
6193             }
6194         } else {
6195             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6196             return;
6197         }
6198     } else {
6199        if (!check_running($vmid, $nocheck)) {
6200             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6201             return;
6202        }
6203        if ($force) {
6204             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6205             kill 15, $pid;
6206         } else {
6207             die "VM quit/powerdown failed\n";
6208         }
6209     }
6210
6211     # wait again
6212     $timeout = 10;
6213
6214     my $count = 0;
6215     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6216         $count++;
6217         sleep 1;
6218     }
6219
6220     if ($count >= $timeout) {
6221         warn "VM still running - terminating now with SIGKILL\n";
6222         kill 9, $pid;
6223         sleep 1;
6224     }
6225
6226     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6227 }
6228
6229 # Note: use $nocheck to skip tests if VM configuration file exists.
6230 # We need that when migration VMs to other nodes (files already moved)
6231 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6232 sub vm_stop {
6233     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6234
6235     $force = 1 if !defined($force) && !$shutdown;
6236
6237     if ($migratedfrom){
6238         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6239         kill 15, $pid if $pid;
6240         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6241         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6242         return;
6243     }
6244
6245     PVE::QemuConfig->lock_config($vmid, sub {
6246         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6247    });
6248 }
6249
6250 sub vm_reboot {
6251     my ($vmid, $timeout) = @_;
6252
6253     PVE::QemuConfig->lock_config($vmid, sub {
6254         eval {
6255
6256             # only reboot if running, as qmeventd starts it again on a stop event
6257             return if !check_running($vmid);
6258
6259             create_reboot_request($vmid);
6260
6261             my $storecfg = PVE::Storage::config();
6262             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6263
6264         };
6265         if (my $err = $@) {
6266             # avoid that the next normal shutdown will be confused for a reboot
6267             clear_reboot_request($vmid);
6268             die $err;
6269         }
6270    });
6271 }
6272
6273 # note: if using the statestorage parameter, the caller has to check privileges
6274 sub vm_suspend {
6275     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6276
6277     my $conf;
6278     my $path;
6279     my $storecfg;
6280     my $vmstate;
6281
6282     PVE::QemuConfig->lock_config($vmid, sub {
6283
6284         $conf = PVE::QemuConfig->load_config($vmid);
6285
6286         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6287         PVE::QemuConfig->check_lock($conf)
6288             if !($skiplock || $is_backing_up);
6289
6290         die "cannot suspend to disk during backup\n"
6291             if $is_backing_up && $includestate;
6292
6293         if ($includestate) {
6294             $conf->{lock} = 'suspending';
6295             my $date = strftime("%Y-%m-%d", localtime(time()));
6296             $storecfg = PVE::Storage::config();
6297             if (!$statestorage) {
6298                 $statestorage = find_vmstate_storage($conf, $storecfg);
6299                 # check permissions for the storage
6300                 my $rpcenv = PVE::RPCEnvironment::get();
6301                 if ($rpcenv->{type} ne 'cli') {
6302                     my $authuser = $rpcenv->get_user();
6303                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6304                 }
6305             }
6306
6307
6308             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6309                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6310             $path = PVE::Storage::path($storecfg, $vmstate);
6311             PVE::QemuConfig->write_config($vmid, $conf);
6312         } else {
6313             mon_cmd($vmid, "stop");
6314         }
6315     });
6316
6317     if ($includestate) {
6318         # save vm state
6319         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6320
6321         eval {
6322             set_migration_caps($vmid, 1);
6323             mon_cmd($vmid, "savevm-start", statefile => $path);
6324             for(;;) {
6325                 my $state = mon_cmd($vmid, "query-savevm");
6326                 if (!$state->{status}) {
6327                     die "savevm not active\n";
6328                 } elsif ($state->{status} eq 'active') {
6329                     sleep(1);
6330                     next;
6331                 } elsif ($state->{status} eq 'completed') {
6332                     print "State saved, quitting\n";
6333                     last;
6334                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6335                     die "query-savevm failed with error '$state->{error}'\n"
6336                 } else {
6337                     die "query-savevm returned status '$state->{status}'\n";
6338                 }
6339             }
6340         };
6341         my $err = $@;
6342
6343         PVE::QemuConfig->lock_config($vmid, sub {
6344             $conf = PVE::QemuConfig->load_config($vmid);
6345             if ($err) {
6346                 # cleanup, but leave suspending lock, to indicate something went wrong
6347                 eval {
6348                     mon_cmd($vmid, "savevm-end");
6349                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6350                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6351                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6352                     PVE::QemuConfig->write_config($vmid, $conf);
6353                 };
6354                 warn $@ if $@;
6355                 die $err;
6356             }
6357
6358             die "lock changed unexpectedly\n"
6359                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6360
6361             mon_cmd($vmid, "quit");
6362             $conf->{lock} = 'suspended';
6363             PVE::QemuConfig->write_config($vmid, $conf);
6364         });
6365     }
6366 }
6367
6368 # $nocheck is set when called as part of a migration - in this context the
6369 # location of the config file (source or target node) is not deterministic,
6370 # since migration cannot wait for pmxcfs to process the rename
6371 sub vm_resume {
6372     my ($vmid, $skiplock, $nocheck) = @_;
6373
6374     PVE::QemuConfig->lock_config($vmid, sub {
6375         my $res = mon_cmd($vmid, 'query-status');
6376         my $resume_cmd = 'cont';
6377         my $reset = 0;
6378         my $conf;
6379         if ($nocheck) {
6380             my $vmlist = PVE::Cluster::get_vmlist();
6381             my $node;
6382             if (exists($vmlist->{ids}->{$vmid})) {
6383                 $node = $vmlist->{ids}->{$vmid}->{node};
6384             }
6385             $conf = PVE::QemuConfig->load_config($vmid, $node);
6386         } else {
6387             $conf = PVE::QemuConfig->load_config($vmid);
6388         }
6389
6390         if ($res->{status}) {
6391             return if $res->{status} eq 'running'; # job done, go home
6392             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6393             $reset = 1 if $res->{status} eq 'shutdown';
6394         }
6395
6396         if (!$nocheck) {
6397             PVE::QemuConfig->check_lock($conf)
6398                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6399         }
6400
6401         if ($reset) {
6402             # required if a VM shuts down during a backup and we get a resume
6403             # request before the backup finishes for example
6404             mon_cmd($vmid, "system_reset");
6405         }
6406
6407         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6408
6409         mon_cmd($vmid, $resume_cmd);
6410     });
6411 }
6412
6413 sub vm_sendkey {
6414     my ($vmid, $skiplock, $key) = @_;
6415
6416     PVE::QemuConfig->lock_config($vmid, sub {
6417
6418         my $conf = PVE::QemuConfig->load_config($vmid);
6419
6420         # there is no qmp command, so we use the human monitor command
6421         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6422         die $res if $res ne '';
6423     });
6424 }
6425
6426 # vzdump restore implementaion
6427
6428 sub tar_archive_read_firstfile {
6429     my $archive = shift;
6430
6431     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6432
6433     # try to detect archive type first
6434     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6435         die "unable to open file '$archive'\n";
6436     my $firstfile = <$fh>;
6437     kill 15, $pid;
6438     close $fh;
6439
6440     die "ERROR: archive contaions no data\n" if !$firstfile;
6441     chomp $firstfile;
6442
6443     return $firstfile;
6444 }
6445
6446 sub tar_restore_cleanup {
6447     my ($storecfg, $statfile) = @_;
6448
6449     print STDERR "starting cleanup\n";
6450
6451     if (my $fd = IO::File->new($statfile, "r")) {
6452         while (defined(my $line = <$fd>)) {
6453             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6454                 my $volid = $2;
6455                 eval {
6456                     if ($volid =~ m|^/|) {
6457                         unlink $volid || die 'unlink failed\n';
6458                     } else {
6459                         PVE::Storage::vdisk_free($storecfg, $volid);
6460                     }
6461                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6462                 };
6463                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6464             } else {
6465                 print STDERR "unable to parse line in statfile - $line";
6466             }
6467         }
6468         $fd->close();
6469     }
6470 }
6471
6472 sub restore_file_archive {
6473     my ($archive, $vmid, $user, $opts) = @_;
6474
6475     return restore_vma_archive($archive, $vmid, $user, $opts)
6476         if $archive eq '-';
6477
6478     my $info = PVE::Storage::archive_info($archive);
6479     my $format = $opts->{format} // $info->{format};
6480     my $comp = $info->{compression};
6481
6482     # try to detect archive format
6483     if ($format eq 'tar') {
6484         return restore_tar_archive($archive, $vmid, $user, $opts);
6485     } else {
6486         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6487     }
6488 }
6489
6490 # hepler to remove disks that will not be used after restore
6491 my $restore_cleanup_oldconf = sub {
6492     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6493
6494     my $kept_disks = {};
6495
6496     PVE::QemuConfig->foreach_volume($oldconf, sub {
6497         my ($ds, $drive) = @_;
6498
6499         return if drive_is_cdrom($drive, 1);
6500
6501         my $volid = $drive->{file};
6502         return if !$volid || $volid =~ m|^/|;
6503
6504         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6505         return if !$path || !$owner || ($owner != $vmid);
6506
6507         # Note: only delete disk we want to restore
6508         # other volumes will become unused
6509         if ($virtdev_hash->{$ds}) {
6510             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6511             if (my $err = $@) {
6512                 warn $err;
6513             }
6514         } else {
6515             $kept_disks->{$volid} = 1;
6516         }
6517     });
6518
6519     # after the restore we have no snapshots anymore
6520     for my $snapname (keys $oldconf->{snapshots}->%*) {
6521         my $snap = $oldconf->{snapshots}->{$snapname};
6522         if ($snap->{vmstate}) {
6523             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6524             if (my $err = $@) {
6525                 warn $err;
6526             }
6527         }
6528
6529         for my $volid (keys $kept_disks->%*) {
6530             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6531             warn $@ if $@;
6532         }
6533     }
6534 };
6535
6536 # Helper to parse vzdump backup device hints
6537 #
6538 # $rpcenv: Environment, used to ckeck storage permissions
6539 # $user: User ID, to check storage permissions
6540 # $storecfg: Storage configuration
6541 # $fh: the file handle for reading the configuration
6542 # $devinfo: should contain device sizes for all backu-up'ed devices
6543 # $options: backup options (pool, default storage)
6544 #
6545 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6546 my $parse_backup_hints = sub {
6547     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6548
6549     my $check_storage = sub { # assert if an image can be allocate
6550         my ($storeid, $scfg) = @_;
6551         die "Content type 'images' is not available on storage '$storeid'\n"
6552             if !$scfg->{content}->{images};
6553         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6554             if $user ne 'root@pam';
6555     };
6556
6557     my $virtdev_hash = {};
6558     while (defined(my $line = <$fh>)) {
6559         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6560             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6561             die "archive does not contain data for drive '$virtdev'\n"
6562                 if !$devinfo->{$devname};
6563
6564             if (defined($options->{storage})) {
6565                 $storeid = $options->{storage} || 'local';
6566             } elsif (!$storeid) {
6567                 $storeid = 'local';
6568             }
6569             $format = 'raw' if !$format;
6570             $devinfo->{$devname}->{devname} = $devname;
6571             $devinfo->{$devname}->{virtdev} = $virtdev;
6572             $devinfo->{$devname}->{format} = $format;
6573             $devinfo->{$devname}->{storeid} = $storeid;
6574
6575             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6576             $check_storage->($storeid, $scfg); # permission and content type check
6577
6578             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6579         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6580             my $virtdev = $1;
6581             my $drive = parse_drive($virtdev, $2);
6582
6583             if (drive_is_cloudinit($drive)) {
6584                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6585                 $storeid = $options->{storage} if defined ($options->{storage});
6586                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6587                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6588
6589                 $check_storage->($storeid, $scfg); # permission and content type check
6590
6591                 $virtdev_hash->{$virtdev} = {
6592                     format => $format,
6593                     storeid => $storeid,
6594                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6595                     is_cloudinit => 1,
6596                 };
6597             }
6598         }
6599     }
6600
6601     return $virtdev_hash;
6602 };
6603
6604 # Helper to allocate and activate all volumes required for a restore
6605 #
6606 # $storecfg: Storage configuration
6607 # $virtdev_hash: as returned by parse_backup_hints()
6608 #
6609 # Returns: { $virtdev => $volid }
6610 my $restore_allocate_devices = sub {
6611     my ($storecfg, $virtdev_hash, $vmid) = @_;
6612
6613     my $map = {};
6614     foreach my $virtdev (sort keys %$virtdev_hash) {
6615         my $d = $virtdev_hash->{$virtdev};
6616         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6617         my $storeid = $d->{storeid};
6618         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6619
6620         # test if requested format is supported
6621         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6622         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6623         $d->{format} = $defFormat if !$supported;
6624
6625         my $name;
6626         if ($d->{is_cloudinit}) {
6627             $name = "vm-$vmid-cloudinit";
6628             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6629             if ($scfg->{path}) {
6630                 $name .= ".$d->{format}";
6631             }
6632         }
6633
6634         my $volid = PVE::Storage::vdisk_alloc(
6635             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6636
6637         print STDERR "new volume ID is '$volid'\n";
6638         $d->{volid} = $volid;
6639
6640         PVE::Storage::activate_volumes($storecfg, [$volid]);
6641
6642         $map->{$virtdev} = $volid;
6643     }
6644
6645     return $map;
6646 };
6647
6648 sub restore_update_config_line {
6649     my ($cookie, $map, $line, $unique) = @_;
6650
6651     return '' if $line =~ m/^\#qmdump\#/;
6652     return '' if $line =~ m/^\#vzdump\#/;
6653     return '' if $line =~ m/^lock:/;
6654     return '' if $line =~ m/^unused\d+:/;
6655     return '' if $line =~ m/^parent:/;
6656
6657     my $res = '';
6658
6659     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6660     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6661         # try to convert old 1.X settings
6662         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6663         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6664             my ($model, $macaddr) = split(/\=/, $devconfig);
6665             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6666             my $net = {
6667                 model => $model,
6668                 bridge => "vmbr$ind",
6669                 macaddr => $macaddr,
6670             };
6671             my $netstr = print_net($net);
6672
6673             $res .= "net$cookie->{netcount}: $netstr\n";
6674             $cookie->{netcount}++;
6675         }
6676     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6677         my ($id, $netstr) = ($1, $2);
6678         my $net = parse_net($netstr);
6679         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6680         $netstr = print_net($net);
6681         $res .= "$id: $netstr\n";
6682     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6683         my $virtdev = $1;
6684         my $value = $3;
6685         my $di = parse_drive($virtdev, $value);
6686         if (defined($di->{backup}) && !$di->{backup}) {
6687             $res .= "#$line";
6688         } elsif ($map->{$virtdev}) {
6689             delete $di->{format}; # format can change on restore
6690             $di->{file} = $map->{$virtdev};
6691             $value = print_drive($di);
6692             $res .= "$virtdev: $value\n";
6693         } else {
6694             $res .= $line;
6695         }
6696     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6697         my $vmgenid = $1;
6698         if ($vmgenid ne '0') {
6699             # always generate a new vmgenid if there was a valid one setup
6700             $vmgenid = generate_uuid();
6701         }
6702         $res .= "vmgenid: $vmgenid\n";
6703     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6704         my ($uuid, $uuid_str);
6705         UUID::generate($uuid);
6706         UUID::unparse($uuid, $uuid_str);
6707         my $smbios1 = parse_smbios1($2);
6708         $smbios1->{uuid} = $uuid_str;
6709         $res .= $1.print_smbios1($smbios1)."\n";
6710     } else {
6711         $res .= $line;
6712     }
6713
6714     return $res;
6715 }
6716
6717 my $restore_deactivate_volumes = sub {
6718     my ($storecfg, $virtdev_hash) = @_;
6719
6720     my $vollist = [];
6721     for my $dev (values $virtdev_hash->%*) {
6722         push $vollist->@*, $dev->{volid} if $dev->{volid};
6723     }
6724
6725     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6726     print STDERR $@ if $@;
6727 };
6728
6729 my $restore_destroy_volumes = sub {
6730     my ($storecfg, $virtdev_hash) = @_;
6731
6732     for my $dev (values $virtdev_hash->%*) {
6733         my $volid = $dev->{volid} or next;
6734         eval {
6735             PVE::Storage::vdisk_free($storecfg, $volid);
6736             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6737         };
6738         print STDERR "unable to cleanup '$volid' - $@" if $@;
6739     }
6740 };
6741
6742 my $restore_merge_config = sub {
6743     my ($filename, $backup_conf_raw, $override_conf) = @_;
6744
6745     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6746     for my $key (keys $override_conf->%*) {
6747         $backup_conf->{$key} = $override_conf->{$key};
6748     }
6749
6750     return $backup_conf;
6751 };
6752
6753 sub scan_volids {
6754     my ($cfg, $vmid) = @_;
6755
6756     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6757
6758     my $volid_hash = {};
6759     foreach my $storeid (keys %$info) {
6760         foreach my $item (@{$info->{$storeid}}) {
6761             next if !($item->{volid} && $item->{size});
6762             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6763             $volid_hash->{$item->{volid}} = $item;
6764         }
6765     }
6766
6767     return $volid_hash;
6768 }
6769
6770 sub update_disk_config {
6771     my ($vmid, $conf, $volid_hash) = @_;
6772
6773     my $changes;
6774     my $prefix = "VM $vmid";
6775
6776     # used and unused disks
6777     my $referenced = {};
6778
6779     # Note: it is allowed to define multiple storages with same path (alias), so
6780     # we need to check both 'volid' and real 'path' (two different volid can point
6781     # to the same path).
6782
6783     my $referencedpath = {};
6784
6785     # update size info
6786     PVE::QemuConfig->foreach_volume($conf, sub {
6787         my ($opt, $drive) = @_;
6788
6789         my $volid = $drive->{file};
6790         return if !$volid;
6791         my $volume = $volid_hash->{$volid};
6792
6793         # mark volid as "in-use" for next step
6794         $referenced->{$volid} = 1;
6795         if ($volume && (my $path = $volume->{path})) {
6796             $referencedpath->{$path} = 1;
6797         }
6798
6799         return if drive_is_cdrom($drive);
6800         return if !$volume;
6801
6802         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6803         if (defined($updated)) {
6804             $changes = 1;
6805             $conf->{$opt} = print_drive($updated);
6806             print "$prefix ($opt): $msg\n";
6807         }
6808     });
6809
6810     # remove 'unusedX' entry if volume is used
6811     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6812         my ($opt, $drive) = @_;
6813
6814         my $volid = $drive->{file};
6815         return if !$volid;
6816
6817         my $path;
6818         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6819         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6820             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6821             $changes = 1;
6822             delete $conf->{$opt};
6823         }
6824
6825         $referenced->{$volid} = 1;
6826         $referencedpath->{$path} = 1 if $path;
6827     });
6828
6829     foreach my $volid (sort keys %$volid_hash) {
6830         next if $volid =~ m/vm-$vmid-state-/;
6831         next if $referenced->{$volid};
6832         my $path = $volid_hash->{$volid}->{path};
6833         next if !$path; # just to be sure
6834         next if $referencedpath->{$path};
6835         $changes = 1;
6836         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6837         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6838         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6839     }
6840
6841     return $changes;
6842 }
6843
6844 sub rescan {
6845     my ($vmid, $nolock, $dryrun) = @_;
6846
6847     my $cfg = PVE::Storage::config();
6848
6849     print "rescan volumes...\n";
6850     my $volid_hash = scan_volids($cfg, $vmid);
6851
6852     my $updatefn =  sub {
6853         my ($vmid) = @_;
6854
6855         my $conf = PVE::QemuConfig->load_config($vmid);
6856
6857         PVE::QemuConfig->check_lock($conf);
6858
6859         my $vm_volids = {};
6860         foreach my $volid (keys %$volid_hash) {
6861             my $info = $volid_hash->{$volid};
6862             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6863         }
6864
6865         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6866
6867         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6868     };
6869
6870     if (defined($vmid)) {
6871         if ($nolock) {
6872             &$updatefn($vmid);
6873         } else {
6874             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6875         }
6876     } else {
6877         my $vmlist = config_list();
6878         foreach my $vmid (keys %$vmlist) {
6879             if ($nolock) {
6880                 &$updatefn($vmid);
6881             } else {
6882                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6883             }
6884         }
6885     }
6886 }
6887
6888 sub restore_proxmox_backup_archive {
6889     my ($archive, $vmid, $user, $options) = @_;
6890
6891     my $storecfg = PVE::Storage::config();
6892
6893     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6894     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6895
6896     my $fingerprint = $scfg->{fingerprint};
6897     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6898
6899     my $repo = PVE::PBSClient::get_repository($scfg);
6900     my $namespace = $scfg->{namespace};
6901
6902     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6903     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6904     local $ENV{PBS_PASSWORD} = $password;
6905     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6906
6907     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6908         PVE::Storage::parse_volname($storecfg, $archive);
6909
6910     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6911
6912     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6913
6914     my $tmpdir = "/var/tmp/vzdumptmp$$";
6915     rmtree $tmpdir;
6916     mkpath $tmpdir;
6917
6918     my $conffile = PVE::QemuConfig->config_file($vmid);
6919      # disable interrupts (always do cleanups)
6920     local $SIG{INT} =
6921         local $SIG{TERM} =
6922         local $SIG{QUIT} =
6923         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6924
6925     # Note: $oldconf is undef if VM does not exists
6926     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6927     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6928     my $new_conf_raw = '';
6929
6930     my $rpcenv = PVE::RPCEnvironment::get();
6931     my $devinfo = {}; # info about drives included in backup
6932     my $virtdev_hash = {}; # info about allocated drives
6933
6934     eval {
6935         # enable interrupts
6936         local $SIG{INT} =
6937             local $SIG{TERM} =
6938             local $SIG{QUIT} =
6939             local $SIG{HUP} =
6940             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6941
6942         my $cfgfn = "$tmpdir/qemu-server.conf";
6943         my $firewall_config_fn = "$tmpdir/fw.conf";
6944         my $index_fn = "$tmpdir/index.json";
6945
6946         my $cmd = "restore";
6947
6948         my $param = [$pbs_backup_name, "index.json", $index_fn];
6949         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6950         my $index = PVE::Tools::file_get_contents($index_fn);
6951         $index = decode_json($index);
6952
6953         foreach my $info (@{$index->{files}}) {
6954             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6955                 my $devname = $1;
6956                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6957                     $devinfo->{$devname}->{size} = $1;
6958                 } else {
6959                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6960                 }
6961             }
6962         }
6963
6964         my $is_qemu_server_backup = scalar(
6965             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6966         );
6967         if (!$is_qemu_server_backup) {
6968             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6969         }
6970         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6971
6972         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6973         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6974
6975         if ($has_firewall_config) {
6976             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6977             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6978
6979             my $pve_firewall_dir = '/etc/pve/firewall';
6980             mkdir $pve_firewall_dir; # make sure the dir exists
6981             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6982         }
6983
6984         my $fh = IO::File->new($cfgfn, "r") ||
6985             die "unable to read qemu-server.conf - $!\n";
6986
6987         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6988
6989         # fixme: rate limit?
6990
6991         # create empty/temp config
6992         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6993
6994         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6995
6996         # allocate volumes
6997         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6998
6999         foreach my $virtdev (sort keys %$virtdev_hash) {
7000             my $d = $virtdev_hash->{$virtdev};
7001             next if $d->{is_cloudinit}; # no need to restore cloudinit
7002
7003             # this fails if storage is unavailable
7004             my $volid = $d->{volid};
7005             my $path = PVE::Storage::path($storecfg, $volid);
7006
7007             # for live-restore we only want to preload the efidisk and TPM state
7008             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7009
7010             my @ns_arg;
7011             if (defined(my $ns = $scfg->{namespace})) {
7012                 @ns_arg = ('--ns', $ns);
7013             }
7014
7015             my $pbs_restore_cmd = [
7016                 '/usr/bin/pbs-restore',
7017                 '--repository', $repo,
7018                 @ns_arg,
7019                 $pbs_backup_name,
7020                 "$d->{devname}.img.fidx",
7021                 $path,
7022                 '--verbose',
7023                 ];
7024
7025             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7026             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7027
7028             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7029                 push @$pbs_restore_cmd, '--skip-zero';
7030             }
7031
7032             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7033             print "restore proxmox backup image: $dbg_cmdstring\n";
7034             run_command($pbs_restore_cmd);
7035         }
7036
7037         $fh->seek(0, 0) || die "seek failed - $!\n";
7038
7039         my $cookie = { netcount => 0 };
7040         while (defined(my $line = <$fh>)) {
7041             $new_conf_raw .= restore_update_config_line(
7042                 $cookie,
7043                 $map,
7044                 $line,
7045                 $options->{unique},
7046             );
7047         }
7048
7049         $fh->close();
7050     };
7051     my $err = $@;
7052
7053     if ($err || !$options->{live}) {
7054         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7055     }
7056
7057     rmtree $tmpdir;
7058
7059     if ($err) {
7060         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7061         die $err;
7062     }
7063
7064     if ($options->{live}) {
7065         # keep lock during live-restore
7066         $new_conf_raw .= "\nlock: create";
7067     }
7068
7069     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7070     PVE::QemuConfig->write_config($vmid, $new_conf);
7071
7072     eval { rescan($vmid, 1); };
7073     warn $@ if $@;
7074
7075     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7076
7077     if ($options->{live}) {
7078         # enable interrupts
7079         local $SIG{INT} =
7080             local $SIG{TERM} =
7081             local $SIG{QUIT} =
7082             local $SIG{HUP} =
7083             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7084
7085         my $conf = PVE::QemuConfig->load_config($vmid);
7086         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7087
7088         # these special drives are already restored before start
7089         delete $devinfo->{'drive-efidisk0'};
7090         delete $devinfo->{'drive-tpmstate0-backup'};
7091
7092         my $pbs_opts = {
7093             repo => $repo,
7094             keyfile => $keyfile,
7095             snapshot => $pbs_backup_name,
7096             namespace => $namespace,
7097         };
7098         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7099
7100         PVE::QemuConfig->remove_lock($vmid, "create");
7101     }
7102 }
7103
7104 sub pbs_live_restore {
7105     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7106
7107     print "starting VM for live-restore\n";
7108     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7109
7110     my $pbs_backing = {};
7111     for my $ds (keys %$restored_disks) {
7112         $ds =~ m/^drive-(.*)$/;
7113         my $confname = $1;
7114         $pbs_backing->{$confname} = {
7115             repository => $opts->{repo},
7116             snapshot => $opts->{snapshot},
7117             archive => "$ds.img.fidx",
7118         };
7119         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7120         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7121
7122         my $drive = parse_drive($confname, $conf->{$confname});
7123         print "restoring '$ds' to '$drive->{file}'\n";
7124     }
7125
7126     my $drives_streamed = 0;
7127     eval {
7128         # make sure HA doesn't interrupt our restore by stopping the VM
7129         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7130             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7131         }
7132
7133         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7134         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7135         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7136
7137         my $qmeventd_fd = register_qmeventd_handle($vmid);
7138
7139         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7140         # this will effectively collapse the backing image chain consisting of
7141         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7142         # removes itself once all backing images vanish with 'auto-remove=on')
7143         my $jobs = {};
7144         for my $ds (sort keys %$restored_disks) {
7145             my $job_id = "restore-$ds";
7146             mon_cmd($vmid, 'block-stream',
7147                 'job-id' => $job_id,
7148                 device => "$ds",
7149             );
7150             $jobs->{$job_id} = {};
7151         }
7152
7153         mon_cmd($vmid, 'cont');
7154         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7155
7156         print "restore-drive jobs finished successfully, removing all tracking block devices"
7157             ." to disconnect from Proxmox Backup Server\n";
7158
7159         for my $ds (sort keys %$restored_disks) {
7160             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7161         }
7162
7163         close($qmeventd_fd);
7164     };
7165
7166     my $err = $@;
7167
7168     if ($err) {
7169         warn "An error occurred during live-restore: $err\n";
7170         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7171         die "live-restore failed\n";
7172     }
7173 }
7174
7175 sub restore_vma_archive {
7176     my ($archive, $vmid, $user, $opts, $comp) = @_;
7177
7178     my $readfrom = $archive;
7179
7180     my $cfg = PVE::Storage::config();
7181     my $commands = [];
7182     my $bwlimit = $opts->{bwlimit};
7183
7184     my $dbg_cmdstring = '';
7185     my $add_pipe = sub {
7186         my ($cmd) = @_;
7187         push @$commands, $cmd;
7188         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7189         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7190         $readfrom = '-';
7191     };
7192
7193     my $input = undef;
7194     if ($archive eq '-') {
7195         $input = '<&STDIN';
7196     } else {
7197         # If we use a backup from a PVE defined storage we also consider that
7198         # storage's rate limit:
7199         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7200         if (defined($volid)) {
7201             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7202             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7203             if ($readlimit) {
7204                 print STDERR "applying read rate limit: $readlimit\n";
7205                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7206                 $add_pipe->($cstream);
7207             }
7208         }
7209     }
7210
7211     if ($comp) {
7212         my $info = PVE::Storage::decompressor_info('vma', $comp);
7213         my $cmd = $info->{decompressor};
7214         push @$cmd, $readfrom;
7215         $add_pipe->($cmd);
7216     }
7217
7218     my $tmpdir = "/var/tmp/vzdumptmp$$";
7219     rmtree $tmpdir;
7220
7221     # disable interrupts (always do cleanups)
7222     local $SIG{INT} =
7223         local $SIG{TERM} =
7224         local $SIG{QUIT} =
7225         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7226
7227     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7228     POSIX::mkfifo($mapfifo, 0600);
7229     my $fifofh;
7230     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7231
7232     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7233
7234     my $oldtimeout;
7235     my $timeout = 5;
7236
7237     my $devinfo = {}; # info about drives included in backup
7238     my $virtdev_hash = {}; # info about allocated drives
7239
7240     my $rpcenv = PVE::RPCEnvironment::get();
7241
7242     my $conffile = PVE::QemuConfig->config_file($vmid);
7243
7244     # Note: $oldconf is undef if VM does not exist
7245     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7246     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7247     my $new_conf_raw = '';
7248
7249     my %storage_limits;
7250
7251     my $print_devmap = sub {
7252         my $cfgfn = "$tmpdir/qemu-server.conf";
7253
7254         # we can read the config - that is already extracted
7255         my $fh = IO::File->new($cfgfn, "r") ||
7256             die "unable to read qemu-server.conf - $!\n";
7257
7258         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7259         if (-f $fwcfgfn) {
7260             my $pve_firewall_dir = '/etc/pve/firewall';
7261             mkdir $pve_firewall_dir; # make sure the dir exists
7262             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7263         }
7264
7265         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7266
7267         foreach my $info (values %{$virtdev_hash}) {
7268             my $storeid = $info->{storeid};
7269             next if defined($storage_limits{$storeid});
7270
7271             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7272             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7273             $storage_limits{$storeid} = $limit * 1024;
7274         }
7275
7276         foreach my $devname (keys %$devinfo) {
7277             die "found no device mapping information for device '$devname'\n"
7278                 if !$devinfo->{$devname}->{virtdev};
7279         }
7280
7281         # create empty/temp config
7282         if ($oldconf) {
7283             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7284             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7285         }
7286
7287         # allocate volumes
7288         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7289
7290         # print restore information to $fifofh
7291         foreach my $virtdev (sort keys %$virtdev_hash) {
7292             my $d = $virtdev_hash->{$virtdev};
7293             next if $d->{is_cloudinit}; # no need to restore cloudinit
7294
7295             my $storeid = $d->{storeid};
7296             my $volid = $d->{volid};
7297
7298             my $map_opts = '';
7299             if (my $limit = $storage_limits{$storeid}) {
7300                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7301             }
7302
7303             my $write_zeros = 1;
7304             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7305                 $write_zeros = 0;
7306             }
7307
7308             my $path = PVE::Storage::path($cfg, $volid);
7309
7310             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7311
7312             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7313         }
7314
7315         $fh->seek(0, 0) || die "seek failed - $!\n";
7316
7317         my $cookie = { netcount => 0 };
7318         while (defined(my $line = <$fh>)) {
7319             $new_conf_raw .= restore_update_config_line(
7320                 $cookie,
7321                 $map,
7322                 $line,
7323                 $opts->{unique},
7324             );
7325         }
7326
7327         $fh->close();
7328     };
7329
7330     eval {
7331         # enable interrupts
7332         local $SIG{INT} =
7333             local $SIG{TERM} =
7334             local $SIG{QUIT} =
7335             local $SIG{HUP} =
7336             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7337         local $SIG{ALRM} = sub { die "got timeout\n"; };
7338
7339         $oldtimeout = alarm($timeout);
7340
7341         my $parser = sub {
7342             my $line = shift;
7343
7344             print "$line\n";
7345
7346             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7347                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7348                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7349             } elsif ($line =~ m/^CTIME: /) {
7350                 # we correctly received the vma config, so we can disable
7351                 # the timeout now for disk allocation (set to 10 minutes, so
7352                 # that we always timeout if something goes wrong)
7353                 alarm(600);
7354                 &$print_devmap();
7355                 print $fifofh "done\n";
7356                 my $tmp = $oldtimeout || 0;
7357                 $oldtimeout = undef;
7358                 alarm($tmp);
7359                 close($fifofh);
7360                 $fifofh = undef;
7361             }
7362         };
7363
7364         print "restore vma archive: $dbg_cmdstring\n";
7365         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7366     };
7367     my $err = $@;
7368
7369     alarm($oldtimeout) if $oldtimeout;
7370
7371     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7372
7373     close($fifofh) if $fifofh;
7374     unlink $mapfifo;
7375     rmtree $tmpdir;
7376
7377     if ($err) {
7378         $restore_destroy_volumes->($cfg, $virtdev_hash);
7379         die $err;
7380     }
7381
7382     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7383     PVE::QemuConfig->write_config($vmid, $new_conf);
7384
7385     eval { rescan($vmid, 1); };
7386     warn $@ if $@;
7387
7388     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7389 }
7390
7391 sub restore_tar_archive {
7392     my ($archive, $vmid, $user, $opts) = @_;
7393
7394     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7395         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7396         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7397     }
7398
7399     if ($archive ne '-') {
7400         my $firstfile = tar_archive_read_firstfile($archive);
7401         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7402             if $firstfile ne 'qemu-server.conf';
7403     }
7404
7405     my $storecfg = PVE::Storage::config();
7406
7407     # avoid zombie disks when restoring over an existing VM -> cleanup first
7408     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7409     # skiplock=1 because qmrestore has set the 'create' lock itself already
7410     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7411     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7412
7413     my $tocmd = "/usr/lib/qemu-server/qmextract";
7414
7415     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7416     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7417     $tocmd .= ' --prealloc' if $opts->{prealloc};
7418     $tocmd .= ' --info' if $opts->{info};
7419
7420     # tar option "xf" does not autodetect compression when read from STDIN,
7421     # so we pipe to zcat
7422     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7423         PVE::Tools::shellquote("--to-command=$tocmd");
7424
7425     my $tmpdir = "/var/tmp/vzdumptmp$$";
7426     mkpath $tmpdir;
7427
7428     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7429     local $ENV{VZDUMP_VMID} = $vmid;
7430     local $ENV{VZDUMP_USER} = $user;
7431
7432     my $conffile = PVE::QemuConfig->config_file($vmid);
7433     my $new_conf_raw = '';
7434
7435     # disable interrupts (always do cleanups)
7436     local $SIG{INT} =
7437         local $SIG{TERM} =
7438         local $SIG{QUIT} =
7439         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7440
7441     eval {
7442         # enable interrupts
7443         local $SIG{INT} =
7444             local $SIG{TERM} =
7445             local $SIG{QUIT} =
7446             local $SIG{HUP} =
7447             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7448
7449         if ($archive eq '-') {
7450             print "extracting archive from STDIN\n";
7451             run_command($cmd, input => "<&STDIN");
7452         } else {
7453             print "extracting archive '$archive'\n";
7454             run_command($cmd);
7455         }
7456
7457         return if $opts->{info};
7458
7459         # read new mapping
7460         my $map = {};
7461         my $statfile = "$tmpdir/qmrestore.stat";
7462         if (my $fd = IO::File->new($statfile, "r")) {
7463             while (defined (my $line = <$fd>)) {
7464                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7465                     $map->{$1} = $2 if $1;
7466                 } else {
7467                     print STDERR "unable to parse line in statfile - $line\n";
7468                 }
7469             }
7470             $fd->close();
7471         }
7472
7473         my $confsrc = "$tmpdir/qemu-server.conf";
7474
7475         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7476
7477         my $cookie = { netcount => 0 };
7478         while (defined (my $line = <$srcfd>)) {
7479             $new_conf_raw .= restore_update_config_line(
7480                 $cookie,
7481                 $map,
7482                 $line,
7483                 $opts->{unique},
7484             );
7485         }
7486
7487         $srcfd->close();
7488     };
7489     if (my $err = $@) {
7490         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7491         die $err;
7492     }
7493
7494     rmtree $tmpdir;
7495
7496     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7497
7498     PVE::Cluster::cfs_update(); # make sure we read new file
7499
7500     eval { rescan($vmid, 1); };
7501     warn $@ if $@;
7502 };
7503
7504 sub foreach_storage_used_by_vm {
7505     my ($conf, $func) = @_;
7506
7507     my $sidhash = {};
7508
7509     PVE::QemuConfig->foreach_volume($conf, sub {
7510         my ($ds, $drive) = @_;
7511         return if drive_is_cdrom($drive);
7512
7513         my $volid = $drive->{file};
7514
7515         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7516         $sidhash->{$sid} = $sid if $sid;
7517     });
7518
7519     foreach my $sid (sort keys %$sidhash) {
7520         &$func($sid);
7521     }
7522 }
7523
7524 my $qemu_snap_storage = {
7525     rbd => 1,
7526 };
7527 sub do_snapshots_with_qemu {
7528     my ($storecfg, $volid, $deviceid) = @_;
7529
7530     return if $deviceid =~ m/tpmstate0/;
7531
7532     my $storage_name = PVE::Storage::parse_volume_id($volid);
7533     my $scfg = $storecfg->{ids}->{$storage_name};
7534     die "could not find storage '$storage_name'\n" if !defined($scfg);
7535
7536     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7537         return 1;
7538     }
7539
7540     if ($volid =~ m/\.(qcow2|qed)$/){
7541         return 1;
7542     }
7543
7544     return;
7545 }
7546
7547 sub qga_check_running {
7548     my ($vmid, $nowarn) = @_;
7549
7550     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7551     if ($@) {
7552         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7553         return 0;
7554     }
7555     return 1;
7556 }
7557
7558 sub template_create {
7559     my ($vmid, $conf, $disk) = @_;
7560
7561     my $storecfg = PVE::Storage::config();
7562
7563     PVE::QemuConfig->foreach_volume($conf, sub {
7564         my ($ds, $drive) = @_;
7565
7566         return if drive_is_cdrom($drive);
7567         return if $disk && $ds ne $disk;
7568
7569         my $volid = $drive->{file};
7570         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7571
7572         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7573         $drive->{file} = $voliddst;
7574         $conf->{$ds} = print_drive($drive);
7575         PVE::QemuConfig->write_config($vmid, $conf);
7576     });
7577 }
7578
7579 sub convert_iscsi_path {
7580     my ($path) = @_;
7581
7582     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7583         my $portal = $1;
7584         my $target = $2;
7585         my $lun = $3;
7586
7587         my $initiator_name = get_initiator_name();
7588
7589         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7590                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7591     }
7592
7593     die "cannot convert iscsi path '$path', unkown format\n";
7594 }
7595
7596 sub qemu_img_convert {
7597     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7598
7599     my $storecfg = PVE::Storage::config();
7600     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7601     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7602
7603     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7604
7605     my $cachemode;
7606     my $src_path;
7607     my $src_is_iscsi = 0;
7608     my $src_format;
7609
7610     if ($src_storeid) {
7611         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7612         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7613         $src_format = qemu_img_format($src_scfg, $src_volname);
7614         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7615         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7616         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7617     } elsif (-f $src_volid || -b $src_volid) {
7618         $src_path = $src_volid;
7619         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7620             $src_format = $1;
7621         }
7622     }
7623
7624     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7625
7626     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7627     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7628     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7629     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7630
7631     my $cmd = [];
7632     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7633     push @$cmd, '-l', "snapshot.name=$snapname"
7634         if $snapname && $src_format && $src_format eq "qcow2";
7635     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7636     push @$cmd, '-T', $cachemode if defined($cachemode);
7637
7638     if ($src_is_iscsi) {
7639         push @$cmd, '--image-opts';
7640         $src_path = convert_iscsi_path($src_path);
7641     } elsif ($src_format) {
7642         push @$cmd, '-f', $src_format;
7643     }
7644
7645     if ($dst_is_iscsi) {
7646         push @$cmd, '--target-image-opts';
7647         $dst_path = convert_iscsi_path($dst_path);
7648     } else {
7649         push @$cmd, '-O', $dst_format;
7650     }
7651
7652     push @$cmd, $src_path;
7653
7654     if (!$dst_is_iscsi && $is_zero_initialized) {
7655         push @$cmd, "zeroinit:$dst_path";
7656     } else {
7657         push @$cmd, $dst_path;
7658     }
7659
7660     my $parser = sub {
7661         my $line = shift;
7662         if($line =~ m/\((\S+)\/100\%\)/){
7663             my $percent = $1;
7664             my $transferred = int($size * $percent / 100);
7665             my $total_h = render_bytes($size, 1);
7666             my $transferred_h = render_bytes($transferred, 1);
7667
7668             print "transferred $transferred_h of $total_h ($percent%)\n";
7669         }
7670
7671     };
7672
7673     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7674     my $err = $@;
7675     die "copy failed: $err" if $err;
7676 }
7677
7678 sub qemu_img_format {
7679     my ($scfg, $volname) = @_;
7680
7681     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7682         return $1;
7683     } else {
7684         return "raw";
7685     }
7686 }
7687
7688 sub qemu_drive_mirror {
7689     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7690
7691     $jobs = {} if !$jobs;
7692
7693     my $qemu_target;
7694     my $format;
7695     $jobs->{"drive-$drive"} = {};
7696
7697     if ($dst_volid =~ /^nbd:/) {
7698         $qemu_target = $dst_volid;
7699         $format = "nbd";
7700     } else {
7701         my $storecfg = PVE::Storage::config();
7702         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7703
7704         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7705
7706         $format = qemu_img_format($dst_scfg, $dst_volname);
7707
7708         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7709
7710         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7711     }
7712
7713     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7714     $opts->{format} = $format if $format;
7715
7716     if (defined($src_bitmap)) {
7717         $opts->{sync} = 'incremental';
7718         $opts->{bitmap} = $src_bitmap;
7719         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7720     }
7721
7722     if (defined($bwlimit)) {
7723         $opts->{speed} = $bwlimit * 1024;
7724         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7725     } else {
7726         print "drive mirror is starting for drive-$drive\n";
7727     }
7728
7729     # if a job already runs for this device we get an error, catch it for cleanup
7730     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7731     if (my $err = $@) {
7732         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7733         warn "$@\n" if $@;
7734         die "mirroring error: $err\n";
7735     }
7736
7737     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7738 }
7739
7740 # $completion can be either
7741 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7742 # 'cancel': wait until all jobs are ready, block-job-cancel them
7743 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7744 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7745 sub qemu_drive_mirror_monitor {
7746     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7747
7748     $completion //= 'complete';
7749     $op //= "mirror";
7750
7751     eval {
7752         my $err_complete = 0;
7753
7754         my $starttime = time ();
7755         while (1) {
7756             die "block job ('$op') timed out\n" if $err_complete > 300;
7757
7758             my $stats = mon_cmd($vmid, "query-block-jobs");
7759             my $ctime = time();
7760
7761             my $running_jobs = {};
7762             for my $stat (@$stats) {
7763                 next if $stat->{type} ne $op;
7764                 $running_jobs->{$stat->{device}} = $stat;
7765             }
7766
7767             my $readycounter = 0;
7768
7769             for my $job_id (sort keys %$jobs) {
7770                 my $job = $running_jobs->{$job_id};
7771
7772                 my $vanished = !defined($job);
7773                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7774                 if($complete || ($vanished && $completion eq 'auto')) {
7775                     print "$job_id: $op-job finished\n";
7776                     delete $jobs->{$job_id};
7777                     next;
7778                 }
7779
7780                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7781
7782                 my $busy = $job->{busy};
7783                 my $ready = $job->{ready};
7784                 if (my $total = $job->{len}) {
7785                     my $transferred = $job->{offset} || 0;
7786                     my $remaining = $total - $transferred;
7787                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7788
7789                     my $duration = $ctime - $starttime;
7790                     my $total_h = render_bytes($total, 1);
7791                     my $transferred_h = render_bytes($transferred, 1);
7792
7793                     my $status = sprintf(
7794                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7795                         render_duration($duration),
7796                     );
7797
7798                     if ($ready) {
7799                         if ($busy) {
7800                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7801                         } else {
7802                             $status .= ", ready";
7803                         }
7804                     }
7805                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7806                     $jobs->{$job_id}->{ready} = $ready;
7807                 }
7808
7809                 $readycounter++ if $job->{ready};
7810             }
7811
7812             last if scalar(keys %$jobs) == 0;
7813
7814             if ($readycounter == scalar(keys %$jobs)) {
7815                 print "all '$op' jobs are ready\n";
7816
7817                 # do the complete later (or has already been done)
7818                 last if $completion eq 'skip' || $completion eq 'auto';
7819
7820                 if ($vmiddst && $vmiddst != $vmid) {
7821                     my $agent_running = $qga && qga_check_running($vmid);
7822                     if ($agent_running) {
7823                         print "freeze filesystem\n";
7824                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7825                         warn $@ if $@;
7826                     } else {
7827                         print "suspend vm\n";
7828                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7829                         warn $@ if $@;
7830                     }
7831
7832                     # if we clone a disk for a new target vm, we don't switch the disk
7833                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7834
7835                     if ($agent_running) {
7836                         print "unfreeze filesystem\n";
7837                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7838                         warn $@ if $@;
7839                     } else {
7840                         print "resume vm\n";
7841                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7842                         warn $@ if $@;
7843                     }
7844
7845                     last;
7846                 } else {
7847
7848                     for my $job_id (sort keys %$jobs) {
7849                         # try to switch the disk if source and destination are on the same guest
7850                         print "$job_id: Completing block job_id...\n";
7851
7852                         my $op;
7853                         if ($completion eq 'complete') {
7854                             $op = 'block-job-complete';
7855                         } elsif ($completion eq 'cancel') {
7856                             $op = 'block-job-cancel';
7857                         } else {
7858                             die "invalid completion value: $completion\n";
7859                         }
7860                         eval { mon_cmd($vmid, $op, device => $job_id) };
7861                         if ($@ =~ m/cannot be completed/) {
7862                             print "$job_id: block job cannot be completed, trying again.\n";
7863                             $err_complete++;
7864                         }else {
7865                             print "$job_id: Completed successfully.\n";
7866                             $jobs->{$job_id}->{complete} = 1;
7867                         }
7868                     }
7869                 }
7870             }
7871             sleep 1;
7872         }
7873     };
7874     my $err = $@;
7875
7876     if ($err) {
7877         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7878         die "block job ($op) error: $err";
7879     }
7880 }
7881
7882 sub qemu_blockjobs_cancel {
7883     my ($vmid, $jobs) = @_;
7884
7885     foreach my $job (keys %$jobs) {
7886         print "$job: Cancelling block job\n";
7887         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7888         $jobs->{$job}->{cancel} = 1;
7889     }
7890
7891     while (1) {
7892         my $stats = mon_cmd($vmid, "query-block-jobs");
7893
7894         my $running_jobs = {};
7895         foreach my $stat (@$stats) {
7896             $running_jobs->{$stat->{device}} = $stat;
7897         }
7898
7899         foreach my $job (keys %$jobs) {
7900
7901             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7902                 print "$job: Done.\n";
7903                 delete $jobs->{$job};
7904             }
7905         }
7906
7907         last if scalar(keys %$jobs) == 0;
7908
7909         sleep 1;
7910     }
7911 }
7912
7913 sub clone_disk {
7914     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7915
7916     my ($vmid, $running) = $source->@{qw(vmid running)};
7917     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7918
7919     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7920     my ($storage, $format) = $dest->@{qw(storage format)};
7921
7922     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7923
7924     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7925         die "cloning from/to EFI disk requires EFI disk\n"
7926             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7927         die "cloning from/to TPM state requires TPM state\n"
7928             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7929
7930         # This would lead to two device nodes in QEMU pointing to the same backing image!
7931         die "cannot change drive name when cloning disk from/to the same VM\n"
7932             if $use_drive_mirror && $vmid == $newvmid;
7933     }
7934
7935     die "cannot move TPM state while VM is running\n"
7936         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7937
7938     my $newvolid;
7939
7940     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7941     print "$src_drivename " if $src_drivename;
7942     print "($drive->{file})\n";
7943
7944     if (!$full) {
7945         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7946         push @$newvollist, $newvolid;
7947     } else {
7948
7949         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7950         $storeid = $storage if $storage;
7951
7952         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7953
7954         my $name = undef;
7955         my $size = undef;
7956         if (drive_is_cloudinit($drive)) {
7957             $name = "vm-$newvmid-cloudinit";
7958             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7959             if ($scfg->{path}) {
7960                 $name .= ".$dst_format";
7961             }
7962             $snapname = undef;
7963             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7964         } elsif ($dst_drivename eq 'efidisk0') {
7965             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7966         } elsif ($dst_drivename eq 'tpmstate0') {
7967             $dst_format = 'raw';
7968             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7969         } else {
7970             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7971         }
7972         $newvolid = PVE::Storage::vdisk_alloc(
7973             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7974         );
7975         push @$newvollist, $newvolid;
7976
7977         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7978
7979         if (drive_is_cloudinit($drive)) {
7980             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7981             # if this is the case, we have to complete any block-jobs still there from
7982             # previous drive-mirrors
7983             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7984                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7985             }
7986             goto no_data_clone;
7987         }
7988
7989         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7990         if ($use_drive_mirror) {
7991             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7992                 $completion, $qga, $bwlimit);
7993         } else {
7994             # TODO: handle bwlimits
7995             if ($dst_drivename eq 'efidisk0') {
7996                 # the relevant data on the efidisk may be smaller than the source
7997                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7998                 # that is given by the OVMF_VARS.fd
7999                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8000                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8001
8002                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8003
8004                 # better for Ceph if block size is not too small, see bug #3324
8005                 my $bs = 1024*1024;
8006
8007                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8008
8009                 if ($src_format eq 'qcow2' && $snapname) {
8010                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8011                         if !min_version(kvm_user_version(), 6, 2);
8012                     push $cmd->@*, '-l', $snapname;
8013                 }
8014                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8015                 run_command($cmd);
8016             } else {
8017                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8018             }
8019         }
8020     }
8021
8022 no_data_clone:
8023     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8024
8025     my $disk = dclone($drive);
8026     delete $disk->{format};
8027     $disk->{file} = $newvolid;
8028     $disk->{size} = $size if defined($size);
8029
8030     return $disk;
8031 }
8032
8033 sub get_running_qemu_version {
8034     my ($vmid) = @_;
8035     my $res = mon_cmd($vmid, "query-version");
8036     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8037 }
8038
8039 sub qemu_use_old_bios_files {
8040     my ($machine_type) = @_;
8041
8042     return if !$machine_type;
8043
8044     my $use_old_bios_files = undef;
8045
8046     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8047         $machine_type = $1;
8048         $use_old_bios_files = 1;
8049     } else {
8050         my $version = extract_version($machine_type, kvm_user_version());
8051         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8052         # load new efi bios files on migration. So this hack is required to allow
8053         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8054         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8055         $use_old_bios_files = !min_version($version, 2, 4);
8056     }
8057
8058     return ($use_old_bios_files, $machine_type);
8059 }
8060
8061 sub get_efivars_size {
8062     my ($conf, $efidisk) = @_;
8063
8064     my $arch = get_vm_arch($conf);
8065     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8066     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8067     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8068     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8069     return -s $ovmf_vars;
8070 }
8071
8072 sub update_efidisk_size {
8073     my ($conf) = @_;
8074
8075     return if !defined($conf->{efidisk0});
8076
8077     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8078     $disk->{size} = get_efivars_size($conf);
8079     $conf->{efidisk0} = print_drive($disk);
8080
8081     return;
8082 }
8083
8084 sub update_tpmstate_size {
8085     my ($conf) = @_;
8086
8087     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8088     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8089     $conf->{tpmstate0} = print_drive($disk);
8090 }
8091
8092 sub create_efidisk($$$$$$$) {
8093     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8094
8095     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8096     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8097
8098     my $vars_size_b = -s $ovmf_vars;
8099     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8100     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8101     PVE::Storage::activate_volumes($storecfg, [$volid]);
8102
8103     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8104     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8105
8106     return ($volid, $size/1024);
8107 }
8108
8109 sub vm_iothreads_list {
8110     my ($vmid) = @_;
8111
8112     my $res = mon_cmd($vmid, 'query-iothreads');
8113
8114     my $iothreads = {};
8115     foreach my $iothread (@$res) {
8116         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8117     }
8118
8119     return $iothreads;
8120 }
8121
8122 sub scsihw_infos {
8123     my ($conf, $drive) = @_;
8124
8125     my $maxdev = 0;
8126
8127     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8128         $maxdev = 7;
8129     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8130         $maxdev = 1;
8131     } else {
8132         $maxdev = 256;
8133     }
8134
8135     my $controller = int($drive->{index} / $maxdev);
8136     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8137         ? "virtioscsi"
8138         : "scsihw";
8139
8140     return ($maxdev, $controller, $controller_prefix);
8141 }
8142
8143 sub resolve_dst_disk_format {
8144         my ($storecfg, $storeid, $src_volname, $format) = @_;
8145         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8146
8147         if (!$format) {
8148             # if no target format is specified, use the source disk format as hint
8149             if ($src_volname) {
8150                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8151                 $format = qemu_img_format($scfg, $src_volname);
8152             } else {
8153                 return $defFormat;
8154             }
8155         }
8156
8157         # test if requested format is supported - else use default
8158         my $supported = grep { $_ eq $format } @$validFormats;
8159         $format = $defFormat if !$supported;
8160         return $format;
8161 }
8162
8163 # NOTE: if this logic changes, please update docs & possibly gui logic
8164 sub find_vmstate_storage {
8165     my ($conf, $storecfg) = @_;
8166
8167     # first, return storage from conf if set
8168     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8169
8170     my ($target, $shared, $local);
8171
8172     foreach_storage_used_by_vm($conf, sub {
8173         my ($sid) = @_;
8174         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8175         my $dst = $scfg->{shared} ? \$shared : \$local;
8176         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8177     });
8178
8179     # second, use shared storage where VM has at least one disk
8180     # third, use local storage where VM has at least one disk
8181     # fall back to local storage
8182     $target = $shared // $local // 'local';
8183
8184     return $target;
8185 }
8186
8187 sub generate_uuid {
8188     my ($uuid, $uuid_str);
8189     UUID::generate($uuid);
8190     UUID::unparse($uuid, $uuid_str);
8191     return $uuid_str;
8192 }
8193
8194 sub generate_smbios1_uuid {
8195     return "uuid=".generate_uuid();
8196 }
8197
8198 sub nbd_stop {
8199     my ($vmid) = @_;
8200
8201     mon_cmd($vmid, 'nbd-server-stop');
8202 }
8203
8204 sub create_reboot_request {
8205     my ($vmid) = @_;
8206     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8207         or die "failed to create reboot trigger file: $!\n";
8208     close($fh);
8209 }
8210
8211 sub clear_reboot_request {
8212     my ($vmid) = @_;
8213     my $path = "/run/qemu-server/$vmid.reboot";
8214     my $res = 0;
8215
8216     $res = unlink($path);
8217     die "could not remove reboot request for $vmid: $!"
8218         if !$res && $! != POSIX::ENOENT;
8219
8220     return $res;
8221 }
8222
8223 sub bootorder_from_legacy {
8224     my ($conf, $bootcfg) = @_;
8225
8226     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8227     my $bootindex_hash = {};
8228     my $i = 1;
8229     foreach my $o (split(//, $boot)) {
8230         $bootindex_hash->{$o} = $i*100;
8231         $i++;
8232     }
8233
8234     my $bootorder = {};
8235
8236     PVE::QemuConfig->foreach_volume($conf, sub {
8237         my ($ds, $drive) = @_;
8238
8239         if (drive_is_cdrom ($drive, 1)) {
8240             if ($bootindex_hash->{d}) {
8241                 $bootorder->{$ds} = $bootindex_hash->{d};
8242                 $bootindex_hash->{d} += 1;
8243             }
8244         } elsif ($bootindex_hash->{c}) {
8245             $bootorder->{$ds} = $bootindex_hash->{c}
8246                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8247             $bootindex_hash->{c} += 1;
8248         }
8249     });
8250
8251     if ($bootindex_hash->{n}) {
8252         for (my $i = 0; $i < $MAX_NETS; $i++) {
8253             my $netname = "net$i";
8254             next if !$conf->{$netname};
8255             $bootorder->{$netname} = $bootindex_hash->{n};
8256             $bootindex_hash->{n} += 1;
8257         }
8258     }
8259
8260     return $bootorder;
8261 }
8262
8263 # Generate default device list for 'boot: order=' property. Matches legacy
8264 # default boot order, but with explicit device names. This is important, since
8265 # the fallback for when neither 'order' nor the old format is specified relies
8266 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8267 sub get_default_bootdevices {
8268     my ($conf) = @_;
8269
8270     my @ret = ();
8271
8272     # harddisk
8273     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8274     push @ret, $first if $first;
8275
8276     # cdrom
8277     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8278     push @ret, $first if $first;
8279
8280     # network
8281     for (my $i = 0; $i < $MAX_NETS; $i++) {
8282         my $netname = "net$i";
8283         next if !$conf->{$netname};
8284         push @ret, $netname;
8285         last;
8286     }
8287
8288     return \@ret;
8289 }
8290
8291 sub device_bootorder {
8292     my ($conf) = @_;
8293
8294     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8295
8296     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8297
8298     my $bootorder = {};
8299     if (!defined($boot) || $boot->{legacy}) {
8300         $bootorder = bootorder_from_legacy($conf, $boot);
8301     } elsif ($boot->{order}) {
8302         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8303         for my $dev (PVE::Tools::split_list($boot->{order})) {
8304             $bootorder->{$dev} = $i++;
8305         }
8306     }
8307
8308     return $bootorder;
8309 }
8310
8311 sub register_qmeventd_handle {
8312     my ($vmid) = @_;
8313
8314     my $fh;
8315     my $peer = "/var/run/qmeventd.sock";
8316     my $count = 0;
8317
8318     for (;;) {
8319         $count++;
8320         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8321         last if $fh;
8322         if ($! != EINTR && $! != EAGAIN) {
8323             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8324         }
8325         if ($count > 4) {
8326             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8327               . "after $count retries\n";
8328         }
8329         usleep(25000);
8330     }
8331
8332     # send handshake to mark VM as backing up
8333     print $fh to_json({vzdump => {vmid => "$vmid"}});
8334
8335     # return handle to be closed later when inhibit is no longer required
8336     return $fh;
8337 }
8338
8339 # bash completion helper
8340
8341 sub complete_backup_archives {
8342     my ($cmdname, $pname, $cvalue) = @_;
8343
8344     my $cfg = PVE::Storage::config();
8345
8346     my $storeid;
8347
8348     if ($cvalue =~ m/^([^:]+):/) {
8349         $storeid = $1;
8350     }
8351
8352     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8353
8354     my $res = [];
8355     foreach my $id (keys %$data) {
8356         foreach my $item (@{$data->{$id}}) {
8357             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8358             push @$res, $item->{volid} if defined($item->{volid});
8359         }
8360     }
8361
8362     return $res;
8363 }
8364
8365 my $complete_vmid_full = sub {
8366     my ($running) = @_;
8367
8368     my $idlist = vmstatus();
8369
8370     my $res = [];
8371
8372     foreach my $id (keys %$idlist) {
8373         my $d = $idlist->{$id};
8374         if (defined($running)) {
8375             next if $d->{template};
8376             next if $running && $d->{status} ne 'running';
8377             next if !$running && $d->{status} eq 'running';
8378         }
8379         push @$res, $id;
8380
8381     }
8382     return $res;
8383 };
8384
8385 sub complete_vmid {
8386     return &$complete_vmid_full();
8387 }
8388
8389 sub complete_vmid_stopped {
8390     return &$complete_vmid_full(0);
8391 }
8392
8393 sub complete_vmid_running {
8394     return &$complete_vmid_full(1);
8395 }
8396
8397 sub complete_storage {
8398
8399     my $cfg = PVE::Storage::config();
8400     my $ids = $cfg->{ids};
8401
8402     my $res = [];
8403     foreach my $sid (keys %$ids) {
8404         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8405         next if !$ids->{$sid}->{content}->{images};
8406         push @$res, $sid;
8407     }
8408
8409     return $res;
8410 }
8411
8412 sub complete_migration_storage {
8413     my ($cmd, $param, $current_value, $all_args) = @_;
8414
8415     my $targetnode = @$all_args[1];
8416
8417     my $cfg = PVE::Storage::config();
8418     my $ids = $cfg->{ids};
8419
8420     my $res = [];
8421     foreach my $sid (keys %$ids) {
8422         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8423         next if !$ids->{$sid}->{content}->{images};
8424         push @$res, $sid;
8425     }
8426
8427     return $res;
8428 }
8429
8430 sub vm_is_paused {
8431     my ($vmid) = @_;
8432     my $qmpstatus = eval {
8433         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8434         mon_cmd($vmid, "query-status");
8435     };
8436     warn "$@\n" if $@;
8437     return $qmpstatus && $qmpstatus->{status} eq "paused";
8438 }
8439
8440 sub check_volume_storage_type {
8441     my ($storecfg, $vol) = @_;
8442
8443     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8444     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8445     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8446
8447     die "storage '$storeid' does not support content-type '$vtype'\n"
8448         if !$scfg->{content}->{$vtype};
8449
8450     return 1;
8451 }
8452
8453 sub add_nets_bridge_fdb {
8454     my ($conf, $vmid) = @_;
8455
8456     for my $opt (keys %$conf) {
8457         next if $opt !~ m/^net(\d+)$/;
8458         my $iface = "tap${vmid}i$1";
8459         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8460         my $net = parse_net($conf->{$opt}, 1) or next;
8461
8462         my $mac = $net->{macaddr};
8463         if (!$mac) {
8464             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8465                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8466             next;
8467         }
8468
8469         my $bridge = $net->{bridge};
8470         if ($have_sdn) {
8471             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8472         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8473             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8474         }
8475     }
8476 }
8477
8478 sub del_nets_bridge_fdb {
8479     my ($conf, $vmid) = @_;
8480
8481     for my $opt (keys %$conf) {
8482         next if $opt !~ m/^net(\d+)$/;
8483         my $iface = "tap${vmid}i$1";
8484
8485         my $net = parse_net($conf->{$opt}) or next;
8486         my $mac = $net->{macaddr} or next;
8487
8488         my $bridge = $net->{bridge};
8489         if ($have_sdn) {
8490             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8491         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8492             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8493         }
8494     }
8495 }
8496
8497 1;