PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2345
2346     if ($conf->{template}) {
2347         # check if any base image is still used by a linked clone
2348         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2349                 my ($ds, $drive) = @_;
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353                 return if !$volid || $volid =~ m|^/|;
2354
2355                 die "base volume '$volid' is still in use by linked cloned\n"
2356                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2357
2358         });
2359     }
2360
2361     my $volids = {};
2362     my $remove_owned_drive = sub {
2363         my ($ds, $drive) = @_;
2364         return if drive_is_cdrom($drive, 1);
2365
2366         my $volid = $drive->{file};
2367         return if !$volid || $volid =~ m|^/|;
2368         return if $volids->{$volid};
2369
2370         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2371         return if !$path || !$owner || ($owner != $vmid);
2372
2373         $volids->{$volid} = 1;
2374         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2375         warn "Could not remove disk '$volid', check manually: $@" if $@;
2376     };
2377
2378     # only remove disks owned by this VM (referenced in the config)
2379     my $include_opts = {
2380         include_unused => 1,
2381         extra_keys => ['vmstate'],
2382     };
2383     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2384
2385     for my $snap (values %{$conf->{snapshots}}) {
2386         next if !defined($snap->{vmstate});
2387         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2388         next if !defined($drive);
2389         $remove_owned_drive->('vmstate', $drive);
2390     }
2391
2392     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2393
2394     if ($purge_unreferenced) { # also remove unreferenced disk
2395         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2396         PVE::Storage::foreach_volid($vmdisks, sub {
2397             my ($volid, $sid, $volname, $d) = @_;
2398             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2399             warn $@ if $@;
2400         });
2401     }
2402
2403     if (defined $replacement_conf) {
2404         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2405     } else {
2406         PVE::QemuConfig->destroy_config($vmid);
2407     }
2408 }
2409
2410 sub parse_vm_config {
2411     my ($filename, $raw, $strict) = @_;
2412
2413     return if !defined($raw);
2414
2415     my $res = {
2416         digest => Digest::SHA::sha1_hex($raw),
2417         snapshots => {},
2418         pending => {},
2419         cloudinit => {},
2420     };
2421
2422     my $handle_error = sub {
2423         my ($msg) = @_;
2424
2425         if ($strict) {
2426             die $msg;
2427         } else {
2428             warn $msg;
2429         }
2430     };
2431
2432     $filename =~ m|/qemu-server/(\d+)\.conf$|
2433         || die "got strange filename '$filename'";
2434
2435     my $vmid = $1;
2436
2437     my $conf = $res;
2438     my $descr;
2439     my $finish_description = sub {
2440         if (defined($descr)) {
2441             $descr =~ s/\s+$//;
2442             $conf->{description} = $descr;
2443         }
2444         $descr = undef;
2445     };
2446     my $section = '';
2447
2448     my @lines = split(/\n/, $raw);
2449     foreach my $line (@lines) {
2450         next if $line =~ m/^\s*$/;
2451
2452         if ($line =~ m/^\[PENDING\]\s*$/i) {
2453             $section = 'pending';
2454             $finish_description->();
2455             $conf = $res->{$section} = {};
2456             next;
2457         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458             $section = 'cloudinit';
2459             $finish_description->();
2460             $conf = $res->{$section} = {};
2461             next;
2462
2463         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2464             $section = $1;
2465             $finish_description->();
2466             $conf = $res->{snapshots}->{$section} = {};
2467             next;
2468         }
2469
2470         if ($line =~ m/^\#(.*)$/) {
2471             $descr = '' if !defined($descr);
2472             $descr .= PVE::Tools::decode_text($1) . "\n";
2473             next;
2474         }
2475
2476         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477             $descr = '' if !defined($descr);
2478             $descr .= PVE::Tools::decode_text($2);
2479         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480             $conf->{snapstate} = $1;
2481         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2482             my $key = $1;
2483             my $value = $2;
2484             $conf->{$key} = $value;
2485         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2486             my $value = $1;
2487             if ($section eq 'pending') {
2488                 $conf->{delete} = $value; # we parse this later
2489             } else {
2490                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2491             }
2492         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             if ($section eq 'cloudinit') {
2496                 # ignore validation only used for informative purpose
2497                 $conf->{$key} = $value;
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             if ($@) {
2502                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2503             } else {
2504                 $key = 'ide2' if $key eq 'cdrom';
2505                 my $fmt = $confdesc->{$key}->{format};
2506                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507                     my $v = parse_drive($key, $value);
2508                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2509                         $v->{file} = $volid;
2510                         $value = print_drive($v);
2511                     } else {
2512                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513                         next;
2514                     }
2515                 }
2516
2517                 $conf->{$key} = $value;
2518             }
2519         } else {
2520             $handle_error->("vm $vmid - unable to parse config: $line\n");
2521         }
2522     }
2523
2524     $finish_description->();
2525     delete $res->{snapstate}; # just to be sure
2526
2527     return $res;
2528 }
2529
2530 sub write_vm_config {
2531     my ($filename, $conf) = @_;
2532
2533     delete $conf->{snapstate}; # just to be sure
2534
2535     if ($conf->{cdrom}) {
2536         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2537         $conf->{ide2} = $conf->{cdrom};
2538         delete $conf->{cdrom};
2539     }
2540
2541     # we do not use 'smp' any longer
2542     if ($conf->{sockets}) {
2543         delete $conf->{smp};
2544     } elsif ($conf->{smp}) {
2545         $conf->{sockets} = $conf->{smp};
2546         delete $conf->{cores};
2547         delete $conf->{smp};
2548     }
2549
2550     my $used_volids = {};
2551
2552     my $cleanup_config = sub {
2553         my ($cref, $pending, $snapname) = @_;
2554
2555         foreach my $key (keys %$cref) {
2556             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558             my $value = $cref->{$key};
2559             if ($key eq 'delete') {
2560                 die "propertry 'delete' is only allowed in [PENDING]\n"
2561                     if !$pending;
2562                 # fixme: check syntax?
2563                 next;
2564             }
2565             eval { $value = check_type($key, $value); };
2566             die "unable to parse value of '$key' - $@" if $@;
2567
2568             $cref->{$key} = $value;
2569
2570             if (!$snapname && is_valid_drivename($key)) {
2571                 my $drive = parse_drive($key, $value);
2572                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2573             }
2574         }
2575     };
2576
2577     &$cleanup_config($conf);
2578
2579     &$cleanup_config($conf->{pending}, 1);
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2584     }
2585
2586     # remove 'unusedX' settings if we re-add a volume
2587     foreach my $key (keys %$conf) {
2588         my $value = $conf->{$key};
2589         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590             delete $conf->{$key};
2591         }
2592     }
2593
2594     my $generate_raw_config = sub {
2595         my ($conf, $pending) = @_;
2596
2597         my $raw = '';
2598
2599         # add description as comment to top of file
2600         if (defined(my $descr = $conf->{description})) {
2601             if ($descr) {
2602                 foreach my $cl (split(/\n/, $descr)) {
2603                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2604                 }
2605             } else {
2606                 $raw .= "#\n" if $pending;
2607             }
2608         }
2609
2610         foreach my $key (sort keys %$conf) {
2611             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612             $raw .= "$key: $conf->{$key}\n";
2613         }
2614         return $raw;
2615     };
2616
2617     my $raw = &$generate_raw_config($conf);
2618
2619     if (scalar(keys %{$conf->{pending}})){
2620         $raw .= "\n[PENDING]\n";
2621         $raw .= &$generate_raw_config($conf->{pending}, 1);
2622     }
2623
2624     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2625         $raw .= "\n[special:cloudinit]\n";
2626         $raw .= &$generate_raw_config($conf->{cloudinit});
2627     }
2628
2629     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2630         $raw .= "\n[$snapname]\n";
2631         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2632     }
2633
2634     return $raw;
2635 }
2636
2637 sub load_defaults {
2638
2639     my $res = {};
2640
2641     # we use static defaults from our JSON schema configuration
2642     foreach my $key (keys %$confdesc) {
2643         if (defined(my $default = $confdesc->{$key}->{default})) {
2644             $res->{$key} = $default;
2645         }
2646     }
2647
2648     return $res;
2649 }
2650
2651 sub config_list {
2652     my $vmlist = PVE::Cluster::get_vmlist();
2653     my $res = {};
2654     return $res if !$vmlist || !$vmlist->{ids};
2655     my $ids = $vmlist->{ids};
2656     my $nodename = nodename();
2657
2658     foreach my $vmid (keys %$ids) {
2659         my $d = $ids->{$vmid};
2660         next if !$d->{node} || $d->{node} ne $nodename;
2661         next if !$d->{type} || $d->{type} ne 'qemu';
2662         $res->{$vmid}->{exists} = 1;
2663     }
2664     return $res;
2665 }
2666
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources {
2669     my ($conf, $noerr) = @_;
2670
2671     my @loc_res = ();
2672
2673     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2674     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2675
2676     push @loc_res, "ivshmem" if $conf->{ivshmem};
2677
2678     foreach my $k (keys %$conf) {
2679         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680         # sockets are safe: they will recreated be on the target side post-migrate
2681         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2683     }
2684
2685     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2686
2687     return \@loc_res;
2688 }
2689
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability {
2692     my ($storecfg, $conf, $node) = @_;
2693
2694     PVE::QemuConfig->foreach_volume($conf, sub {
2695         my ($ds, $drive) = @_;
2696
2697         my $volid = $drive->{file};
2698         return if !$volid;
2699
2700         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2701         return if !$sid;
2702
2703         # check if storage is available on both nodes
2704         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2705         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2706
2707         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2708
2709         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710             if !$scfg->{content}->{$vtype};
2711    });
2712 }
2713
2714 # list nodes where all VM images are available (used by has_feature API)
2715 sub shared_nodes {
2716     my ($conf, $storecfg) = @_;
2717
2718     my $nodelist = PVE::Cluster::get_nodelist();
2719     my $nodehash = { map { $_ => 1 } @$nodelist };
2720     my $nodename = nodename();
2721
2722     PVE::QemuConfig->foreach_volume($conf, sub {
2723         my ($ds, $drive) = @_;
2724
2725         my $volid = $drive->{file};
2726         return if !$volid;
2727
2728         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2729         if ($storeid) {
2730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2731             if ($scfg->{disable}) {
2732                 $nodehash = {};
2733             } elsif (my $avail = $scfg->{nodes}) {
2734                 foreach my $node (keys %$nodehash) {
2735                     delete $nodehash->{$node} if !$avail->{$node};
2736                 }
2737             } elsif (!$scfg->{shared}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     delete $nodehash->{$node} if $node ne $nodename
2740                 }
2741             }
2742         }
2743     });
2744
2745     return $nodehash
2746 }
2747
2748 sub check_local_storage_availability {
2749     my ($conf, $storecfg) = @_;
2750
2751     my $nodelist = PVE::Cluster::get_nodelist();
2752     my $nodehash = { map { $_ => {} } @$nodelist };
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763
2764             if ($scfg->{disable}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2767                 }
2768             } elsif (my $avail = $scfg->{nodes}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     if (!$avail->{$node}) {
2771                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2772                     }
2773                 }
2774             }
2775         }
2776     });
2777
2778     foreach my $node (values %$nodehash) {
2779         if (my $unavail = $node->{unavailable_storages}) {
2780             $node->{unavailable_storages} = [ sort keys %$unavail ];
2781         }
2782     }
2783
2784     return $nodehash
2785 }
2786
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2788 sub check_running {
2789     my ($vmid, $nocheck, $node) = @_;
2790
2791     # $nocheck is set when called during a migration, in which case the config
2792     # file might still or already reside on the *other* node
2793     # - because rename has already happened, and current node is source
2794     # - because rename hasn't happened yet, and current node is target
2795     # - because rename has happened, current node is target, but hasn't yet
2796     # processed it yet
2797     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2798     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2799 }
2800
2801 sub vzlist {
2802
2803     my $vzlist = config_list();
2804
2805     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2806
2807     while (defined(my $de = $fd->read)) {
2808         next if $de !~ m/^(\d+)\.pid$/;
2809         my $vmid = $1;
2810         next if !defined($vzlist->{$vmid});
2811         if (my $pid = check_running($vmid)) {
2812             $vzlist->{$vmid}->{pid} = $pid;
2813         }
2814     }
2815
2816     return $vzlist;
2817 }
2818
2819 our $vmstatus_return_properties = {
2820     vmid => get_standard_option('pve-vmid'),
2821     status => {
2822         description => "Qemu process status.",
2823         type => 'string',
2824         enum => ['stopped', 'running'],
2825     },
2826     maxmem => {
2827         description => "Maximum memory in bytes.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'bytes',
2831     },
2832     maxdisk => {
2833         description => "Root disk size in bytes.",
2834         type => 'integer',
2835         optional => 1,
2836         renderer => 'bytes',
2837     },
2838     name => {
2839         description => "VM name.",
2840         type => 'string',
2841         optional => 1,
2842     },
2843     qmpstatus => {
2844         description => "Qemu QMP agent status.",
2845         type => 'string',
2846         optional => 1,
2847     },
2848     pid => {
2849         description => "PID of running qemu process.",
2850         type => 'integer',
2851         optional => 1,
2852     },
2853     uptime => {
2854         description => "Uptime.",
2855         type => 'integer',
2856         optional => 1,
2857         renderer => 'duration',
2858     },
2859     cpus => {
2860         description => "Maximum usable CPUs.",
2861         type => 'number',
2862         optional => 1,
2863     },
2864     lock => {
2865         description => "The current config lock, if any.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     tags => {
2870         description  => "The current configured tags, if any",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     'running-machine' => {
2875         description => "The currently running machine type (if running).",
2876         type => 'string',
2877         optional => 1,
2878     },
2879     'running-qemu' => {
2880         description => "The currently running QEMU version (if running).",
2881         type => 'string',
2882         optional => 1,
2883     },
2884 };
2885
2886 my $last_proc_pid_stat;
2887
2888 # get VM status information
2889 # This must be fast and should not block ($full == false)
2890 # We only query KVM using QMP if $full == true (this can be slow)
2891 sub vmstatus {
2892     my ($opt_vmid, $full) = @_;
2893
2894     my $res = {};
2895
2896     my $storecfg = PVE::Storage::config();
2897
2898     my $list = vzlist();
2899     my $defaults = load_defaults();
2900
2901     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2902
2903     my $cpucount = $cpuinfo->{cpus} || 1;
2904
2905     foreach my $vmid (keys %$list) {
2906         next if $opt_vmid && ($vmid ne $opt_vmid);
2907
2908         my $conf = PVE::QemuConfig->load_config($vmid);
2909
2910         my $d = { vmid => int($vmid) };
2911         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2912
2913         # fixme: better status?
2914         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2915
2916         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2917         if (defined($size)) {
2918             $d->{disk} = 0; # no info available
2919             $d->{maxdisk} = $size;
2920         } else {
2921             $d->{disk} = 0;
2922             $d->{maxdisk} = 0;
2923         }
2924
2925         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2926             * ($conf->{cores} || $defaults->{cores});
2927         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2928         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2929
2930         $d->{name} = $conf->{name} || "VM $vmid";
2931         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2932             : $defaults->{memory}*(1024*1024);
2933
2934         if ($conf->{balloon}) {
2935             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2936             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2937                 : $defaults->{shares};
2938         }
2939
2940         $d->{uptime} = 0;
2941         $d->{cpu} = 0;
2942         $d->{mem} = 0;
2943
2944         $d->{netout} = 0;
2945         $d->{netin} = 0;
2946
2947         $d->{diskread} = 0;
2948         $d->{diskwrite} = 0;
2949
2950         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2951
2952         $d->{serial} = 1 if conf_has_serial($conf);
2953         $d->{lock} = $conf->{lock} if $conf->{lock};
2954         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2955
2956         $res->{$vmid} = $d;
2957     }
2958
2959     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2960     foreach my $dev (keys %$netdev) {
2961         next if $dev !~ m/^tap([1-9]\d*)i/;
2962         my $vmid = $1;
2963         my $d = $res->{$vmid};
2964         next if !$d;
2965
2966         $d->{netout} += $netdev->{$dev}->{receive};
2967         $d->{netin} += $netdev->{$dev}->{transmit};
2968
2969         if ($full) {
2970             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2971             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2972         }
2973
2974     }
2975
2976     my $ctime = gettimeofday;
2977
2978     foreach my $vmid (keys %$list) {
2979
2980         my $d = $res->{$vmid};
2981         my $pid = $d->{pid};
2982         next if !$pid;
2983
2984         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2985         next if !$pstat; # not running
2986
2987         my $used = $pstat->{utime} + $pstat->{stime};
2988
2989         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2990
2991         if ($pstat->{vsize}) {
2992             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2993         }
2994
2995         my $old = $last_proc_pid_stat->{$pid};
2996         if (!$old) {
2997             $last_proc_pid_stat->{$pid} = {
2998                 time => $ctime,
2999                 used => $used,
3000                 cpu => 0,
3001             };
3002             next;
3003         }
3004
3005         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3006
3007         if ($dtime > 1000) {
3008             my $dutime = $used -  $old->{used};
3009
3010             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3011             $last_proc_pid_stat->{$pid} = {
3012                 time => $ctime,
3013                 used => $used,
3014                 cpu => $d->{cpu},
3015             };
3016         } else {
3017             $d->{cpu} = $old->{cpu};
3018         }
3019     }
3020
3021     return $res if !$full;
3022
3023     my $qmpclient = PVE::QMPClient->new();
3024
3025     my $ballooncb = sub {
3026         my ($vmid, $resp) = @_;
3027
3028         my $info = $resp->{'return'};
3029         return if !$info->{max_mem};
3030
3031         my $d = $res->{$vmid};
3032
3033         # use memory assigned to VM
3034         $d->{maxmem} = $info->{max_mem};
3035         $d->{balloon} = $info->{actual};
3036
3037         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3038             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3039             $d->{freemem} = $info->{free_mem};
3040         }
3041
3042         $d->{ballooninfo} = $info;
3043     };
3044
3045     my $blockstatscb = sub {
3046         my ($vmid, $resp) = @_;
3047         my $data = $resp->{'return'} || [];
3048         my $totalrdbytes = 0;
3049         my $totalwrbytes = 0;
3050
3051         for my $blockstat (@$data) {
3052             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3053             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3054
3055             $blockstat->{device} =~ s/drive-//;
3056             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3057         }
3058         $res->{$vmid}->{diskread} = $totalrdbytes;
3059         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3060     };
3061
3062     my $machinecb = sub {
3063         my ($vmid, $resp) = @_;
3064         my $data = $resp->{'return'} || [];
3065
3066         $res->{$vmid}->{'running-machine'} =
3067             PVE::QemuServer::Machine::current_from_query_machines($data);
3068     };
3069
3070     my $versioncb = sub {
3071         my ($vmid, $resp) = @_;
3072         my $data = $resp->{'return'} // {};
3073         my $version = 'unknown';
3074
3075         if (my $v = $data->{qemu}) {
3076             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3077         }
3078
3079         $res->{$vmid}->{'running-qemu'} = $version;
3080     };
3081
3082     my $statuscb = sub {
3083         my ($vmid, $resp) = @_;
3084
3085         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3086         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3087         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3088         # this fails if ballon driver is not loaded, so this must be
3089         # the last commnand (following command are aborted if this fails).
3090         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3091
3092         my $status = 'unknown';
3093         if (!defined($status = $resp->{'return'}->{status})) {
3094             warn "unable to get VM status\n";
3095             return;
3096         }
3097
3098         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3099     };
3100
3101     foreach my $vmid (keys %$list) {
3102         next if $opt_vmid && ($vmid ne $opt_vmid);
3103         next if !$res->{$vmid}->{pid}; # not running
3104         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3105     }
3106
3107     $qmpclient->queue_execute(undef, 2);
3108
3109     foreach my $vmid (keys %$list) {
3110         next if $opt_vmid && ($vmid ne $opt_vmid);
3111         next if !$res->{$vmid}->{pid}; #not running
3112
3113         # we can't use the $qmpclient since it might have already aborted on
3114         # 'query-balloon', but this might also fail for older versions...
3115         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3116         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3117     }
3118
3119     foreach my $vmid (keys %$list) {
3120         next if $opt_vmid && ($vmid ne $opt_vmid);
3121         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3122     }
3123
3124     return $res;
3125 }
3126
3127 sub conf_has_serial {
3128     my ($conf) = @_;
3129
3130     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3131         if ($conf->{"serial$i"}) {
3132             return 1;
3133         }
3134     }
3135
3136     return 0;
3137 }
3138
3139 sub conf_has_audio {
3140     my ($conf, $id) = @_;
3141
3142     $id //= 0;
3143     my $audio = $conf->{"audio$id"};
3144     return if !defined($audio);
3145
3146     my $audioproperties = parse_property_string($audio_fmt, $audio);
3147     my $audiodriver = $audioproperties->{driver} // 'spice';
3148
3149     return {
3150         dev => $audioproperties->{device},
3151         dev_id => "audiodev$id",
3152         backend => $audiodriver,
3153         backend_id => "$audiodriver-backend${id}",
3154     };
3155 }
3156
3157 sub audio_devs {
3158     my ($audio, $audiopciaddr, $machine_version) = @_;
3159
3160     my $devs = [];
3161
3162     my $id = $audio->{dev_id};
3163     my $audiodev = "";
3164     if (min_version($machine_version, 4, 2)) {
3165         $audiodev = ",audiodev=$audio->{backend_id}";
3166     }
3167
3168     if ($audio->{dev} eq 'AC97') {
3169         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3170     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3171         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3172         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3173         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3174     } else {
3175         die "unkown audio device '$audio->{dev}', implement me!";
3176     }
3177
3178     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3179
3180     return $devs;
3181 }
3182
3183 sub get_tpm_paths {
3184     my ($vmid) = @_;
3185     return {
3186         socket => "/var/run/qemu-server/$vmid.swtpm",
3187         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3188     };
3189 }
3190
3191 sub add_tpm_device {
3192     my ($vmid, $devices, $conf) = @_;
3193
3194     return if !$conf->{tpmstate0};
3195
3196     my $paths = get_tpm_paths($vmid);
3197
3198     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3199     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3200     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3201 }
3202
3203 sub start_swtpm {
3204     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3205
3206     return if !$tpmdrive;
3207
3208     my $state;
3209     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3210     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3211     if ($storeid) {
3212         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3213     } else {
3214         $state = $tpm->{file};
3215     }
3216
3217     my $paths = get_tpm_paths($vmid);
3218
3219     # during migration, we will get state from remote
3220     #
3221     if (!$migration) {
3222         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3223         my $setup_cmd = [
3224             "swtpm_setup",
3225             "--tpmstate",
3226             "file://$state",
3227             "--createek",
3228             "--create-ek-cert",
3229             "--create-platform-cert",
3230             "--lock-nvram",
3231             "--config",
3232             "/etc/swtpm_setup.conf", # do not use XDG configs
3233             "--runas",
3234             "0", # force creation as root, error if not possible
3235             "--not-overwrite", # ignore existing state, do not modify
3236         ];
3237
3238         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3239         # TPM 2.0 supports ECC crypto, use if possible
3240         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3241
3242         run_command($setup_cmd, outfunc => sub {
3243             print "swtpm_setup: $1\n";
3244         });
3245     }
3246
3247     my $emulator_cmd = [
3248         "swtpm",
3249         "socket",
3250         "--tpmstate",
3251         "backend-uri=file://$state,mode=0600",
3252         "--ctrl",
3253         "type=unixio,path=$paths->{socket},mode=0600",
3254         "--pid",
3255         "file=$paths->{pid}",
3256         "--terminate", # terminate on QEMU disconnect
3257         "--daemon",
3258     ];
3259     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3260     run_command($emulator_cmd, outfunc => sub { print $1; });
3261
3262     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3263     while (! -e $paths->{pid}) {
3264         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3265         usleep(50_000);
3266     }
3267
3268     # return untainted PID of swtpm daemon so it can be killed on error
3269     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3270     return $1;
3271 }
3272
3273 sub vga_conf_has_spice {
3274     my ($vga) = @_;
3275
3276     my $vgaconf = parse_vga($vga);
3277     my $vgatype = $vgaconf->{type};
3278     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3279
3280     return $1 || 1;
3281 }
3282
3283 sub is_native($) {
3284     my ($arch) = @_;
3285     return get_host_arch() eq $arch;
3286 }
3287
3288 sub get_vm_arch {
3289     my ($conf) = @_;
3290     return $conf->{arch} // get_host_arch();
3291 }
3292
3293 my $default_machines = {
3294     x86_64 => 'pc',
3295     aarch64 => 'virt',
3296 };
3297
3298 sub get_installed_machine_version {
3299     my ($kvmversion) = @_;
3300     $kvmversion = kvm_user_version() if !defined($kvmversion);
3301     $kvmversion =~ m/^(\d+\.\d+)/;
3302     return $1;
3303 }
3304
3305 sub windows_get_pinned_machine_version {
3306     my ($machine, $base_version, $kvmversion) = @_;
3307
3308     my $pin_version = $base_version;
3309     if (!defined($base_version) ||
3310         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3311     ) {
3312         $pin_version = get_installed_machine_version($kvmversion);
3313     }
3314     if (!$machine || $machine eq 'pc') {
3315         $machine = "pc-i440fx-$pin_version";
3316     } elsif ($machine eq 'q35') {
3317         $machine = "pc-q35-$pin_version";
3318     } elsif ($machine eq 'virt') {
3319         $machine = "virt-$pin_version";
3320     } else {
3321         warn "unknown machine type '$machine', not touching that!\n";
3322     }
3323
3324     return $machine;
3325 }
3326
3327 sub get_vm_machine {
3328     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3329
3330     my $machine = $forcemachine || $conf->{machine};
3331
3332     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3333         $kvmversion //= kvm_user_version();
3334         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3335         # layout which confuses windows quite a bit and may result in various regressions..
3336         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3337         if (windows_version($conf->{ostype})) {
3338             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3339         }
3340         $arch //= 'x86_64';
3341         $machine ||= $default_machines->{$arch};
3342         if ($add_pve_version) {
3343             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3344             $machine .= "+pve$pvever";
3345         }
3346     }
3347
3348     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3349         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3350         $machine = $1 if $is_pxe;
3351
3352         # for version-pinned machines that do not include a pve-version (e.g.
3353         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3354         $machine .= '+pve0';
3355
3356         $machine .= '.pxe' if $is_pxe;
3357     }
3358
3359     return $machine;
3360 }
3361
3362 sub get_ovmf_files($$$) {
3363     my ($arch, $efidisk, $smm) = @_;
3364
3365     my $types = $OVMF->{$arch}
3366         or die "no OVMF images known for architecture '$arch'\n";
3367
3368     my $type = 'default';
3369     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3370         $type = $smm ? "4m" : "4m-no-smm";
3371         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3372     }
3373
3374     return $types->{$type}->@*;
3375 }
3376
3377 my $Arch2Qemu = {
3378     aarch64 => '/usr/bin/qemu-system-aarch64',
3379     x86_64 => '/usr/bin/qemu-system-x86_64',
3380 };
3381 sub get_command_for_arch($) {
3382     my ($arch) = @_;
3383     return '/usr/bin/kvm' if is_native($arch);
3384
3385     my $cmd = $Arch2Qemu->{$arch}
3386         or die "don't know how to emulate architecture '$arch'\n";
3387     return $cmd;
3388 }
3389
3390 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3391 # to use in a QEMU command line (-cpu element), first array_intersect the result
3392 # of query_supported_ with query_understood_. This is necessary because:
3393 #
3394 # a) query_understood_ returns flags the host cannot use and
3395 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3396 #    flags, but CPU settings - with most of them being flags. Those settings
3397 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3398 #
3399 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3400 # expensive. If you need the value returned from this, you can get it much
3401 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3402 # $accel being 'kvm' or 'tcg'.
3403 #
3404 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3405 # changes, automatically populating pmxcfs.
3406 #
3407 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3408 # since kvm and tcg machines support different flags
3409 #
3410 sub query_supported_cpu_flags {
3411     my ($arch) = @_;
3412
3413     $arch //= get_host_arch();
3414     my $default_machine = $default_machines->{$arch};
3415
3416     my $flags = {};
3417
3418     # FIXME: Once this is merged, the code below should work for ARM as well:
3419     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3420     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3421         $arch eq "aarch64";
3422
3423     my $kvm_supported = defined(kvm_version());
3424     my $qemu_cmd = get_command_for_arch($arch);
3425     my $fakevmid = -1;
3426     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3427
3428     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3429     my $query_supported_run_qemu = sub {
3430         my ($kvm) = @_;
3431
3432         my $flags = {};
3433         my $cmd = [
3434             $qemu_cmd,
3435             '-machine', $default_machine,
3436             '-display', 'none',
3437             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3438             '-mon', 'chardev=qmp,mode=control',
3439             '-pidfile', $pidfile,
3440             '-S', '-daemonize'
3441         ];
3442
3443         if (!$kvm) {
3444             push @$cmd, '-accel', 'tcg';
3445         }
3446
3447         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3448         die "QEMU flag querying VM exited with code " . $rc if $rc;
3449
3450         eval {
3451             my $cmd_result = mon_cmd(
3452                 $fakevmid,
3453                 'query-cpu-model-expansion',
3454                 type => 'full',
3455                 model => { name => 'host' }
3456             );
3457
3458             my $props = $cmd_result->{model}->{props};
3459             foreach my $prop (keys %$props) {
3460                 next if $props->{$prop} ne '1';
3461                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3462                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3463                 # We only keep those with underscores, to match /proc/cpuinfo
3464                 $prop =~ s/\.|-/_/g;
3465                 $flags->{$prop} = 1;
3466             }
3467         };
3468         my $err = $@;
3469
3470         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3471         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3472
3473         die $err if $err;
3474
3475         return [ sort keys %$flags ];
3476     };
3477
3478     # We need to query QEMU twice, since KVM and TCG have different supported flags
3479     PVE::QemuConfig->lock_config($fakevmid, sub {
3480         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3481         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3482
3483         if ($kvm_supported) {
3484             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3485             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3486         }
3487     });
3488
3489     return $flags;
3490 }
3491
3492 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3493 my $understood_cpu_flag_dir = "/usr/share/kvm";
3494 sub query_understood_cpu_flags {
3495     my $arch = get_host_arch();
3496     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3497
3498     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3499         if ! -e $filepath;
3500
3501     my $raw = file_get_contents($filepath);
3502     $raw =~ s/^\s+|\s+$//g;
3503     my @flags = split(/\s+/, $raw);
3504
3505     return \@flags;
3506 }
3507
3508 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3509 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3510 my sub should_disable_smm {
3511     my ($conf, $vga) = @_;
3512
3513     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3514         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3515 }
3516
3517 sub config_to_command {
3518     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3519         $pbs_backing) = @_;
3520
3521     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3522     my $devices = [];
3523     my $bridges = {};
3524     my $ostype = $conf->{ostype};
3525     my $winversion = windows_version($ostype);
3526     my $kvm = $conf->{kvm};
3527     my $nodename = nodename();
3528
3529     my $arch = get_vm_arch($conf);
3530     my $kvm_binary = get_command_for_arch($arch);
3531     my $kvmver = kvm_user_version($kvm_binary);
3532
3533     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3534         $kvmver //= "undefined";
3535         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3536     }
3537
3538     my $add_pve_version = min_version($kvmver, 4, 1);
3539
3540     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3541     my $machine_version = extract_version($machine_type, $kvmver);
3542     $kvm //= 1 if is_native($arch);
3543
3544     $machine_version =~ m/(\d+)\.(\d+)/;
3545     my ($machine_major, $machine_minor) = ($1, $2);
3546
3547     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3548         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3549     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3550         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3551             ." please upgrade node '$nodename'\n"
3552     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3553         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3554         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3555             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3556             ." node '$nodename'\n";
3557     }
3558
3559     # if a specific +pve version is required for a feature, use $version_guard
3560     # instead of min_version to allow machines to be run with the minimum
3561     # required version
3562     my $required_pve_version = 0;
3563     my $version_guard = sub {
3564         my ($major, $minor, $pve) = @_;
3565         return 0 if !min_version($machine_version, $major, $minor, $pve);
3566         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3567         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3568         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3569         return 1;
3570     };
3571
3572     if ($kvm && !defined kvm_version()) {
3573         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3574             ." or enable in BIOS.\n";
3575     }
3576
3577     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3578     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3579     my $use_old_bios_files = undef;
3580     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3581
3582     my $cmd = [];
3583     if ($conf->{affinity}) {
3584         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3585     }
3586
3587     push @$cmd, $kvm_binary;
3588
3589     push @$cmd, '-id', $vmid;
3590
3591     my $vmname = $conf->{name} || "vm$vmid";
3592
3593     push @$cmd, '-name', "$vmname,debug-threads=on";
3594
3595     push @$cmd, '-no-shutdown';
3596
3597     my $use_virtio = 0;
3598
3599     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3600     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3601     push @$cmd, '-mon', "chardev=qmp,mode=control";
3602
3603     if (min_version($machine_version, 2, 12)) {
3604         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3605         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3606     }
3607
3608     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3609
3610     push @$cmd, '-daemonize';
3611
3612     if ($conf->{smbios1}) {
3613         my $smbios_conf = parse_smbios1($conf->{smbios1});
3614         if ($smbios_conf->{base64}) {
3615             # Do not pass base64 flag to qemu
3616             delete $smbios_conf->{base64};
3617             my $smbios_string = "";
3618             foreach my $key (keys %$smbios_conf) {
3619                 my $value;
3620                 if ($key eq "uuid") {
3621                     $value = $smbios_conf->{uuid}
3622                 } else {
3623                     $value = decode_base64($smbios_conf->{$key});
3624                 }
3625                 # qemu accepts any binary data, only commas need escaping by double comma
3626                 $value =~ s/,/,,/g;
3627                 $smbios_string .= "," . $key . "=" . $value if $value;
3628             }
3629             push @$cmd, '-smbios', "type=1" . $smbios_string;
3630         } else {
3631             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3632         }
3633     }
3634
3635     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3636         my $d;
3637         if (my $efidisk = $conf->{efidisk0}) {
3638             $d = parse_drive('efidisk0', $efidisk);
3639         }
3640
3641         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3642         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3643
3644         my ($path, $format);
3645         my $read_only_str = '';
3646         if ($d) {
3647             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3648             $format = $d->{format};
3649             if ($storeid) {
3650                 $path = PVE::Storage::path($storecfg, $d->{file});
3651                 if (!defined($format)) {
3652                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3653                     $format = qemu_img_format($scfg, $volname);
3654                 }
3655             } else {
3656                 $path = $d->{file};
3657                 die "efidisk format must be specified\n"
3658                     if !defined($format);
3659             }
3660
3661             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3662         } else {
3663             log_warn("no efidisk configured! Using temporary efivars disk.");
3664             $path = "/tmp/$vmid-ovmf.fd";
3665             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3666             $format = 'raw';
3667         }
3668
3669         my $size_str = "";
3670
3671         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3672             $size_str = ",size=" . (-s $ovmf_vars);
3673         }
3674
3675         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3676         my $cache = "";
3677         if ($path =~ m/^rbd:/) {
3678                 $cache = ',cache=writeback';
3679                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3680         }
3681
3682         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3683         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3684     }
3685
3686     if ($q35) { # tell QEMU to load q35 config early
3687         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3688         if (min_version($machine_version, 4, 0)) {
3689             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3690         } else {
3691             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3692         }
3693     }
3694
3695     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3696         push @$cmd, $fixups->@*;
3697     }
3698
3699     if ($conf->{vmgenid}) {
3700         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3701     }
3702
3703     # add usb controllers
3704     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3705         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3706     push @$devices, @usbcontrollers if @usbcontrollers;
3707     my $vga = parse_vga($conf->{vga});
3708
3709     my $qxlnum = vga_conf_has_spice($conf->{vga});
3710     $vga->{type} = 'qxl' if $qxlnum;
3711
3712     if (!$vga->{type}) {
3713         if ($arch eq 'aarch64') {
3714             $vga->{type} = 'virtio';
3715         } elsif (min_version($machine_version, 2, 9)) {
3716             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3717         } else {
3718             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3719         }
3720     }
3721
3722     # enable absolute mouse coordinates (needed by vnc)
3723     my $tablet = $conf->{tablet};
3724     if (!defined($tablet)) {
3725         $tablet = $defaults->{tablet};
3726         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3727         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3728     }
3729
3730     if ($tablet) {
3731         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3732         my $kbd = print_keyboarddevice_full($conf, $arch);
3733         push @$devices, '-device', $kbd if defined($kbd);
3734     }
3735
3736     my $bootorder = device_bootorder($conf);
3737
3738     # host pci device passthrough
3739     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3740         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3741
3742     # usb devices
3743     my $usb_dev_features = {};
3744     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3745
3746     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3747         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3748     push @$devices, @usbdevices if @usbdevices;
3749
3750     # serial devices
3751     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3752         my $path = $conf->{"serial$i"} or next;
3753         if ($path eq 'socket') {
3754             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3755             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3756             # On aarch64, serial0 is the UART device. Qemu only allows
3757             # connecting UART devices via the '-serial' command line, as
3758             # the device has a fixed slot on the hardware...
3759             if ($arch eq 'aarch64' && $i == 0) {
3760                 push @$devices, '-serial', "chardev:serial$i";
3761             } else {
3762                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3763             }
3764         } else {
3765             die "no such serial device\n" if ! -c $path;
3766             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3767             push @$devices, '-device', "isa-serial,chardev=serial$i";
3768         }
3769     }
3770
3771     # parallel devices
3772     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3773         if (my $path = $conf->{"parallel$i"}) {
3774             die "no such parallel device\n" if ! -c $path;
3775             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3776             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3777             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3778         }
3779     }
3780
3781     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3782         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3783         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3784         push @$devices, @$audio_devs;
3785     }
3786
3787     add_tpm_device($vmid, $devices, $conf);
3788
3789     my $sockets = 1;
3790     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3791     $sockets = $conf->{sockets} if  $conf->{sockets};
3792
3793     my $cores = $conf->{cores} || 1;
3794
3795     my $maxcpus = $sockets * $cores;
3796
3797     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3798
3799     my $allowed_vcpus = $cpuinfo->{cpus};
3800
3801     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3802
3803     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3804         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3805         for (my $i = 2; $i <= $vcpus; $i++)  {
3806             my $cpustr = print_cpu_device($conf,$i);
3807             push @$cmd, '-device', $cpustr;
3808         }
3809
3810     } else {
3811
3812         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3813     }
3814     push @$cmd, '-nodefaults';
3815
3816     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3817
3818     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3819
3820     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3821
3822     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3823         push @$devices, '-device', print_vga_device(
3824             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3825
3826         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3827
3828         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3829         push @$cmd,  '-vnc', "unix:$socket,password=on";
3830     } else {
3831         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3832         push @$cmd, '-nographic';
3833     }
3834
3835     # time drift fix
3836     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3837     my $useLocaltime = $conf->{localtime};
3838
3839     if ($winversion >= 5) { # windows
3840         $useLocaltime = 1 if !defined($conf->{localtime});
3841
3842         # use time drift fix when acpi is enabled
3843         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3844             $tdf = 1 if !defined($conf->{tdf});
3845         }
3846     }
3847
3848     if ($winversion >= 6) {
3849         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3850         push @$cmd, '-no-hpet';
3851     }
3852
3853     push @$rtcFlags, 'driftfix=slew' if $tdf;
3854
3855     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3856         push @$rtcFlags, "base=$conf->{startdate}";
3857     } elsif ($useLocaltime) {
3858         push @$rtcFlags, 'base=localtime';
3859     }
3860
3861     if ($forcecpu) {
3862         push @$cmd, '-cpu', $forcecpu;
3863     } else {
3864         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3865     }
3866
3867     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3868
3869     push @$cmd, '-S' if $conf->{freeze};
3870
3871     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3872
3873     my $guest_agent = parse_guest_agent($conf);
3874
3875     if ($guest_agent->{enabled}) {
3876         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3877         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3878
3879         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3880             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3881             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3882             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3883         } elsif ($guest_agent->{type} eq 'isa') {
3884             push @$devices, '-device', "isa-serial,chardev=qga0";
3885         }
3886     }
3887
3888     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3889     if ($rng && $version_guard->(4, 1, 2)) {
3890         check_rng_source($rng->{source});
3891
3892         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3893         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3894         my $limiter_str = "";
3895         if ($max_bytes) {
3896             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3897         }
3898
3899         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3900         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3901         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3902     }
3903
3904     my $spice_port;
3905
3906     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3907         if ($qxlnum > 1) {
3908             if ($winversion){
3909                 for (my $i = 1; $i < $qxlnum; $i++){
3910                     push @$devices, '-device', print_vga_device(
3911                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3912                 }
3913             } else {
3914                 # assume other OS works like Linux
3915                 my ($ram, $vram) = ("134217728", "67108864");
3916                 if ($vga->{memory}) {
3917                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3918                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3919                 }
3920                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3921                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3922             }
3923         }
3924
3925         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3926
3927         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3928         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3929         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3930
3931         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3932         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3933         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3934
3935         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3936         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3937
3938         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3939         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3940         if ($spice_enhancement->{foldersharing}) {
3941             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3942             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3943         }
3944
3945         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3946         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3947             if $spice_enhancement->{videostreaming};
3948
3949         push @$devices, '-spice', "$spice_opts";
3950     }
3951
3952     # enable balloon by default, unless explicitly disabled
3953     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3954         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3955         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3956         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3957         push @$devices, '-device', $ballooncmd;
3958     }
3959
3960     if ($conf->{watchdog}) {
3961         my $wdopts = parse_watchdog($conf->{watchdog});
3962         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3963         my $watchdog = $wdopts->{model} || 'i6300esb';
3964         push @$devices, '-device', "$watchdog$pciaddr";
3965         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3966     }
3967
3968     my $vollist = [];
3969     my $scsicontroller = {};
3970     my $ahcicontroller = {};
3971     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3972
3973     # Add iscsi initiator name if available
3974     if (my $initiator = get_initiator_name()) {
3975         push @$devices, '-iscsi', "initiator-name=$initiator";
3976     }
3977
3978     PVE::QemuConfig->foreach_volume($conf, sub {
3979         my ($ds, $drive) = @_;
3980
3981         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3982             check_volume_storage_type($storecfg, $drive->{file});
3983             push @$vollist, $drive->{file};
3984         }
3985
3986         # ignore efidisk here, already added in bios/fw handling code above
3987         return if $drive->{interface} eq 'efidisk';
3988         # similar for TPM
3989         return if $drive->{interface} eq 'tpmstate';
3990
3991         $use_virtio = 1 if $ds =~ m/^virtio/;
3992
3993         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3994
3995         if ($drive->{interface} eq 'virtio'){
3996            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3997         }
3998
3999         if ($drive->{interface} eq 'scsi') {
4000
4001             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4002
4003             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4004                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4005
4006             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4007             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4008
4009             my $iothread = '';
4010             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4011                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4012                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4013             } elsif ($drive->{iothread}) {
4014                 log_warn(
4015                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4016                 );
4017             }
4018
4019             my $queues = '';
4020             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4021                 $queues = ",num_queues=$drive->{queues}";
4022             }
4023
4024             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4025                 if !$scsicontroller->{$controller};
4026             $scsicontroller->{$controller}=1;
4027         }
4028
4029         if ($drive->{interface} eq 'sata') {
4030             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4031             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4032             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4033                 if !$ahcicontroller->{$controller};
4034             $ahcicontroller->{$controller}=1;
4035         }
4036
4037         my $pbs_conf = $pbs_backing->{$ds};
4038         my $pbs_name = undef;
4039         if ($pbs_conf) {
4040             $pbs_name = "drive-$ds-pbs";
4041             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4042         }
4043
4044         my $drive_cmd = print_drive_commandline_full(
4045             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4046
4047         # extra protection for templates, but SATA and IDE don't support it..
4048         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4049
4050         push @$devices, '-drive',$drive_cmd;
4051         push @$devices, '-device', print_drivedevice_full(
4052             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4053     });
4054
4055     for (my $i = 0; $i < $MAX_NETS; $i++) {
4056         my $netname = "net$i";
4057
4058         next if !$conf->{$netname};
4059         my $d = parse_net($conf->{$netname});
4060         next if !$d;
4061         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4062
4063         $use_virtio = 1 if $d->{model} eq 'virtio';
4064
4065         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4066
4067         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4068         push @$devices, '-netdev', $netdevfull;
4069
4070         my $netdevicefull = print_netdevice_full(
4071             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4072
4073         push @$devices, '-device', $netdevicefull;
4074     }
4075
4076     if ($conf->{ivshmem}) {
4077         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4078
4079         my $bus;
4080         if ($q35) {
4081             $bus = print_pcie_addr("ivshmem");
4082         } else {
4083             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4084         }
4085
4086         my $ivshmem_name = $ivshmem->{name} // $vmid;
4087         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4088
4089         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4090         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4091             .",size=$ivshmem->{size}M";
4092     }
4093
4094     # pci.4 is nested in pci.1
4095     $bridges->{1} = 1 if $bridges->{4};
4096
4097     if (!$q35) { # add pci bridges
4098         if (min_version($machine_version, 2, 3)) {
4099            $bridges->{1} = 1;
4100            $bridges->{2} = 1;
4101         }
4102         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4103     }
4104
4105     for my $k (sort {$b cmp $a} keys %$bridges) {
4106         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4107
4108         my $k_name = $k;
4109         if ($k == 2 && $legacy_igd) {
4110             $k_name = "$k-igd";
4111         }
4112         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4113         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4114
4115         if ($q35) { # add after -readconfig pve-q35.cfg
4116             splice @$devices, 2, 0, '-device', $devstr;
4117         } else {
4118             unshift @$devices, '-device', $devstr if $k > 0;
4119         }
4120     }
4121
4122     if (!$kvm) {
4123         push @$machineFlags, 'accel=tcg';
4124     }
4125
4126     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4127
4128     my $machine_type_min = $machine_type;
4129     if ($add_pve_version) {
4130         $machine_type_min =~ s/\+pve\d+$//;
4131         $machine_type_min .= "+pve$required_pve_version";
4132     }
4133     push @$machineFlags, "type=${machine_type_min}";
4134
4135     push @$cmd, @$devices;
4136     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4137     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4138     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4139
4140     if (my $vmstate = $conf->{vmstate}) {
4141         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4142         push @$vollist, $vmstate;
4143         push @$cmd, '-loadstate', $statepath;
4144         print "activating and using '$vmstate' as vmstate\n";
4145     }
4146
4147     if (PVE::QemuConfig->is_template($conf)) {
4148         # needed to workaround base volumes being read-only
4149         push @$cmd, '-snapshot';
4150     }
4151
4152     # add custom args
4153     if ($conf->{args}) {
4154         my $aa = PVE::Tools::split_args($conf->{args});
4155         push @$cmd, @$aa;
4156     }
4157
4158     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4159 }
4160
4161 sub check_rng_source {
4162     my ($source) = @_;
4163
4164     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4165     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4166         if ! -e $source;
4167
4168     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4169     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4170         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4171         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4172         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4173             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4174             ." to the host.\n";
4175     }
4176 }
4177
4178 sub spice_port {
4179     my ($vmid) = @_;
4180
4181     my $res = mon_cmd($vmid, 'query-spice');
4182
4183     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4184 }
4185
4186 sub vm_devices_list {
4187     my ($vmid) = @_;
4188
4189     my $res = mon_cmd($vmid, 'query-pci');
4190     my $devices_to_check = [];
4191     my $devices = {};
4192     foreach my $pcibus (@$res) {
4193         push @$devices_to_check, @{$pcibus->{devices}},
4194     }
4195
4196     while (@$devices_to_check) {
4197         my $to_check = [];
4198         for my $d (@$devices_to_check) {
4199             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4200             next if !$d->{'pci_bridge'};
4201
4202             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4203             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4204         }
4205         $devices_to_check = $to_check;
4206     }
4207
4208     my $resblock = mon_cmd($vmid, 'query-block');
4209     foreach my $block (@$resblock) {
4210         if($block->{device} =~ m/^drive-(\S+)/){
4211                 $devices->{$1} = 1;
4212         }
4213     }
4214
4215     my $resmice = mon_cmd($vmid, 'query-mice');
4216     foreach my $mice (@$resmice) {
4217         if ($mice->{name} eq 'QEMU HID Tablet') {
4218             $devices->{tablet} = 1;
4219             last;
4220         }
4221     }
4222
4223     # for usb devices there is no query-usb
4224     # but we can iterate over the entries in
4225     # qom-list path=/machine/peripheral
4226     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4227     foreach my $per (@$resperipheral) {
4228         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4229             $devices->{$per->{name}} = 1;
4230         }
4231     }
4232
4233     return $devices;
4234 }
4235
4236 sub vm_deviceplug {
4237     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4238
4239     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4240
4241     my $devices_list = vm_devices_list($vmid);
4242     return 1 if defined($devices_list->{$deviceid});
4243
4244     # add PCI bridge if we need it for the device
4245     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4246
4247     if ($deviceid eq 'tablet') {
4248         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4249     } elsif ($deviceid eq 'keyboard') {
4250         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4251     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4252         my $id = $1;
4253         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4254         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4255     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4256         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4257     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4258         qemu_iothread_add($vmid, $deviceid, $device);
4259
4260         qemu_driveadd($storecfg, $vmid, $device);
4261         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4262
4263         qemu_deviceadd($vmid, $devicefull);
4264         eval { qemu_deviceaddverify($vmid, $deviceid); };
4265         if (my $err = $@) {
4266             eval { qemu_drivedel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4271         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4272         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4273         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4274
4275         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4276
4277         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4278             qemu_iothread_add($vmid, $deviceid, $device);
4279             $devicefull .= ",iothread=iothread-$deviceid";
4280         }
4281
4282         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4283             $devicefull .= ",num_queues=$device->{queues}";
4284         }
4285
4286         qemu_deviceadd($vmid, $devicefull);
4287         qemu_deviceaddverify($vmid, $deviceid);
4288     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4289         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4290         qemu_driveadd($storecfg, $vmid, $device);
4291
4292         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4293         eval { qemu_deviceadd($vmid, $devicefull); };
4294         if (my $err = $@) {
4295             eval { qemu_drivedel($vmid, $deviceid); };
4296             warn $@ if $@;
4297             die $err;
4298         }
4299     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4300         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4301
4302         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4303         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4304         my $use_old_bios_files = undef;
4305         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4306
4307         my $netdevicefull = print_netdevice_full(
4308             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4309         qemu_deviceadd($vmid, $netdevicefull);
4310         eval {
4311             qemu_deviceaddverify($vmid, $deviceid);
4312             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4313         };
4314         if (my $err = $@) {
4315             eval { qemu_netdevdel($vmid, $deviceid); };
4316             warn $@ if $@;
4317             die $err;
4318         }
4319     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4320         my $bridgeid = $2;
4321         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4322         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4323
4324         qemu_deviceadd($vmid, $devicefull);
4325         qemu_deviceaddverify($vmid, $deviceid);
4326     } else {
4327         die "can't hotplug device '$deviceid'\n";
4328     }
4329
4330     return 1;
4331 }
4332
4333 # fixme: this should raise exceptions on error!
4334 sub vm_deviceunplug {
4335     my ($vmid, $conf, $deviceid) = @_;
4336
4337     my $devices_list = vm_devices_list($vmid);
4338     return 1 if !defined($devices_list->{$deviceid});
4339
4340     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4341     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4342
4343     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4344         qemu_devicedel($vmid, $deviceid);
4345     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348     } elsif ($deviceid =~ m/^usb\d+$/) {
4349         qemu_devicedel($vmid, $deviceid);
4350         qemu_devicedelverify($vmid, $deviceid);
4351     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4352         my $device = parse_drive($deviceid, $conf->{$deviceid});
4353
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356         qemu_drivedel($vmid, $deviceid);
4357         qemu_iothread_del($vmid, $deviceid, $device);
4358     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4359         qemu_devicedel($vmid, $deviceid);
4360         qemu_devicedelverify($vmid, $deviceid);
4361     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4362         my $device = parse_drive($deviceid, $conf->{$deviceid});
4363
4364         qemu_devicedel($vmid, $deviceid);
4365         qemu_devicedelverify($vmid, $deviceid);
4366         qemu_drivedel($vmid, $deviceid);
4367         qemu_deletescsihw($conf, $vmid, $deviceid);
4368
4369         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4370             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4371     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4372         qemu_devicedel($vmid, $deviceid);
4373         qemu_devicedelverify($vmid, $deviceid);
4374         qemu_netdevdel($vmid, $deviceid);
4375     } else {
4376         die "can't unplug device '$deviceid'\n";
4377     }
4378
4379     return 1;
4380 }
4381
4382 sub qemu_spice_usbredir_chardev_add {
4383     my ($vmid, $id) = @_;
4384
4385     mon_cmd($vmid, "chardev-add" , (
4386         id => $id,
4387         backend => {
4388             type => 'spicevmc',
4389             data => {
4390                 type => "usbredir",
4391             },
4392         },
4393     ));
4394 }
4395
4396 sub qemu_deviceadd {
4397     my ($vmid, $devicefull) = @_;
4398
4399     $devicefull = "driver=".$devicefull;
4400     my %options =  split(/[=,]/, $devicefull);
4401
4402     mon_cmd($vmid, "device_add" , %options);
4403 }
4404
4405 sub qemu_devicedel {
4406     my ($vmid, $deviceid) = @_;
4407
4408     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4409 }
4410
4411 sub qemu_iothread_add {
4412     my ($vmid, $deviceid, $device) = @_;
4413
4414     if ($device->{iothread}) {
4415         my $iothreads = vm_iothreads_list($vmid);
4416         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4417     }
4418 }
4419
4420 sub qemu_iothread_del {
4421     my ($vmid, $deviceid, $device) = @_;
4422
4423     if ($device->{iothread}) {
4424         my $iothreads = vm_iothreads_list($vmid);
4425         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4426     }
4427 }
4428
4429 sub qemu_objectadd {
4430     my ($vmid, $objectid, $qomtype) = @_;
4431
4432     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4433
4434     return 1;
4435 }
4436
4437 sub qemu_objectdel {
4438     my ($vmid, $objectid) = @_;
4439
4440     mon_cmd($vmid, "object-del", id => $objectid);
4441
4442     return 1;
4443 }
4444
4445 sub qemu_driveadd {
4446     my ($storecfg, $vmid, $device) = @_;
4447
4448     my $kvmver = get_running_qemu_version($vmid);
4449     my $io_uring = min_version($kvmver, 6, 0);
4450     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4451     $drive =~ s/\\/\\\\/g;
4452     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4453
4454     # If the command succeeds qemu prints: "OK"
4455     return 1 if $ret =~ m/OK/s;
4456
4457     die "adding drive failed: $ret\n";
4458 }
4459
4460 sub qemu_drivedel {
4461     my ($vmid, $deviceid) = @_;
4462
4463     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4464     $ret =~ s/^\s+//;
4465
4466     return 1 if $ret eq "";
4467
4468     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4469     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4470
4471     die "deleting drive $deviceid failed : $ret\n";
4472 }
4473
4474 sub qemu_deviceaddverify {
4475     my ($vmid, $deviceid) = @_;
4476
4477     for (my $i = 0; $i <= 5; $i++) {
4478          my $devices_list = vm_devices_list($vmid);
4479          return 1 if defined($devices_list->{$deviceid});
4480          sleep 1;
4481     }
4482
4483     die "error on hotplug device '$deviceid'\n";
4484 }
4485
4486
4487 sub qemu_devicedelverify {
4488     my ($vmid, $deviceid) = @_;
4489
4490     # need to verify that the device is correctly removed as device_del
4491     # is async and empty return is not reliable
4492
4493     for (my $i = 0; $i <= 5; $i++) {
4494          my $devices_list = vm_devices_list($vmid);
4495          return 1 if !defined($devices_list->{$deviceid});
4496          sleep 1;
4497     }
4498
4499     die "error on hot-unplugging device '$deviceid'\n";
4500 }
4501
4502 sub qemu_findorcreatescsihw {
4503     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4504
4505     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4506
4507     my $scsihwid="$controller_prefix$controller";
4508     my $devices_list = vm_devices_list($vmid);
4509
4510     if (!defined($devices_list->{$scsihwid})) {
4511         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4512     }
4513
4514     return 1;
4515 }
4516
4517 sub qemu_deletescsihw {
4518     my ($conf, $vmid, $opt) = @_;
4519
4520     my $device = parse_drive($opt, $conf->{$opt});
4521
4522     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4523         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4524         return 1;
4525     }
4526
4527     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4528
4529     my $devices_list = vm_devices_list($vmid);
4530     foreach my $opt (keys %{$devices_list}) {
4531         if (is_valid_drivename($opt)) {
4532             my $drive = parse_drive($opt, $conf->{$opt});
4533             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4534                 return 1;
4535             }
4536         }
4537     }
4538
4539     my $scsihwid="scsihw$controller";
4540
4541     vm_deviceunplug($vmid, $conf, $scsihwid);
4542
4543     return 1;
4544 }
4545
4546 sub qemu_add_pci_bridge {
4547     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4548
4549     my $bridges = {};
4550
4551     my $bridgeid;
4552
4553     print_pci_addr($device, $bridges, $arch, $machine_type);
4554
4555     while (my ($k, $v) = each %$bridges) {
4556         $bridgeid = $k;
4557     }
4558     return 1 if !defined($bridgeid) || $bridgeid < 1;
4559
4560     my $bridge = "pci.$bridgeid";
4561     my $devices_list = vm_devices_list($vmid);
4562
4563     if (!defined($devices_list->{$bridge})) {
4564         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4565     }
4566
4567     return 1;
4568 }
4569
4570 sub qemu_set_link_status {
4571     my ($vmid, $device, $up) = @_;
4572
4573     mon_cmd($vmid, "set_link", name => $device,
4574                up => $up ? JSON::true : JSON::false);
4575 }
4576
4577 sub qemu_netdevadd {
4578     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4579
4580     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4581     my %options =  split(/[=,]/, $netdev);
4582
4583     if (defined(my $vhost = $options{vhost})) {
4584         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4585     }
4586
4587     if (defined(my $queues = $options{queues})) {
4588         $options{queues} = $queues + 0;
4589     }
4590
4591     mon_cmd($vmid, "netdev_add",  %options);
4592     return 1;
4593 }
4594
4595 sub qemu_netdevdel {
4596     my ($vmid, $deviceid) = @_;
4597
4598     mon_cmd($vmid, "netdev_del", id => $deviceid);
4599 }
4600
4601 sub qemu_usb_hotplug {
4602     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4603
4604     return if !$device;
4605
4606     # remove the old one first
4607     vm_deviceunplug($vmid, $conf, $deviceid);
4608
4609     # check if xhci controller is necessary and available
4610     my $devicelist = vm_devices_list($vmid);
4611
4612     if (!$devicelist->{xhci}) {
4613         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4614         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4615     }
4616
4617     # print_usbdevice_full expects the parsed device
4618     my $d = parse_usb_device($device->{host});
4619     $d->{usb3} = $device->{usb3};
4620
4621     # add the new one
4622     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4623 }
4624
4625 sub qemu_cpu_hotplug {
4626     my ($vmid, $conf, $vcpus) = @_;
4627
4628     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4629
4630     my $sockets = 1;
4631     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4632     $sockets = $conf->{sockets} if  $conf->{sockets};
4633     my $cores = $conf->{cores} || 1;
4634     my $maxcpus = $sockets * $cores;
4635
4636     $vcpus = $maxcpus if !$vcpus;
4637
4638     die "you can't add more vcpus than maxcpus\n"
4639         if $vcpus > $maxcpus;
4640
4641     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4642
4643     if ($vcpus < $currentvcpus) {
4644
4645         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4646
4647             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4648                 qemu_devicedel($vmid, "cpu$i");
4649                 my $retry = 0;
4650                 my $currentrunningvcpus = undef;
4651                 while (1) {
4652                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4653                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4654                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4655                     $retry++;
4656                     sleep 1;
4657                 }
4658                 #update conf after each succesfull cpu unplug
4659                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4660                 PVE::QemuConfig->write_config($vmid, $conf);
4661             }
4662         } else {
4663             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4664         }
4665
4666         return;
4667     }
4668
4669     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4670     die "vcpus in running vm does not match its configuration\n"
4671         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4672
4673     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4674
4675         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4676             my $cpustr = print_cpu_device($conf, $i);
4677             qemu_deviceadd($vmid, $cpustr);
4678
4679             my $retry = 0;
4680             my $currentrunningvcpus = undef;
4681             while (1) {
4682                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4683                 last if scalar(@{$currentrunningvcpus}) == $i;
4684                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4685                 sleep 1;
4686                 $retry++;
4687             }
4688             #update conf after each succesfull cpu hotplug
4689             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4690             PVE::QemuConfig->write_config($vmid, $conf);
4691         }
4692     } else {
4693
4694         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4695             mon_cmd($vmid, "cpu-add", id => int($i));
4696         }
4697     }
4698 }
4699
4700 sub qemu_block_set_io_throttle {
4701     my ($vmid, $deviceid,
4702         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4703         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4704         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4705         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4706
4707     return if !check_running($vmid) ;
4708
4709     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4710         bps => int($bps),
4711         bps_rd => int($bps_rd),
4712         bps_wr => int($bps_wr),
4713         iops => int($iops),
4714         iops_rd => int($iops_rd),
4715         iops_wr => int($iops_wr),
4716         bps_max => int($bps_max),
4717         bps_rd_max => int($bps_rd_max),
4718         bps_wr_max => int($bps_wr_max),
4719         iops_max => int($iops_max),
4720         iops_rd_max => int($iops_rd_max),
4721         iops_wr_max => int($iops_wr_max),
4722         bps_max_length => int($bps_max_length),
4723         bps_rd_max_length => int($bps_rd_max_length),
4724         bps_wr_max_length => int($bps_wr_max_length),
4725         iops_max_length => int($iops_max_length),
4726         iops_rd_max_length => int($iops_rd_max_length),
4727         iops_wr_max_length => int($iops_wr_max_length),
4728     );
4729
4730 }
4731
4732 sub qemu_block_resize {
4733     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4734
4735     my $running = check_running($vmid);
4736
4737     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4738
4739     return if !$running;
4740
4741     my $padding = (1024 - $size % 1024) % 1024;
4742     $size = $size + $padding;
4743
4744     mon_cmd(
4745         $vmid,
4746         "block_resize",
4747         device => $deviceid,
4748         size => int($size),
4749         timeout => 60,
4750     );
4751 }
4752
4753 sub qemu_volume_snapshot {
4754     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4759         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4760     } else {
4761         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4762     }
4763 }
4764
4765 sub qemu_volume_snapshot_delete {
4766     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4767
4768     my $running = check_running($vmid);
4769
4770     if($running) {
4771
4772         $running = undef;
4773         my $conf = PVE::QemuConfig->load_config($vmid);
4774         PVE::QemuConfig->foreach_volume($conf, sub {
4775             my ($ds, $drive) = @_;
4776             $running = 1 if $drive->{file} eq $volid;
4777         });
4778     }
4779
4780     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4781         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4782     } else {
4783         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4784     }
4785 }
4786
4787 sub set_migration_caps {
4788     my ($vmid, $savevm) = @_;
4789
4790     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4791
4792     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4793     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4794
4795     my $cap_ref = [];
4796
4797     my $enabled_cap = {
4798         "auto-converge" => 1,
4799         "xbzrle" => 1,
4800         "x-rdma-pin-all" => 0,
4801         "zero-blocks" => 0,
4802         "compress" => 0,
4803         "dirty-bitmaps" => $dirty_bitmaps,
4804     };
4805
4806     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4807
4808     for my $supported_capability (@$supported_capabilities) {
4809         push @$cap_ref, {
4810             capability => $supported_capability->{capability},
4811             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4812         };
4813     }
4814
4815     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4816 }
4817
4818 sub foreach_volid {
4819     my ($conf, $func, @param) = @_;
4820
4821     my $volhash = {};
4822
4823     my $test_volid = sub {
4824         my ($key, $drive, $snapname) = @_;
4825
4826         my $volid = $drive->{file};
4827         return if !$volid;
4828
4829         $volhash->{$volid}->{cdrom} //= 1;
4830         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4831
4832         my $replicate = $drive->{replicate} // 1;
4833         $volhash->{$volid}->{replicate} //= 0;
4834         $volhash->{$volid}->{replicate} = 1 if $replicate;
4835
4836         $volhash->{$volid}->{shared} //= 0;
4837         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4838
4839         $volhash->{$volid}->{referenced_in_config} //= 0;
4840         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4841
4842         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4843             if defined($snapname);
4844
4845         my $size = $drive->{size};
4846         $volhash->{$volid}->{size} //= $size if $size;
4847
4848         $volhash->{$volid}->{is_vmstate} //= 0;
4849         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4850
4851         $volhash->{$volid}->{is_tpmstate} //= 0;
4852         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4853
4854         $volhash->{$volid}->{is_unused} //= 0;
4855         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4856
4857         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4858     };
4859
4860     my $include_opts = {
4861         extra_keys => ['vmstate'],
4862         include_unused => 1,
4863     };
4864
4865     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4866     foreach my $snapname (keys %{$conf->{snapshots}}) {
4867         my $snap = $conf->{snapshots}->{$snapname};
4868         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4869     }
4870
4871     foreach my $volid (keys %$volhash) {
4872         &$func($volid, $volhash->{$volid}, @param);
4873     }
4874 }
4875
4876 my $fast_plug_option = {
4877     'lock' => 1,
4878     'name' => 1,
4879     'onboot' => 1,
4880     'shares' => 1,
4881     'startup' => 1,
4882     'description' => 1,
4883     'protection' => 1,
4884     'vmstatestorage' => 1,
4885     'hookscript' => 1,
4886     'tags' => 1,
4887 };
4888
4889 for my $opt (keys %$confdesc_cloudinit) {
4890     $fast_plug_option->{$opt} = 1;
4891 };
4892
4893 # hotplug changes in [PENDING]
4894 # $selection hash can be used to only apply specified options, for
4895 # example: { cores => 1 } (only apply changed 'cores')
4896 # $errors ref is used to return error messages
4897 sub vmconfig_hotplug_pending {
4898     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4899
4900     my $defaults = load_defaults();
4901     my $arch = get_vm_arch($conf);
4902     my $machine_type = get_vm_machine($conf, undef, $arch);
4903
4904     # commit values which do not have any impact on running VM first
4905     # Note: those option cannot raise errors, we we do not care about
4906     # $selection and always apply them.
4907
4908     my $add_error = sub {
4909         my ($opt, $msg) = @_;
4910         $errors->{$opt} = "hotplug problem - $msg";
4911     };
4912
4913     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4914
4915     my $cloudinit_record_changed = sub {
4916         my ($conf, $opt, $old, $new) = @_;
4917         return if !$cloudinit_pending_properties->{$opt};
4918
4919         my $ci = ($conf->{cloudinit} //= {});
4920
4921         my $recorded = $ci->{$opt};
4922         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4923
4924         if (defined($new)) {
4925             if (defined($old)) {
4926                 # an existing value is being modified
4927                 if (defined($recorded)) {
4928                     # the value was already not in sync
4929                     if ($new eq $recorded) {
4930                         # a value is being reverted to the cloud-init state:
4931                         delete $ci->{$opt};
4932                         delete $added{$opt};
4933                     } else {
4934                         # the value was changed multiple times, do nothing
4935                     }
4936                 } elsif ($added{$opt}) {
4937                     # the value had been marked as added and is being changed, do nothing
4938                 } else {
4939                     # the value is new, record it:
4940                     $ci->{$opt} = $old;
4941                 }
4942             } else {
4943                 # a new value is being added
4944                 if (defined($recorded)) {
4945                     # it was already not in sync
4946                     if ($new eq $recorded) {
4947                         # a value is being reverted to the cloud-init state:
4948                         delete $ci->{$opt};
4949                         delete $added{$opt};
4950                     } else {
4951                         # the value had temporarily been removed, do nothing
4952                     }
4953                 } elsif ($added{$opt}) {
4954                     # the value had been marked as added already, do nothing
4955                 } else {
4956                     # the value is new, add it
4957                     $added{$opt} = 1;
4958                 }
4959             }
4960         } elsif (!defined($old)) {
4961             # a non-existent value is being removed? ignore...
4962         } else {
4963             # a value is being deleted
4964             if (defined($recorded)) {
4965                 # a value was already recorded, just keep it
4966             } elsif ($added{$opt}) {
4967                 # the value was marked as added, remove it
4968                 delete $added{$opt};
4969             } else {
4970                 # a previously unrecorded value is being removed, record the old value:
4971                 $ci->{$opt} = $old;
4972             }
4973         }
4974
4975         my $added = join(',', sort keys %added);
4976         $ci->{added} = $added if length($added);
4977     };
4978
4979     my $changes = 0;
4980     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4981         if ($fast_plug_option->{$opt}) {
4982             my $new = delete $conf->{pending}->{$opt};
4983             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4984             $conf->{$opt} = $new;
4985             $changes = 1;
4986         }
4987     }
4988
4989     if ($changes) {
4990         PVE::QemuConfig->write_config($vmid, $conf);
4991     }
4992
4993     my $ostype = $conf->{ostype};
4994     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4995     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4996     my $usb_hotplug = $hotplug_features->{usb}
4997         && min_version($version, 7, 1)
4998         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4999
5000     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5001     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5002
5003     foreach my $opt (sort keys %$pending_delete_hash) {
5004         next if $selection && !$selection->{$opt};
5005         my $force = $pending_delete_hash->{$opt}->{force};
5006         eval {
5007             if ($opt eq 'hotplug') {
5008                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5009             } elsif ($opt eq 'tablet') {
5010                 die "skip\n" if !$hotplug_features->{usb};
5011                 if ($defaults->{tablet}) {
5012                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5013                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5014                         if $arch eq 'aarch64';
5015                 } else {
5016                     vm_deviceunplug($vmid, $conf, 'tablet');
5017                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5018                 }
5019             } elsif ($opt =~ m/^usb(\d+)$/) {
5020                 my $index = $1;
5021                 die "skip\n" if !$usb_hotplug;
5022                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5023                 vm_deviceunplug($vmid, $conf, $opt);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, undef);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable balloon device is not hotpluggable
5029                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5030                 # here we reset the ballooning value to memory
5031                 my $balloon = $conf->{memory} || $defaults->{memory};
5032                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5033             } elsif ($fast_plug_option->{$opt}) {
5034                 # do nothing
5035             } elsif ($opt =~ m/^net(\d+)$/) {
5036                 die "skip\n" if !$hotplug_features->{network};
5037                 vm_deviceunplug($vmid, $conf, $opt);
5038             } elsif (is_valid_drivename($opt)) {
5039                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5040                 vm_deviceunplug($vmid, $conf, $opt);
5041                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5042             } elsif ($opt =~ m/^memory$/) {
5043                 die "skip\n" if !$hotplug_features->{memory};
5044                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5045             } elsif ($opt eq 'cpuunits') {
5046                 $cgroup->change_cpu_shares(undef);
5047             } elsif ($opt eq 'cpulimit') {
5048                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5049             } else {
5050                 die "skip\n";
5051             }
5052         };
5053         if (my $err = $@) {
5054             &$add_error($opt, $err) if $err ne "skip\n";
5055         } else {
5056             my $old = delete $conf->{$opt};
5057             $cloudinit_record_changed->($conf, $opt, $old, undef);
5058             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5059         }
5060     }
5061
5062     my $cloudinit_opt;
5063     foreach my $opt (keys %{$conf->{pending}}) {
5064         next if $selection && !$selection->{$opt};
5065         my $value = $conf->{pending}->{$opt};
5066         eval {
5067             if ($opt eq 'hotplug') {
5068                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5069             } elsif ($opt eq 'tablet') {
5070                 die "skip\n" if !$hotplug_features->{usb};
5071                 if ($value == 1) {
5072                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5073                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5074                         if $arch eq 'aarch64';
5075                 } elsif ($value == 0) {
5076                     vm_deviceunplug($vmid, $conf, 'tablet');
5077                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5078                 }
5079             } elsif ($opt =~ m/^usb(\d+)$/) {
5080                 my $index = $1;
5081                 die "skip\n" if !$usb_hotplug;
5082                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5083                 my $id = $opt;
5084                 if ($d->{host} eq 'spice')  {
5085                     $id = "usbredirdev$index";
5086                 }
5087                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5088             } elsif ($opt eq 'vcpus') {
5089                 die "skip\n" if !$hotplug_features->{cpu};
5090                 qemu_cpu_hotplug($vmid, $conf, $value);
5091             } elsif ($opt eq 'balloon') {
5092                 # enable/disable balloning device is not hotpluggable
5093                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5094                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5095                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5096
5097                 # allow manual ballooning if shares is set to zero
5098                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5099                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5100                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5101                 }
5102             } elsif ($opt =~ m/^net(\d+)$/) {
5103                 # some changes can be done without hotplug
5104                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5105                                     $vmid, $opt, $value, $arch, $machine_type);
5106             } elsif (is_valid_drivename($opt)) {
5107                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5108                 # some changes can be done without hotplug
5109                 my $drive = parse_drive($opt, $value);
5110                 if (drive_is_cloudinit($drive)) {
5111                     $cloudinit_opt = [$opt, $drive];
5112                     # apply all the other changes first, then generate the cloudinit disk
5113                     die "skip\n";
5114                 }
5115                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5116                                      $vmid, $opt, $value, $arch, $machine_type);
5117             } elsif ($opt =~ m/^memory$/) { #dimms
5118                 die "skip\n" if !$hotplug_features->{memory};
5119                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5120             } elsif ($opt eq 'cpuunits') {
5121                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5122                 $cgroup->change_cpu_shares($new_cpuunits);
5123             } elsif ($opt eq 'cpulimit') {
5124                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5125                 $cgroup->change_cpu_quota($cpulimit, 100000);
5126             } elsif ($opt eq 'agent') {
5127                 vmconfig_update_agent($conf, $opt, $value);
5128             } else {
5129                 die "skip\n";  # skip non-hot-pluggable options
5130             }
5131         };
5132         if (my $err = $@) {
5133             &$add_error($opt, $err) if $err ne "skip\n";
5134         } else {
5135             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5136             $conf->{$opt} = $value;
5137             delete $conf->{pending}->{$opt};
5138         }
5139     }
5140
5141     if (defined($cloudinit_opt)) {
5142         my ($opt, $drive) = @$cloudinit_opt;
5143         my $value = $conf->{pending}->{$opt};
5144         eval {
5145             my $temp = {%$conf, $opt => $value};
5146             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5147             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5148                                  $vmid, $opt, $value, $arch, $machine_type);
5149         };
5150         if (my $err = $@) {
5151             &$add_error($opt, $err) if $err ne "skip\n";
5152         } else {
5153             $conf->{$opt} = $value;
5154             delete $conf->{pending}->{$opt};
5155         }
5156     }
5157
5158     # unplug xhci controller if no usb device is left
5159     if ($usb_hotplug) {
5160         my $has_usb = 0;
5161         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5162             next if !defined($conf->{"usb$i"});
5163             $has_usb = 1;
5164             last;
5165         }
5166         if (!$has_usb) {
5167             vm_deviceunplug($vmid, $conf, 'xhci');
5168         }
5169     }
5170
5171     PVE::QemuConfig->write_config($vmid, $conf);
5172
5173     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5174         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5175     }
5176 }
5177
5178 sub try_deallocate_drive {
5179     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5180
5181     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5182         my $volid = $drive->{file};
5183         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5184             my $sid = PVE::Storage::parse_volume_id($volid);
5185             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5186
5187             # check if the disk is really unused
5188             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5189                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5190             PVE::Storage::vdisk_free($storecfg, $volid);
5191             return 1;
5192         } else {
5193             # If vm is not owner of this disk remove from config
5194             return 1;
5195         }
5196     }
5197
5198     return;
5199 }
5200
5201 sub vmconfig_delete_or_detach_drive {
5202     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5203
5204     my $drive = parse_drive($opt, $conf->{$opt});
5205
5206     my $rpcenv = PVE::RPCEnvironment::get();
5207     my $authuser = $rpcenv->get_user();
5208
5209     if ($force) {
5210         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5211         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5212     } else {
5213         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5214     }
5215 }
5216
5217
5218
5219 sub vmconfig_apply_pending {
5220     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5221
5222     return if !scalar(keys %{$conf->{pending}});
5223
5224     my $add_apply_error = sub {
5225         my ($opt, $msg) = @_;
5226         my $err_msg = "unable to apply pending change $opt : $msg";
5227         $errors->{$opt} = $err_msg;
5228         warn $err_msg;
5229     };
5230
5231     # cold plug
5232
5233     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5234     foreach my $opt (sort keys %$pending_delete_hash) {
5235         my $force = $pending_delete_hash->{$opt}->{force};
5236         eval {
5237             if ($opt =~ m/^unused/) {
5238                 die "internal error";
5239             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5240                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5241             }
5242         };
5243         if (my $err = $@) {
5244             $add_apply_error->($opt, $err);
5245         } else {
5246             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5247             delete $conf->{$opt};
5248         }
5249     }
5250
5251     PVE::QemuConfig->cleanup_pending($conf);
5252
5253     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5254
5255     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5256         next if $opt eq 'delete'; # just to be sure
5257         eval {
5258             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5259                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5260             }
5261         };
5262         if (my $err = $@) {
5263             $add_apply_error->($opt, $err);
5264         } else {
5265
5266             if (is_valid_drivename($opt)) {
5267                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5268                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5269             }
5270
5271             $conf->{$opt} = delete $conf->{pending}->{$opt};
5272         }
5273     }
5274
5275     # write all changes at once to avoid unnecessary i/o
5276     PVE::QemuConfig->write_config($vmid, $conf);
5277     if ($generate_cloudinit) {
5278         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5279             # After successful generation and if there were changes to be applied, update the
5280             # config to drop the {cloudinit} entry.
5281             PVE::QemuConfig->write_config($vmid, $conf);
5282         }
5283     }
5284 }
5285
5286 sub vmconfig_update_net {
5287     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5288
5289     my $newnet = parse_net($value);
5290
5291     if ($conf->{$opt}) {
5292         my $oldnet = parse_net($conf->{$opt});
5293
5294         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5295             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5296             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5297             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5298
5299             # for non online change, we try to hot-unplug
5300             die "skip\n" if !$hotplug;
5301             vm_deviceunplug($vmid, $conf, $opt);
5302         } else {
5303
5304             die "internal error" if $opt !~ m/net(\d+)/;
5305             my $iface = "tap${vmid}i$1";
5306
5307             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5308                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5309                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5310                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5311                 PVE::Network::tap_unplug($iface);
5312
5313                 if ($have_sdn) {
5314                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5315                 } else {
5316                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5317                 }
5318             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5319                 # Rate can be applied on its own but any change above needs to
5320                 # include the rate in tap_plug since OVS resets everything.
5321                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5322             }
5323
5324             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5325                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5326             }
5327
5328             return 1;
5329         }
5330     }
5331
5332     if ($hotplug) {
5333         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5334     } else {
5335         die "skip\n";
5336     }
5337 }
5338
5339 sub vmconfig_update_agent {
5340     my ($conf, $opt, $value) = @_;
5341
5342     die "skip\n" if !$conf->{$opt};
5343
5344     my $hotplug_options = { fstrim_cloned_disks => 1 };
5345
5346     my $old_agent = parse_guest_agent($conf);
5347     my $agent = parse_guest_agent({$opt => $value});
5348
5349     for my $option (keys %$agent) { # added/changed options
5350         next if defined($hotplug_options->{$option});
5351         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5352     }
5353
5354     for my $option (keys %$old_agent) { # removed options
5355         next if defined($hotplug_options->{$option});
5356         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5357     }
5358
5359     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5360 }
5361
5362 sub vmconfig_update_disk {
5363     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5364
5365     my $drive = parse_drive($opt, $value);
5366
5367     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5368         my $media = $drive->{media} || 'disk';
5369         my $oldmedia = $old_drive->{media} || 'disk';
5370         die "unable to change media type\n" if $media ne $oldmedia;
5371
5372         if (!drive_is_cdrom($old_drive)) {
5373
5374             if ($drive->{file} ne $old_drive->{file}) {
5375
5376                 die "skip\n" if !$hotplug;
5377
5378                 # unplug and register as unused
5379                 vm_deviceunplug($vmid, $conf, $opt);
5380                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5381
5382             } else {
5383                 # update existing disk
5384
5385                 # skip non hotpluggable value
5386                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5387                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5388                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5389                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5390                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5391                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5392                     die "skip\n";
5393                 }
5394
5395                 # apply throttle
5396                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5397                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5398                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5399                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5400                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5401                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5402                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5403                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5404                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5405                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5406                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5407                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5408                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5409                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5410                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5411                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5412                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5413                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5414
5415                     qemu_block_set_io_throttle(
5416                         $vmid,"drive-$opt",
5417                         ($drive->{mbps} || 0)*1024*1024,
5418                         ($drive->{mbps_rd} || 0)*1024*1024,
5419                         ($drive->{mbps_wr} || 0)*1024*1024,
5420                          $drive->{iops} || 0,
5421                          $drive->{iops_rd} || 0,
5422                          $drive->{iops_wr} || 0,
5423                         ($drive->{mbps_max} || 0)*1024*1024,
5424                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5425                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5426                          $drive->{iops_max} || 0,
5427                          $drive->{iops_rd_max} || 0,
5428                          $drive->{iops_wr_max} || 0,
5429                          $drive->{bps_max_length} || 1,
5430                          $drive->{bps_rd_max_length} || 1,
5431                          $drive->{bps_wr_max_length} || 1,
5432                          $drive->{iops_max_length} || 1,
5433                          $drive->{iops_rd_max_length} || 1,
5434                          $drive->{iops_wr_max_length} || 1,
5435                     );
5436
5437                 }
5438
5439                 return 1;
5440             }
5441
5442         } else { # cdrom
5443
5444             if ($drive->{file} eq 'none') {
5445                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5446                 if (drive_is_cloudinit($old_drive)) {
5447                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5448                 }
5449             } else {
5450                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5451
5452                 # force eject if locked
5453                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5454
5455                 if ($path) {
5456                     mon_cmd($vmid, "blockdev-change-medium",
5457                         id => "$opt", filename => "$path");
5458                 }
5459             }
5460
5461             return 1;
5462         }
5463     }
5464
5465     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5466     # hotplug new disks
5467     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5468     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5469 }
5470
5471 sub vmconfig_update_cloudinit_drive {
5472     my ($storecfg, $conf, $vmid) = @_;
5473
5474     my $cloudinit_ds = undef;
5475     my $cloudinit_drive = undef;
5476
5477     PVE::QemuConfig->foreach_volume($conf, sub {
5478         my ($ds, $drive) = @_;
5479         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5480             $cloudinit_ds = $ds;
5481             $cloudinit_drive = $drive;
5482         }
5483     });
5484
5485     return if !$cloudinit_drive;
5486
5487     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5488         PVE::QemuConfig->write_config($vmid, $conf);
5489     }
5490
5491     my $running = PVE::QemuServer::check_running($vmid);
5492
5493     if ($running) {
5494         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5495         if ($path) {
5496             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5497             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5498         }
5499     }
5500 }
5501
5502 # called in locked context by incoming migration
5503 sub vm_migrate_get_nbd_disks {
5504     my ($storecfg, $conf, $replicated_volumes) = @_;
5505
5506     my $local_volumes = {};
5507     PVE::QemuConfig->foreach_volume($conf, sub {
5508         my ($ds, $drive) = @_;
5509
5510         return if drive_is_cdrom($drive);
5511         return if $ds eq 'tpmstate0';
5512
5513         my $volid = $drive->{file};
5514
5515         return if !$volid;
5516
5517         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5518
5519         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5520         return if $scfg->{shared};
5521
5522         # replicated disks re-use existing state via bitmap
5523         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5524         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5525     });
5526     return $local_volumes;
5527 }
5528
5529 # called in locked context by incoming migration
5530 sub vm_migrate_alloc_nbd_disks {
5531     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5532
5533     my $nbd = {};
5534     foreach my $opt (sort keys %$source_volumes) {
5535         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5536
5537         if ($use_existing) {
5538             $nbd->{$opt}->{drivestr} = print_drive($drive);
5539             $nbd->{$opt}->{volid} = $volid;
5540             $nbd->{$opt}->{replicated} = 1;
5541             next;
5542         }
5543
5544         # storage mapping + volname = regular migration
5545         # storage mapping + format = remote migration
5546         # order of precedence, filtered by whether storage supports it:
5547         # 1. explicit requested format
5548         # 2. format of current volume
5549         # 3. default format of storage
5550         if (!$storagemap->{identity}) {
5551             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5552             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5553             if (!$format || !grep { $format eq $_ } @$validFormats) {
5554                 if ($volname) {
5555                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5556                     my $fileFormat = qemu_img_format($scfg, $volname);
5557                     $format = $fileFormat
5558                         if grep { $fileFormat eq $_ } @$validFormats;
5559                 }
5560                 $format //= $defFormat;
5561             }
5562         } else {
5563             # can't happen for remote migration, so $volname is always defined
5564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5565             $format = qemu_img_format($scfg, $volname);
5566         }
5567
5568         my $size = $drive->{size} / 1024;
5569         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5570         my $newdrive = $drive;
5571         $newdrive->{format} = $format;
5572         $newdrive->{file} = $newvolid;
5573         my $drivestr = print_drive($newdrive);
5574         $nbd->{$opt}->{drivestr} = $drivestr;
5575         $nbd->{$opt}->{volid} = $newvolid;
5576     }
5577
5578     return $nbd;
5579 }
5580
5581 # see vm_start_nolock for parameters, additionally:
5582 # migrate_opts:
5583 #   storagemap = parsed storage map for allocating NBD disks
5584 sub vm_start {
5585     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5586
5587     return PVE::QemuConfig->lock_config($vmid, sub {
5588         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5589
5590         die "you can't start a vm if it's a template\n"
5591             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5592
5593         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5594         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5595
5596         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5597
5598         if ($has_backup_lock && $running) {
5599             # a backup is currently running, attempt to start the guest in the
5600             # existing QEMU instance
5601             return vm_resume($vmid);
5602         }
5603
5604         PVE::QemuConfig->check_lock($conf)
5605             if !($params->{skiplock} || $has_suspended_lock);
5606
5607         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5608
5609         die "VM $vmid already running\n" if $running;
5610
5611         if (my $storagemap = $migrate_opts->{storagemap}) {
5612             my $replicated = $migrate_opts->{replicated_volumes};
5613             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5614             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5615
5616             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5617                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5618             }
5619         }
5620
5621         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5622     });
5623 }
5624
5625
5626 # params:
5627 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5628 #   skiplock => 0/1, skip checking for config lock
5629 #   skiptemplate => 0/1, skip checking whether VM is template
5630 #   forcemachine => to force Qemu machine (rollback/migration)
5631 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5632 #   timeout => in seconds
5633 #   paused => start VM in paused state (backup)
5634 #   resume => resume from hibernation
5635 #   pbs-backing => {
5636 #      sata0 => {
5637 #         repository
5638 #         snapshot
5639 #         keyfile
5640 #         archive
5641 #      },
5642 #      virtio2 => ...
5643 #   }
5644 # migrate_opts:
5645 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5646 #   migratedfrom => source node
5647 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5648 #   network => CIDR of migration network
5649 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5650 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5651 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5652 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5653 #       contained in config
5654 sub vm_start_nolock {
5655     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5656
5657     my $statefile = $params->{statefile};
5658     my $resume = $params->{resume};
5659
5660     my $migratedfrom = $migrate_opts->{migratedfrom};
5661     my $migration_type = $migrate_opts->{type};
5662
5663     my $res = {};
5664
5665     # clean up leftover reboot request files
5666     eval { clear_reboot_request($vmid); };
5667     warn $@ if $@;
5668
5669     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5670         vmconfig_apply_pending($vmid, $conf, $storecfg);
5671         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5672     }
5673
5674     # don't regenerate the ISO if the VM is started as part of a live migration
5675     # this way we can reuse the old ISO with the correct config
5676     if (!$migratedfrom) {
5677         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5678             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5679             # $conf->{cloudinit}, so we could just not do this?
5680             # But we do it above, so for now let's be consistent.
5681             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5682         }
5683     }
5684
5685     # override offline migrated volumes, conf is out of date still
5686     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5687         for my $key (sort keys $offline_volumes->%*) {
5688             my $parsed = parse_drive($key, $conf->{$key});
5689             $parsed->{file} = $offline_volumes->{$key};
5690             $conf->{$key} = print_drive($parsed);
5691         }
5692     }
5693
5694     my $defaults = load_defaults();
5695
5696     # set environment variable useful inside network script
5697     # for remote migration the config is available on the target node!
5698     if (!$migrate_opts->{remote_node}) {
5699         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5700     }
5701
5702     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5703
5704     my $forcemachine = $params->{forcemachine};
5705     my $forcecpu = $params->{forcecpu};
5706     if ($resume) {
5707         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5708         $forcemachine = $conf->{runningmachine};
5709         $forcecpu = $conf->{runningcpu};
5710         print "Resuming suspended VM\n";
5711     }
5712
5713     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5714         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5715
5716     my $migration_ip;
5717     my $get_migration_ip = sub {
5718         my ($nodename) = @_;
5719
5720         return $migration_ip if defined($migration_ip);
5721
5722         my $cidr = $migrate_opts->{network};
5723
5724         if (!defined($cidr)) {
5725             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5726             $cidr = $dc_conf->{migration}->{network};
5727         }
5728
5729         if (defined($cidr)) {
5730             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5731
5732             die "could not get IP: no address configured on local " .
5733                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5734
5735             die "could not get IP: multiple addresses configured on local " .
5736                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5737
5738             $migration_ip = @$ips[0];
5739         }
5740
5741         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5742             if !defined($migration_ip);
5743
5744         return $migration_ip;
5745     };
5746
5747     if ($statefile) {
5748         if ($statefile eq 'tcp') {
5749             my $migrate = $res->{migrate} = { proto => 'tcp' };
5750             $migrate->{addr} = "localhost";
5751             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5752             my $nodename = nodename();
5753
5754             if (!defined($migration_type)) {
5755                 if (defined($datacenterconf->{migration}->{type})) {
5756                     $migration_type = $datacenterconf->{migration}->{type};
5757                 } else {
5758                     $migration_type = 'secure';
5759                 }
5760             }
5761
5762             if ($migration_type eq 'insecure') {
5763                 $migrate->{addr} = $get_migration_ip->($nodename);
5764                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5765             }
5766
5767             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5768             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5769             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5770             push @$cmd, '-incoming', $migrate->{uri};
5771             push @$cmd, '-S';
5772
5773         } elsif ($statefile eq 'unix') {
5774             # should be default for secure migrations as a ssh TCP forward
5775             # tunnel is not deterministic reliable ready and fails regurarly
5776             # to set up in time, so use UNIX socket forwards
5777             my $migrate = $res->{migrate} = { proto => 'unix' };
5778             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5779             unlink $migrate->{addr};
5780
5781             $migrate->{uri} = "unix:$migrate->{addr}";
5782             push @$cmd, '-incoming', $migrate->{uri};
5783             push @$cmd, '-S';
5784
5785         } elsif (-e $statefile) {
5786             push @$cmd, '-loadstate', $statefile;
5787         } else {
5788             my $statepath = PVE::Storage::path($storecfg, $statefile);
5789             push @$vollist, $statefile;
5790             push @$cmd, '-loadstate', $statepath;
5791         }
5792     } elsif ($params->{paused}) {
5793         push @$cmd, '-S';
5794     }
5795
5796     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5797
5798     my $pci_devices = {}; # host pci devices
5799     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5800         my $dev = $conf->{"hostpci$i"} or next;
5801         $pci_devices->{$i} = parse_hostpci($dev);
5802     }
5803
5804     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5805     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5806
5807     # map to a flat list of pci ids
5808     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5809
5810     # reserve all PCI IDs before actually doing anything with them
5811     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5812
5813     eval {
5814         my $uuid;
5815         for my $id (sort keys %$pci_devices) {
5816             my $d = $pci_devices->{$id};
5817             for my $dev ($d->{pciid}->@*) {
5818                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5819
5820                 # nvidia grid needs the uuid of the mdev as qemu parameter
5821                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5822                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5823                 }
5824             }
5825         }
5826         push @$cmd, '-uuid', $uuid if defined($uuid);
5827     };
5828     if (my $err = $@) {
5829         eval { cleanup_pci_devices($vmid, $conf) };
5830         warn $@ if $@;
5831         die $err;
5832     }
5833
5834     PVE::Storage::activate_volumes($storecfg, $vollist);
5835
5836     eval {
5837         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5838     };
5839     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5840     # timeout should be more than enough here...
5841     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5842
5843     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5844
5845     my %run_params = (
5846         timeout => $statefile ? undef : $start_timeout,
5847         umask => 0077,
5848         noerr => 1,
5849     );
5850
5851     # when migrating, prefix QEMU output so other side can pick up any
5852     # errors that might occur and show the user
5853     if ($migratedfrom) {
5854         $run_params{quiet} = 1;
5855         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5856     }
5857
5858     my %systemd_properties = (
5859         Slice => 'qemu.slice',
5860         KillMode => 'process',
5861         SendSIGKILL => 0,
5862         TimeoutStopUSec => ULONG_MAX, # infinity
5863     );
5864
5865     if (PVE::CGroup::cgroup_mode() == 2) {
5866         $systemd_properties{CPUWeight} = $cpuunits;
5867     } else {
5868         $systemd_properties{CPUShares} = $cpuunits;
5869     }
5870
5871     if (my $cpulimit = $conf->{cpulimit}) {
5872         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5873     }
5874     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5875
5876     my $run_qemu = sub {
5877         PVE::Tools::run_fork sub {
5878             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5879
5880             my $tpmpid;
5881             if (my $tpm = $conf->{tpmstate0}) {
5882                 # start the TPM emulator so QEMU can connect on start
5883                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5884             }
5885
5886             my $exitcode = run_command($cmd, %run_params);
5887             if ($exitcode) {
5888                 if ($tpmpid) {
5889                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5890                     kill 'TERM', $tpmpid;
5891                 }
5892                 die "QEMU exited with code $exitcode\n";
5893             }
5894         };
5895     };
5896
5897     if ($conf->{hugepages}) {
5898
5899         my $code = sub {
5900             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5901             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5902
5903             PVE::QemuServer::Memory::hugepages_mount();
5904             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5905
5906             eval { $run_qemu->() };
5907             if (my $err = $@) {
5908                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5909                     if !$conf->{keephugepages};
5910                 die $err;
5911             }
5912
5913             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5914                 if !$conf->{keephugepages};
5915         };
5916         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5917
5918     } else {
5919         eval { $run_qemu->() };
5920     }
5921
5922     if (my $err = $@) {
5923         # deactivate volumes if start fails
5924         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5925         warn $@ if $@;
5926         eval { cleanup_pci_devices($vmid, $conf) };
5927         warn $@ if $@;
5928
5929         die "start failed: $err";
5930     }
5931
5932     # re-reserve all PCI IDs now that we can know the actual VM PID
5933     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5934     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5935     warn $@ if $@;
5936
5937     if (defined($res->{migrate})) {
5938         print "migration listens on $res->{migrate}->{uri}\n";
5939     } elsif ($statefile) {
5940         eval { mon_cmd($vmid, "cont"); };
5941         warn $@ if $@;
5942     }
5943
5944     #start nbd server for storage migration
5945     if (my $nbd = $migrate_opts->{nbd}) {
5946         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5947
5948         my $migrate_storage_uri;
5949         # nbd_protocol_version > 0 for unix socket support
5950         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5951             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5952             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5953             $migrate_storage_uri = "nbd:unix:$socket_path";
5954             $res->{migrate}->{unix_sockets} = [$socket_path];
5955         } else {
5956             my $nodename = nodename();
5957             my $localip = $get_migration_ip->($nodename);
5958             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5959             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5960
5961             mon_cmd($vmid, "nbd-server-start", addr => {
5962                 type => 'inet',
5963                 data => {
5964                     host => "${localip}",
5965                     port => "${storage_migrate_port}",
5966                 },
5967             });
5968             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5969             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5970         }
5971
5972         foreach my $opt (sort keys %$nbd) {
5973             my $drivestr = $nbd->{$opt}->{drivestr};
5974             my $volid = $nbd->{$opt}->{volid};
5975             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5976             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5977             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5978             print "re-using replicated volume: $opt - $volid\n"
5979                 if $nbd->{$opt}->{replicated};
5980
5981             $res->{drives}->{$opt} = $nbd->{$opt};
5982             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5983         }
5984     }
5985
5986     if ($migratedfrom) {
5987         eval {
5988             set_migration_caps($vmid);
5989         };
5990         warn $@ if $@;
5991
5992         if ($spice_port) {
5993             print "spice listens on port $spice_port\n";
5994             $res->{spice_port} = $spice_port;
5995             if ($migrate_opts->{spice_ticket}) {
5996                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5997                     $migrate_opts->{spice_ticket});
5998                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5999             }
6000         }
6001
6002     } else {
6003         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6004             if !$statefile && $conf->{balloon};
6005
6006         foreach my $opt (keys %$conf) {
6007             next if $opt !~  m/^net\d+$/;
6008             my $nicconf = parse_net($conf->{$opt});
6009             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6010         }
6011         add_nets_bridge_fdb($conf, $vmid);
6012     }
6013
6014     mon_cmd($vmid, 'qom-set',
6015                 path => "machine/peripheral/balloon0",
6016                 property => "guest-stats-polling-interval",
6017                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6018
6019     if ($resume) {
6020         print "Resumed VM, removing state\n";
6021         if (my $vmstate = $conf->{vmstate}) {
6022             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6023             PVE::Storage::vdisk_free($storecfg, $vmstate);
6024         }
6025         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6026         PVE::QemuConfig->write_config($vmid, $conf);
6027     }
6028
6029     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6030
6031     return $res;
6032 }
6033
6034 sub vm_commandline {
6035     my ($storecfg, $vmid, $snapname) = @_;
6036
6037     my $conf = PVE::QemuConfig->load_config($vmid);
6038
6039     my ($forcemachine, $forcecpu);
6040     if ($snapname) {
6041         my $snapshot = $conf->{snapshots}->{$snapname};
6042         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6043
6044         # check for machine or CPU overrides in snapshot
6045         $forcemachine = $snapshot->{runningmachine};
6046         $forcecpu = $snapshot->{runningcpu};
6047
6048         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6049
6050         $conf = $snapshot;
6051     }
6052
6053     my $defaults = load_defaults();
6054
6055     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6056
6057     return PVE::Tools::cmd2string($cmd);
6058 }
6059
6060 sub vm_reset {
6061     my ($vmid, $skiplock) = @_;
6062
6063     PVE::QemuConfig->lock_config($vmid, sub {
6064
6065         my $conf = PVE::QemuConfig->load_config($vmid);
6066
6067         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6068
6069         mon_cmd($vmid, "system_reset");
6070     });
6071 }
6072
6073 sub get_vm_volumes {
6074     my ($conf) = @_;
6075
6076     my $vollist = [];
6077     foreach_volid($conf, sub {
6078         my ($volid, $attr) = @_;
6079
6080         return if $volid =~ m|^/|;
6081
6082         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6083         return if !$sid;
6084
6085         push @$vollist, $volid;
6086     });
6087
6088     return $vollist;
6089 }
6090
6091 sub cleanup_pci_devices {
6092     my ($vmid, $conf) = @_;
6093
6094     foreach my $key (keys %$conf) {
6095         next if $key !~ m/^hostpci(\d+)$/;
6096         my $hostpciindex = $1;
6097         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6098         my $d = parse_hostpci($conf->{$key});
6099         if ($d->{mdev}) {
6100             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6101             # don't want to break ABI just for this two liner
6102             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6103             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6104         }
6105     }
6106     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6107 }
6108
6109 sub vm_stop_cleanup {
6110     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6111
6112     eval {
6113
6114         if (!$keepActive) {
6115             my $vollist = get_vm_volumes($conf);
6116             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6117
6118             if (my $tpmdrive = $conf->{tpmstate0}) {
6119                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6120                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6121                 if ($storeid) {
6122                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6123                 }
6124             }
6125         }
6126
6127         foreach my $ext (qw(mon qmp pid vnc qga)) {
6128             unlink "/var/run/qemu-server/${vmid}.$ext";
6129         }
6130
6131         if ($conf->{ivshmem}) {
6132             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6133             # just delete it for now, VMs which have this already open do not
6134             # are affected, but new VMs will get a separated one. If this
6135             # becomes an issue we either add some sort of ref-counting or just
6136             # add a "don't delete on stop" flag to the ivshmem format.
6137             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6138         }
6139
6140         cleanup_pci_devices($vmid, $conf);
6141
6142         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6143     };
6144     warn $@ if $@; # avoid errors - just warn
6145 }
6146
6147 # call only in locked context
6148 sub _do_vm_stop {
6149     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6150
6151     my $pid = check_running($vmid, $nocheck);
6152     return if !$pid;
6153
6154     my $conf;
6155     if (!$nocheck) {
6156         $conf = PVE::QemuConfig->load_config($vmid);
6157         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6158         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6159             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6160             $timeout = $opts->{down} if $opts->{down};
6161         }
6162         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6163     }
6164
6165     eval {
6166         if ($shutdown) {
6167             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6168                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6169             } else {
6170                 mon_cmd($vmid, "system_powerdown");
6171             }
6172         } else {
6173             mon_cmd($vmid, "quit");
6174         }
6175     };
6176     my $err = $@;
6177
6178     if (!$err) {
6179         $timeout = 60 if !defined($timeout);
6180
6181         my $count = 0;
6182         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6183             $count++;
6184             sleep 1;
6185         }
6186
6187         if ($count >= $timeout) {
6188             if ($force) {
6189                 warn "VM still running - terminating now with SIGTERM\n";
6190                 kill 15, $pid;
6191             } else {
6192                 die "VM quit/powerdown failed - got timeout\n";
6193             }
6194         } else {
6195             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6196             return;
6197         }
6198     } else {
6199        if (!check_running($vmid, $nocheck)) {
6200             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6201             return;
6202        }
6203        if ($force) {
6204             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6205             kill 15, $pid;
6206         } else {
6207             die "VM quit/powerdown failed\n";
6208         }
6209     }
6210
6211     # wait again
6212     $timeout = 10;
6213
6214     my $count = 0;
6215     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6216         $count++;
6217         sleep 1;
6218     }
6219
6220     if ($count >= $timeout) {
6221         warn "VM still running - terminating now with SIGKILL\n";
6222         kill 9, $pid;
6223         sleep 1;
6224     }
6225
6226     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6227 }
6228
6229 # Note: use $nocheck to skip tests if VM configuration file exists.
6230 # We need that when migration VMs to other nodes (files already moved)
6231 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6232 sub vm_stop {
6233     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6234
6235     $force = 1 if !defined($force) && !$shutdown;
6236
6237     if ($migratedfrom){
6238         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6239         kill 15, $pid if $pid;
6240         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6241         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6242         return;
6243     }
6244
6245     PVE::QemuConfig->lock_config($vmid, sub {
6246         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6247    });
6248 }
6249
6250 sub vm_reboot {
6251     my ($vmid, $timeout) = @_;
6252
6253     PVE::QemuConfig->lock_config($vmid, sub {
6254         eval {
6255
6256             # only reboot if running, as qmeventd starts it again on a stop event
6257             return if !check_running($vmid);
6258
6259             create_reboot_request($vmid);
6260
6261             my $storecfg = PVE::Storage::config();
6262             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6263
6264         };
6265         if (my $err = $@) {
6266             # avoid that the next normal shutdown will be confused for a reboot
6267             clear_reboot_request($vmid);
6268             die $err;
6269         }
6270    });
6271 }
6272
6273 # note: if using the statestorage parameter, the caller has to check privileges
6274 sub vm_suspend {
6275     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6276
6277     my $conf;
6278     my $path;
6279     my $storecfg;
6280     my $vmstate;
6281
6282     PVE::QemuConfig->lock_config($vmid, sub {
6283
6284         $conf = PVE::QemuConfig->load_config($vmid);
6285
6286         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6287         PVE::QemuConfig->check_lock($conf)
6288             if !($skiplock || $is_backing_up);
6289
6290         die "cannot suspend to disk during backup\n"
6291             if $is_backing_up && $includestate;
6292
6293         if ($includestate) {
6294             $conf->{lock} = 'suspending';
6295             my $date = strftime("%Y-%m-%d", localtime(time()));
6296             $storecfg = PVE::Storage::config();
6297             if (!$statestorage) {
6298                 $statestorage = find_vmstate_storage($conf, $storecfg);
6299                 # check permissions for the storage
6300                 my $rpcenv = PVE::RPCEnvironment::get();
6301                 if ($rpcenv->{type} ne 'cli') {
6302                     my $authuser = $rpcenv->get_user();
6303                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6304                 }
6305             }
6306
6307
6308             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6309                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6310             $path = PVE::Storage::path($storecfg, $vmstate);
6311             PVE::QemuConfig->write_config($vmid, $conf);
6312         } else {
6313             mon_cmd($vmid, "stop");
6314         }
6315     });
6316
6317     if ($includestate) {
6318         # save vm state
6319         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6320
6321         eval {
6322             set_migration_caps($vmid, 1);
6323             mon_cmd($vmid, "savevm-start", statefile => $path);
6324             for(;;) {
6325                 my $state = mon_cmd($vmid, "query-savevm");
6326                 if (!$state->{status}) {
6327                     die "savevm not active\n";
6328                 } elsif ($state->{status} eq 'active') {
6329                     sleep(1);
6330                     next;
6331                 } elsif ($state->{status} eq 'completed') {
6332                     print "State saved, quitting\n";
6333                     last;
6334                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6335                     die "query-savevm failed with error '$state->{error}'\n"
6336                 } else {
6337                     die "query-savevm returned status '$state->{status}'\n";
6338                 }
6339             }
6340         };
6341         my $err = $@;
6342
6343         PVE::QemuConfig->lock_config($vmid, sub {
6344             $conf = PVE::QemuConfig->load_config($vmid);
6345             if ($err) {
6346                 # cleanup, but leave suspending lock, to indicate something went wrong
6347                 eval {
6348                     mon_cmd($vmid, "savevm-end");
6349                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6350                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6351                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6352                     PVE::QemuConfig->write_config($vmid, $conf);
6353                 };
6354                 warn $@ if $@;
6355                 die $err;
6356             }
6357
6358             die "lock changed unexpectedly\n"
6359                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6360
6361             mon_cmd($vmid, "quit");
6362             $conf->{lock} = 'suspended';
6363             PVE::QemuConfig->write_config($vmid, $conf);
6364         });
6365     }
6366 }
6367
6368 # $nocheck is set when called as part of a migration - in this context the
6369 # location of the config file (source or target node) is not deterministic,
6370 # since migration cannot wait for pmxcfs to process the rename
6371 sub vm_resume {
6372     my ($vmid, $skiplock, $nocheck) = @_;
6373
6374     PVE::QemuConfig->lock_config($vmid, sub {
6375         my $res = mon_cmd($vmid, 'query-status');
6376         my $resume_cmd = 'cont';
6377         my $reset = 0;
6378         my $conf;
6379         if ($nocheck) {
6380             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6381             if ($@) {
6382                 my $vmlist = PVE::Cluster::get_vmlist();
6383                 if (exists($vmlist->{ids}->{$vmid})) {
6384                     my $node = $vmlist->{ids}->{$vmid}->{node};
6385                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6386                 }
6387                 if (!$conf) {
6388                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6389                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6390                 }
6391             }
6392         } else {
6393             $conf = PVE::QemuConfig->load_config($vmid);
6394         }
6395
6396         if ($res->{status}) {
6397             return if $res->{status} eq 'running'; # job done, go home
6398             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6399             $reset = 1 if $res->{status} eq 'shutdown';
6400         }
6401
6402         if (!$nocheck) {
6403             PVE::QemuConfig->check_lock($conf)
6404                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6405         }
6406
6407         if ($reset) {
6408             # required if a VM shuts down during a backup and we get a resume
6409             # request before the backup finishes for example
6410             mon_cmd($vmid, "system_reset");
6411         }
6412
6413         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6414
6415         mon_cmd($vmid, $resume_cmd);
6416     });
6417 }
6418
6419 sub vm_sendkey {
6420     my ($vmid, $skiplock, $key) = @_;
6421
6422     PVE::QemuConfig->lock_config($vmid, sub {
6423
6424         my $conf = PVE::QemuConfig->load_config($vmid);
6425
6426         # there is no qmp command, so we use the human monitor command
6427         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6428         die $res if $res ne '';
6429     });
6430 }
6431
6432 # vzdump restore implementaion
6433
6434 sub tar_archive_read_firstfile {
6435     my $archive = shift;
6436
6437     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6438
6439     # try to detect archive type first
6440     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6441         die "unable to open file '$archive'\n";
6442     my $firstfile = <$fh>;
6443     kill 15, $pid;
6444     close $fh;
6445
6446     die "ERROR: archive contaions no data\n" if !$firstfile;
6447     chomp $firstfile;
6448
6449     return $firstfile;
6450 }
6451
6452 sub tar_restore_cleanup {
6453     my ($storecfg, $statfile) = @_;
6454
6455     print STDERR "starting cleanup\n";
6456
6457     if (my $fd = IO::File->new($statfile, "r")) {
6458         while (defined(my $line = <$fd>)) {
6459             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6460                 my $volid = $2;
6461                 eval {
6462                     if ($volid =~ m|^/|) {
6463                         unlink $volid || die 'unlink failed\n';
6464                     } else {
6465                         PVE::Storage::vdisk_free($storecfg, $volid);
6466                     }
6467                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6468                 };
6469                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6470             } else {
6471                 print STDERR "unable to parse line in statfile - $line";
6472             }
6473         }
6474         $fd->close();
6475     }
6476 }
6477
6478 sub restore_file_archive {
6479     my ($archive, $vmid, $user, $opts) = @_;
6480
6481     return restore_vma_archive($archive, $vmid, $user, $opts)
6482         if $archive eq '-';
6483
6484     my $info = PVE::Storage::archive_info($archive);
6485     my $format = $opts->{format} // $info->{format};
6486     my $comp = $info->{compression};
6487
6488     # try to detect archive format
6489     if ($format eq 'tar') {
6490         return restore_tar_archive($archive, $vmid, $user, $opts);
6491     } else {
6492         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6493     }
6494 }
6495
6496 # hepler to remove disks that will not be used after restore
6497 my $restore_cleanup_oldconf = sub {
6498     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6499
6500     my $kept_disks = {};
6501
6502     PVE::QemuConfig->foreach_volume($oldconf, sub {
6503         my ($ds, $drive) = @_;
6504
6505         return if drive_is_cdrom($drive, 1);
6506
6507         my $volid = $drive->{file};
6508         return if !$volid || $volid =~ m|^/|;
6509
6510         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6511         return if !$path || !$owner || ($owner != $vmid);
6512
6513         # Note: only delete disk we want to restore
6514         # other volumes will become unused
6515         if ($virtdev_hash->{$ds}) {
6516             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6517             if (my $err = $@) {
6518                 warn $err;
6519             }
6520         } else {
6521             $kept_disks->{$volid} = 1;
6522         }
6523     });
6524
6525     # after the restore we have no snapshots anymore
6526     for my $snapname (keys $oldconf->{snapshots}->%*) {
6527         my $snap = $oldconf->{snapshots}->{$snapname};
6528         if ($snap->{vmstate}) {
6529             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6530             if (my $err = $@) {
6531                 warn $err;
6532             }
6533         }
6534
6535         for my $volid (keys $kept_disks->%*) {
6536             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6537             warn $@ if $@;
6538         }
6539     }
6540 };
6541
6542 # Helper to parse vzdump backup device hints
6543 #
6544 # $rpcenv: Environment, used to ckeck storage permissions
6545 # $user: User ID, to check storage permissions
6546 # $storecfg: Storage configuration
6547 # $fh: the file handle for reading the configuration
6548 # $devinfo: should contain device sizes for all backu-up'ed devices
6549 # $options: backup options (pool, default storage)
6550 #
6551 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6552 my $parse_backup_hints = sub {
6553     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6554
6555     my $check_storage = sub { # assert if an image can be allocate
6556         my ($storeid, $scfg) = @_;
6557         die "Content type 'images' is not available on storage '$storeid'\n"
6558             if !$scfg->{content}->{images};
6559         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6560             if $user ne 'root@pam';
6561     };
6562
6563     my $virtdev_hash = {};
6564     while (defined(my $line = <$fh>)) {
6565         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6566             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6567             die "archive does not contain data for drive '$virtdev'\n"
6568                 if !$devinfo->{$devname};
6569
6570             if (defined($options->{storage})) {
6571                 $storeid = $options->{storage} || 'local';
6572             } elsif (!$storeid) {
6573                 $storeid = 'local';
6574             }
6575             $format = 'raw' if !$format;
6576             $devinfo->{$devname}->{devname} = $devname;
6577             $devinfo->{$devname}->{virtdev} = $virtdev;
6578             $devinfo->{$devname}->{format} = $format;
6579             $devinfo->{$devname}->{storeid} = $storeid;
6580
6581             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6582             $check_storage->($storeid, $scfg); # permission and content type check
6583
6584             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6585         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6586             my $virtdev = $1;
6587             my $drive = parse_drive($virtdev, $2);
6588
6589             if (drive_is_cloudinit($drive)) {
6590                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6591                 $storeid = $options->{storage} if defined ($options->{storage});
6592                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6593                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6594
6595                 $check_storage->($storeid, $scfg); # permission and content type check
6596
6597                 $virtdev_hash->{$virtdev} = {
6598                     format => $format,
6599                     storeid => $storeid,
6600                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6601                     is_cloudinit => 1,
6602                 };
6603             }
6604         }
6605     }
6606
6607     return $virtdev_hash;
6608 };
6609
6610 # Helper to allocate and activate all volumes required for a restore
6611 #
6612 # $storecfg: Storage configuration
6613 # $virtdev_hash: as returned by parse_backup_hints()
6614 #
6615 # Returns: { $virtdev => $volid }
6616 my $restore_allocate_devices = sub {
6617     my ($storecfg, $virtdev_hash, $vmid) = @_;
6618
6619     my $map = {};
6620     foreach my $virtdev (sort keys %$virtdev_hash) {
6621         my $d = $virtdev_hash->{$virtdev};
6622         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6623         my $storeid = $d->{storeid};
6624         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6625
6626         # test if requested format is supported
6627         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6628         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6629         $d->{format} = $defFormat if !$supported;
6630
6631         my $name;
6632         if ($d->{is_cloudinit}) {
6633             $name = "vm-$vmid-cloudinit";
6634             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6635             if ($scfg->{path}) {
6636                 $name .= ".$d->{format}";
6637             }
6638         }
6639
6640         my $volid = PVE::Storage::vdisk_alloc(
6641             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6642
6643         print STDERR "new volume ID is '$volid'\n";
6644         $d->{volid} = $volid;
6645
6646         PVE::Storage::activate_volumes($storecfg, [$volid]);
6647
6648         $map->{$virtdev} = $volid;
6649     }
6650
6651     return $map;
6652 };
6653
6654 sub restore_update_config_line {
6655     my ($cookie, $map, $line, $unique) = @_;
6656
6657     return '' if $line =~ m/^\#qmdump\#/;
6658     return '' if $line =~ m/^\#vzdump\#/;
6659     return '' if $line =~ m/^lock:/;
6660     return '' if $line =~ m/^unused\d+:/;
6661     return '' if $line =~ m/^parent:/;
6662
6663     my $res = '';
6664
6665     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6666     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6667         # try to convert old 1.X settings
6668         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6669         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6670             my ($model, $macaddr) = split(/\=/, $devconfig);
6671             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6672             my $net = {
6673                 model => $model,
6674                 bridge => "vmbr$ind",
6675                 macaddr => $macaddr,
6676             };
6677             my $netstr = print_net($net);
6678
6679             $res .= "net$cookie->{netcount}: $netstr\n";
6680             $cookie->{netcount}++;
6681         }
6682     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6683         my ($id, $netstr) = ($1, $2);
6684         my $net = parse_net($netstr);
6685         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6686         $netstr = print_net($net);
6687         $res .= "$id: $netstr\n";
6688     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6689         my $virtdev = $1;
6690         my $value = $3;
6691         my $di = parse_drive($virtdev, $value);
6692         if (defined($di->{backup}) && !$di->{backup}) {
6693             $res .= "#$line";
6694         } elsif ($map->{$virtdev}) {
6695             delete $di->{format}; # format can change on restore
6696             $di->{file} = $map->{$virtdev};
6697             $value = print_drive($di);
6698             $res .= "$virtdev: $value\n";
6699         } else {
6700             $res .= $line;
6701         }
6702     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6703         my $vmgenid = $1;
6704         if ($vmgenid ne '0') {
6705             # always generate a new vmgenid if there was a valid one setup
6706             $vmgenid = generate_uuid();
6707         }
6708         $res .= "vmgenid: $vmgenid\n";
6709     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6710         my ($uuid, $uuid_str);
6711         UUID::generate($uuid);
6712         UUID::unparse($uuid, $uuid_str);
6713         my $smbios1 = parse_smbios1($2);
6714         $smbios1->{uuid} = $uuid_str;
6715         $res .= $1.print_smbios1($smbios1)."\n";
6716     } else {
6717         $res .= $line;
6718     }
6719
6720     return $res;
6721 }
6722
6723 my $restore_deactivate_volumes = sub {
6724     my ($storecfg, $virtdev_hash) = @_;
6725
6726     my $vollist = [];
6727     for my $dev (values $virtdev_hash->%*) {
6728         push $vollist->@*, $dev->{volid} if $dev->{volid};
6729     }
6730
6731     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6732     print STDERR $@ if $@;
6733 };
6734
6735 my $restore_destroy_volumes = sub {
6736     my ($storecfg, $virtdev_hash) = @_;
6737
6738     for my $dev (values $virtdev_hash->%*) {
6739         my $volid = $dev->{volid} or next;
6740         eval {
6741             PVE::Storage::vdisk_free($storecfg, $volid);
6742             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6743         };
6744         print STDERR "unable to cleanup '$volid' - $@" if $@;
6745     }
6746 };
6747
6748 my $restore_merge_config = sub {
6749     my ($filename, $backup_conf_raw, $override_conf) = @_;
6750
6751     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6752     for my $key (keys $override_conf->%*) {
6753         $backup_conf->{$key} = $override_conf->{$key};
6754     }
6755
6756     return $backup_conf;
6757 };
6758
6759 sub scan_volids {
6760     my ($cfg, $vmid) = @_;
6761
6762     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6763
6764     my $volid_hash = {};
6765     foreach my $storeid (keys %$info) {
6766         foreach my $item (@{$info->{$storeid}}) {
6767             next if !($item->{volid} && $item->{size});
6768             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6769             $volid_hash->{$item->{volid}} = $item;
6770         }
6771     }
6772
6773     return $volid_hash;
6774 }
6775
6776 sub update_disk_config {
6777     my ($vmid, $conf, $volid_hash) = @_;
6778
6779     my $changes;
6780     my $prefix = "VM $vmid";
6781
6782     # used and unused disks
6783     my $referenced = {};
6784
6785     # Note: it is allowed to define multiple storages with same path (alias), so
6786     # we need to check both 'volid' and real 'path' (two different volid can point
6787     # to the same path).
6788
6789     my $referencedpath = {};
6790
6791     # update size info
6792     PVE::QemuConfig->foreach_volume($conf, sub {
6793         my ($opt, $drive) = @_;
6794
6795         my $volid = $drive->{file};
6796         return if !$volid;
6797         my $volume = $volid_hash->{$volid};
6798
6799         # mark volid as "in-use" for next step
6800         $referenced->{$volid} = 1;
6801         if ($volume && (my $path = $volume->{path})) {
6802             $referencedpath->{$path} = 1;
6803         }
6804
6805         return if drive_is_cdrom($drive);
6806         return if !$volume;
6807
6808         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6809         if (defined($updated)) {
6810             $changes = 1;
6811             $conf->{$opt} = print_drive($updated);
6812             print "$prefix ($opt): $msg\n";
6813         }
6814     });
6815
6816     # remove 'unusedX' entry if volume is used
6817     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6818         my ($opt, $drive) = @_;
6819
6820         my $volid = $drive->{file};
6821         return if !$volid;
6822
6823         my $path;
6824         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6825         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6826             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6827             $changes = 1;
6828             delete $conf->{$opt};
6829         }
6830
6831         $referenced->{$volid} = 1;
6832         $referencedpath->{$path} = 1 if $path;
6833     });
6834
6835     foreach my $volid (sort keys %$volid_hash) {
6836         next if $volid =~ m/vm-$vmid-state-/;
6837         next if $referenced->{$volid};
6838         my $path = $volid_hash->{$volid}->{path};
6839         next if !$path; # just to be sure
6840         next if $referencedpath->{$path};
6841         $changes = 1;
6842         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6843         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6844         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6845     }
6846
6847     return $changes;
6848 }
6849
6850 sub rescan {
6851     my ($vmid, $nolock, $dryrun) = @_;
6852
6853     my $cfg = PVE::Storage::config();
6854
6855     print "rescan volumes...\n";
6856     my $volid_hash = scan_volids($cfg, $vmid);
6857
6858     my $updatefn =  sub {
6859         my ($vmid) = @_;
6860
6861         my $conf = PVE::QemuConfig->load_config($vmid);
6862
6863         PVE::QemuConfig->check_lock($conf);
6864
6865         my $vm_volids = {};
6866         foreach my $volid (keys %$volid_hash) {
6867             my $info = $volid_hash->{$volid};
6868             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6869         }
6870
6871         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6872
6873         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6874     };
6875
6876     if (defined($vmid)) {
6877         if ($nolock) {
6878             &$updatefn($vmid);
6879         } else {
6880             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6881         }
6882     } else {
6883         my $vmlist = config_list();
6884         foreach my $vmid (keys %$vmlist) {
6885             if ($nolock) {
6886                 &$updatefn($vmid);
6887             } else {
6888                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6889             }
6890         }
6891     }
6892 }
6893
6894 sub restore_proxmox_backup_archive {
6895     my ($archive, $vmid, $user, $options) = @_;
6896
6897     my $storecfg = PVE::Storage::config();
6898
6899     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6900     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6901
6902     my $fingerprint = $scfg->{fingerprint};
6903     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6904
6905     my $repo = PVE::PBSClient::get_repository($scfg);
6906     my $namespace = $scfg->{namespace};
6907
6908     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6909     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6910     local $ENV{PBS_PASSWORD} = $password;
6911     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6912
6913     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6914         PVE::Storage::parse_volname($storecfg, $archive);
6915
6916     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6917
6918     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6919
6920     my $tmpdir = "/var/tmp/vzdumptmp$$";
6921     rmtree $tmpdir;
6922     mkpath $tmpdir;
6923
6924     my $conffile = PVE::QemuConfig->config_file($vmid);
6925      # disable interrupts (always do cleanups)
6926     local $SIG{INT} =
6927         local $SIG{TERM} =
6928         local $SIG{QUIT} =
6929         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6930
6931     # Note: $oldconf is undef if VM does not exists
6932     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6933     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6934     my $new_conf_raw = '';
6935
6936     my $rpcenv = PVE::RPCEnvironment::get();
6937     my $devinfo = {}; # info about drives included in backup
6938     my $virtdev_hash = {}; # info about allocated drives
6939
6940     eval {
6941         # enable interrupts
6942         local $SIG{INT} =
6943             local $SIG{TERM} =
6944             local $SIG{QUIT} =
6945             local $SIG{HUP} =
6946             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6947
6948         my $cfgfn = "$tmpdir/qemu-server.conf";
6949         my $firewall_config_fn = "$tmpdir/fw.conf";
6950         my $index_fn = "$tmpdir/index.json";
6951
6952         my $cmd = "restore";
6953
6954         my $param = [$pbs_backup_name, "index.json", $index_fn];
6955         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6956         my $index = PVE::Tools::file_get_contents($index_fn);
6957         $index = decode_json($index);
6958
6959         foreach my $info (@{$index->{files}}) {
6960             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6961                 my $devname = $1;
6962                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6963                     $devinfo->{$devname}->{size} = $1;
6964                 } else {
6965                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6966                 }
6967             }
6968         }
6969
6970         my $is_qemu_server_backup = scalar(
6971             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6972         );
6973         if (!$is_qemu_server_backup) {
6974             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6975         }
6976         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6977
6978         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6979         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6980
6981         if ($has_firewall_config) {
6982             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6983             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6984
6985             my $pve_firewall_dir = '/etc/pve/firewall';
6986             mkdir $pve_firewall_dir; # make sure the dir exists
6987             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6988         }
6989
6990         my $fh = IO::File->new($cfgfn, "r") ||
6991             die "unable to read qemu-server.conf - $!\n";
6992
6993         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6994
6995         # fixme: rate limit?
6996
6997         # create empty/temp config
6998         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6999
7000         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7001
7002         # allocate volumes
7003         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7004
7005         foreach my $virtdev (sort keys %$virtdev_hash) {
7006             my $d = $virtdev_hash->{$virtdev};
7007             next if $d->{is_cloudinit}; # no need to restore cloudinit
7008
7009             # this fails if storage is unavailable
7010             my $volid = $d->{volid};
7011             my $path = PVE::Storage::path($storecfg, $volid);
7012
7013             # for live-restore we only want to preload the efidisk and TPM state
7014             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7015
7016             my @ns_arg;
7017             if (defined(my $ns = $scfg->{namespace})) {
7018                 @ns_arg = ('--ns', $ns);
7019             }
7020
7021             my $pbs_restore_cmd = [
7022                 '/usr/bin/pbs-restore',
7023                 '--repository', $repo,
7024                 @ns_arg,
7025                 $pbs_backup_name,
7026                 "$d->{devname}.img.fidx",
7027                 $path,
7028                 '--verbose',
7029                 ];
7030
7031             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7032             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7033
7034             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7035                 push @$pbs_restore_cmd, '--skip-zero';
7036             }
7037
7038             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7039             print "restore proxmox backup image: $dbg_cmdstring\n";
7040             run_command($pbs_restore_cmd);
7041         }
7042
7043         $fh->seek(0, 0) || die "seek failed - $!\n";
7044
7045         my $cookie = { netcount => 0 };
7046         while (defined(my $line = <$fh>)) {
7047             $new_conf_raw .= restore_update_config_line(
7048                 $cookie,
7049                 $map,
7050                 $line,
7051                 $options->{unique},
7052             );
7053         }
7054
7055         $fh->close();
7056     };
7057     my $err = $@;
7058
7059     if ($err || !$options->{live}) {
7060         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7061     }
7062
7063     rmtree $tmpdir;
7064
7065     if ($err) {
7066         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7067         die $err;
7068     }
7069
7070     if ($options->{live}) {
7071         # keep lock during live-restore
7072         $new_conf_raw .= "\nlock: create";
7073     }
7074
7075     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7076     PVE::QemuConfig->write_config($vmid, $new_conf);
7077
7078     eval { rescan($vmid, 1); };
7079     warn $@ if $@;
7080
7081     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7082
7083     if ($options->{live}) {
7084         # enable interrupts
7085         local $SIG{INT} =
7086             local $SIG{TERM} =
7087             local $SIG{QUIT} =
7088             local $SIG{HUP} =
7089             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7090
7091         my $conf = PVE::QemuConfig->load_config($vmid);
7092         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7093
7094         # these special drives are already restored before start
7095         delete $devinfo->{'drive-efidisk0'};
7096         delete $devinfo->{'drive-tpmstate0-backup'};
7097
7098         my $pbs_opts = {
7099             repo => $repo,
7100             keyfile => $keyfile,
7101             snapshot => $pbs_backup_name,
7102             namespace => $namespace,
7103         };
7104         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7105
7106         PVE::QemuConfig->remove_lock($vmid, "create");
7107     }
7108 }
7109
7110 sub pbs_live_restore {
7111     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7112
7113     print "starting VM for live-restore\n";
7114     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7115
7116     my $pbs_backing = {};
7117     for my $ds (keys %$restored_disks) {
7118         $ds =~ m/^drive-(.*)$/;
7119         my $confname = $1;
7120         $pbs_backing->{$confname} = {
7121             repository => $opts->{repo},
7122             snapshot => $opts->{snapshot},
7123             archive => "$ds.img.fidx",
7124         };
7125         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7126         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7127
7128         my $drive = parse_drive($confname, $conf->{$confname});
7129         print "restoring '$ds' to '$drive->{file}'\n";
7130     }
7131
7132     my $drives_streamed = 0;
7133     eval {
7134         # make sure HA doesn't interrupt our restore by stopping the VM
7135         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7136             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7137         }
7138
7139         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7140         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7141         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7142
7143         my $qmeventd_fd = register_qmeventd_handle($vmid);
7144
7145         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7146         # this will effectively collapse the backing image chain consisting of
7147         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7148         # removes itself once all backing images vanish with 'auto-remove=on')
7149         my $jobs = {};
7150         for my $ds (sort keys %$restored_disks) {
7151             my $job_id = "restore-$ds";
7152             mon_cmd($vmid, 'block-stream',
7153                 'job-id' => $job_id,
7154                 device => "$ds",
7155             );
7156             $jobs->{$job_id} = {};
7157         }
7158
7159         mon_cmd($vmid, 'cont');
7160         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7161
7162         print "restore-drive jobs finished successfully, removing all tracking block devices"
7163             ." to disconnect from Proxmox Backup Server\n";
7164
7165         for my $ds (sort keys %$restored_disks) {
7166             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7167         }
7168
7169         close($qmeventd_fd);
7170     };
7171
7172     my $err = $@;
7173
7174     if ($err) {
7175         warn "An error occurred during live-restore: $err\n";
7176         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7177         die "live-restore failed\n";
7178     }
7179 }
7180
7181 sub restore_vma_archive {
7182     my ($archive, $vmid, $user, $opts, $comp) = @_;
7183
7184     my $readfrom = $archive;
7185
7186     my $cfg = PVE::Storage::config();
7187     my $commands = [];
7188     my $bwlimit = $opts->{bwlimit};
7189
7190     my $dbg_cmdstring = '';
7191     my $add_pipe = sub {
7192         my ($cmd) = @_;
7193         push @$commands, $cmd;
7194         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7195         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7196         $readfrom = '-';
7197     };
7198
7199     my $input = undef;
7200     if ($archive eq '-') {
7201         $input = '<&STDIN';
7202     } else {
7203         # If we use a backup from a PVE defined storage we also consider that
7204         # storage's rate limit:
7205         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7206         if (defined($volid)) {
7207             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7208             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7209             if ($readlimit) {
7210                 print STDERR "applying read rate limit: $readlimit\n";
7211                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7212                 $add_pipe->($cstream);
7213             }
7214         }
7215     }
7216
7217     if ($comp) {
7218         my $info = PVE::Storage::decompressor_info('vma', $comp);
7219         my $cmd = $info->{decompressor};
7220         push @$cmd, $readfrom;
7221         $add_pipe->($cmd);
7222     }
7223
7224     my $tmpdir = "/var/tmp/vzdumptmp$$";
7225     rmtree $tmpdir;
7226
7227     # disable interrupts (always do cleanups)
7228     local $SIG{INT} =
7229         local $SIG{TERM} =
7230         local $SIG{QUIT} =
7231         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7232
7233     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7234     POSIX::mkfifo($mapfifo, 0600);
7235     my $fifofh;
7236     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7237
7238     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7239
7240     my $oldtimeout;
7241     my $timeout = 5;
7242
7243     my $devinfo = {}; # info about drives included in backup
7244     my $virtdev_hash = {}; # info about allocated drives
7245
7246     my $rpcenv = PVE::RPCEnvironment::get();
7247
7248     my $conffile = PVE::QemuConfig->config_file($vmid);
7249
7250     # Note: $oldconf is undef if VM does not exist
7251     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7252     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7253     my $new_conf_raw = '';
7254
7255     my %storage_limits;
7256
7257     my $print_devmap = sub {
7258         my $cfgfn = "$tmpdir/qemu-server.conf";
7259
7260         # we can read the config - that is already extracted
7261         my $fh = IO::File->new($cfgfn, "r") ||
7262             die "unable to read qemu-server.conf - $!\n";
7263
7264         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7265         if (-f $fwcfgfn) {
7266             my $pve_firewall_dir = '/etc/pve/firewall';
7267             mkdir $pve_firewall_dir; # make sure the dir exists
7268             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7269         }
7270
7271         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7272
7273         foreach my $info (values %{$virtdev_hash}) {
7274             my $storeid = $info->{storeid};
7275             next if defined($storage_limits{$storeid});
7276
7277             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7278             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7279             $storage_limits{$storeid} = $limit * 1024;
7280         }
7281
7282         foreach my $devname (keys %$devinfo) {
7283             die "found no device mapping information for device '$devname'\n"
7284                 if !$devinfo->{$devname}->{virtdev};
7285         }
7286
7287         # create empty/temp config
7288         if ($oldconf) {
7289             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7290             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7291         }
7292
7293         # allocate volumes
7294         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7295
7296         # print restore information to $fifofh
7297         foreach my $virtdev (sort keys %$virtdev_hash) {
7298             my $d = $virtdev_hash->{$virtdev};
7299             next if $d->{is_cloudinit}; # no need to restore cloudinit
7300
7301             my $storeid = $d->{storeid};
7302             my $volid = $d->{volid};
7303
7304             my $map_opts = '';
7305             if (my $limit = $storage_limits{$storeid}) {
7306                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7307             }
7308
7309             my $write_zeros = 1;
7310             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7311                 $write_zeros = 0;
7312             }
7313
7314             my $path = PVE::Storage::path($cfg, $volid);
7315
7316             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7317
7318             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7319         }
7320
7321         $fh->seek(0, 0) || die "seek failed - $!\n";
7322
7323         my $cookie = { netcount => 0 };
7324         while (defined(my $line = <$fh>)) {
7325             $new_conf_raw .= restore_update_config_line(
7326                 $cookie,
7327                 $map,
7328                 $line,
7329                 $opts->{unique},
7330             );
7331         }
7332
7333         $fh->close();
7334     };
7335
7336     eval {
7337         # enable interrupts
7338         local $SIG{INT} =
7339             local $SIG{TERM} =
7340             local $SIG{QUIT} =
7341             local $SIG{HUP} =
7342             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7343         local $SIG{ALRM} = sub { die "got timeout\n"; };
7344
7345         $oldtimeout = alarm($timeout);
7346
7347         my $parser = sub {
7348             my $line = shift;
7349
7350             print "$line\n";
7351
7352             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7353                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7354                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7355             } elsif ($line =~ m/^CTIME: /) {
7356                 # we correctly received the vma config, so we can disable
7357                 # the timeout now for disk allocation (set to 10 minutes, so
7358                 # that we always timeout if something goes wrong)
7359                 alarm(600);
7360                 &$print_devmap();
7361                 print $fifofh "done\n";
7362                 my $tmp = $oldtimeout || 0;
7363                 $oldtimeout = undef;
7364                 alarm($tmp);
7365                 close($fifofh);
7366                 $fifofh = undef;
7367             }
7368         };
7369
7370         print "restore vma archive: $dbg_cmdstring\n";
7371         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7372     };
7373     my $err = $@;
7374
7375     alarm($oldtimeout) if $oldtimeout;
7376
7377     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7378
7379     close($fifofh) if $fifofh;
7380     unlink $mapfifo;
7381     rmtree $tmpdir;
7382
7383     if ($err) {
7384         $restore_destroy_volumes->($cfg, $virtdev_hash);
7385         die $err;
7386     }
7387
7388     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7389     PVE::QemuConfig->write_config($vmid, $new_conf);
7390
7391     eval { rescan($vmid, 1); };
7392     warn $@ if $@;
7393
7394     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7395 }
7396
7397 sub restore_tar_archive {
7398     my ($archive, $vmid, $user, $opts) = @_;
7399
7400     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7401         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7402         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7403     }
7404
7405     if ($archive ne '-') {
7406         my $firstfile = tar_archive_read_firstfile($archive);
7407         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7408             if $firstfile ne 'qemu-server.conf';
7409     }
7410
7411     my $storecfg = PVE::Storage::config();
7412
7413     # avoid zombie disks when restoring over an existing VM -> cleanup first
7414     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7415     # skiplock=1 because qmrestore has set the 'create' lock itself already
7416     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7417     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7418
7419     my $tocmd = "/usr/lib/qemu-server/qmextract";
7420
7421     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7422     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7423     $tocmd .= ' --prealloc' if $opts->{prealloc};
7424     $tocmd .= ' --info' if $opts->{info};
7425
7426     # tar option "xf" does not autodetect compression when read from STDIN,
7427     # so we pipe to zcat
7428     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7429         PVE::Tools::shellquote("--to-command=$tocmd");
7430
7431     my $tmpdir = "/var/tmp/vzdumptmp$$";
7432     mkpath $tmpdir;
7433
7434     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7435     local $ENV{VZDUMP_VMID} = $vmid;
7436     local $ENV{VZDUMP_USER} = $user;
7437
7438     my $conffile = PVE::QemuConfig->config_file($vmid);
7439     my $new_conf_raw = '';
7440
7441     # disable interrupts (always do cleanups)
7442     local $SIG{INT} =
7443         local $SIG{TERM} =
7444         local $SIG{QUIT} =
7445         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7446
7447     eval {
7448         # enable interrupts
7449         local $SIG{INT} =
7450             local $SIG{TERM} =
7451             local $SIG{QUIT} =
7452             local $SIG{HUP} =
7453             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7454
7455         if ($archive eq '-') {
7456             print "extracting archive from STDIN\n";
7457             run_command($cmd, input => "<&STDIN");
7458         } else {
7459             print "extracting archive '$archive'\n";
7460             run_command($cmd);
7461         }
7462
7463         return if $opts->{info};
7464
7465         # read new mapping
7466         my $map = {};
7467         my $statfile = "$tmpdir/qmrestore.stat";
7468         if (my $fd = IO::File->new($statfile, "r")) {
7469             while (defined (my $line = <$fd>)) {
7470                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7471                     $map->{$1} = $2 if $1;
7472                 } else {
7473                     print STDERR "unable to parse line in statfile - $line\n";
7474                 }
7475             }
7476             $fd->close();
7477         }
7478
7479         my $confsrc = "$tmpdir/qemu-server.conf";
7480
7481         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7482
7483         my $cookie = { netcount => 0 };
7484         while (defined (my $line = <$srcfd>)) {
7485             $new_conf_raw .= restore_update_config_line(
7486                 $cookie,
7487                 $map,
7488                 $line,
7489                 $opts->{unique},
7490             );
7491         }
7492
7493         $srcfd->close();
7494     };
7495     if (my $err = $@) {
7496         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7497         die $err;
7498     }
7499
7500     rmtree $tmpdir;
7501
7502     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7503
7504     PVE::Cluster::cfs_update(); # make sure we read new file
7505
7506     eval { rescan($vmid, 1); };
7507     warn $@ if $@;
7508 };
7509
7510 sub foreach_storage_used_by_vm {
7511     my ($conf, $func) = @_;
7512
7513     my $sidhash = {};
7514
7515     PVE::QemuConfig->foreach_volume($conf, sub {
7516         my ($ds, $drive) = @_;
7517         return if drive_is_cdrom($drive);
7518
7519         my $volid = $drive->{file};
7520
7521         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7522         $sidhash->{$sid} = $sid if $sid;
7523     });
7524
7525     foreach my $sid (sort keys %$sidhash) {
7526         &$func($sid);
7527     }
7528 }
7529
7530 my $qemu_snap_storage = {
7531     rbd => 1,
7532 };
7533 sub do_snapshots_with_qemu {
7534     my ($storecfg, $volid, $deviceid) = @_;
7535
7536     return if $deviceid =~ m/tpmstate0/;
7537
7538     my $storage_name = PVE::Storage::parse_volume_id($volid);
7539     my $scfg = $storecfg->{ids}->{$storage_name};
7540     die "could not find storage '$storage_name'\n" if !defined($scfg);
7541
7542     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7543         return 1;
7544     }
7545
7546     if ($volid =~ m/\.(qcow2|qed)$/){
7547         return 1;
7548     }
7549
7550     return;
7551 }
7552
7553 sub qga_check_running {
7554     my ($vmid, $nowarn) = @_;
7555
7556     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7557     if ($@) {
7558         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7559         return 0;
7560     }
7561     return 1;
7562 }
7563
7564 sub template_create {
7565     my ($vmid, $conf, $disk) = @_;
7566
7567     my $storecfg = PVE::Storage::config();
7568
7569     PVE::QemuConfig->foreach_volume($conf, sub {
7570         my ($ds, $drive) = @_;
7571
7572         return if drive_is_cdrom($drive);
7573         return if $disk && $ds ne $disk;
7574
7575         my $volid = $drive->{file};
7576         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7577
7578         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7579         $drive->{file} = $voliddst;
7580         $conf->{$ds} = print_drive($drive);
7581         PVE::QemuConfig->write_config($vmid, $conf);
7582     });
7583 }
7584
7585 sub convert_iscsi_path {
7586     my ($path) = @_;
7587
7588     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7589         my $portal = $1;
7590         my $target = $2;
7591         my $lun = $3;
7592
7593         my $initiator_name = get_initiator_name();
7594
7595         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7596                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7597     }
7598
7599     die "cannot convert iscsi path '$path', unkown format\n";
7600 }
7601
7602 sub qemu_img_convert {
7603     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7604
7605     my $storecfg = PVE::Storage::config();
7606     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7607     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7608
7609     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7610
7611     my $cachemode;
7612     my $src_path;
7613     my $src_is_iscsi = 0;
7614     my $src_format;
7615
7616     if ($src_storeid) {
7617         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7618         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7619         $src_format = qemu_img_format($src_scfg, $src_volname);
7620         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7621         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7622         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7623     } elsif (-f $src_volid || -b $src_volid) {
7624         $src_path = $src_volid;
7625         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7626             $src_format = $1;
7627         }
7628     }
7629
7630     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7631
7632     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7633     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7634     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7635     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7636
7637     my $cmd = [];
7638     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7639     push @$cmd, '-l', "snapshot.name=$snapname"
7640         if $snapname && $src_format && $src_format eq "qcow2";
7641     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7642     push @$cmd, '-T', $cachemode if defined($cachemode);
7643
7644     if ($src_is_iscsi) {
7645         push @$cmd, '--image-opts';
7646         $src_path = convert_iscsi_path($src_path);
7647     } elsif ($src_format) {
7648         push @$cmd, '-f', $src_format;
7649     }
7650
7651     if ($dst_is_iscsi) {
7652         push @$cmd, '--target-image-opts';
7653         $dst_path = convert_iscsi_path($dst_path);
7654     } else {
7655         push @$cmd, '-O', $dst_format;
7656     }
7657
7658     push @$cmd, $src_path;
7659
7660     if (!$dst_is_iscsi && $is_zero_initialized) {
7661         push @$cmd, "zeroinit:$dst_path";
7662     } else {
7663         push @$cmd, $dst_path;
7664     }
7665
7666     my $parser = sub {
7667         my $line = shift;
7668         if($line =~ m/\((\S+)\/100\%\)/){
7669             my $percent = $1;
7670             my $transferred = int($size * $percent / 100);
7671             my $total_h = render_bytes($size, 1);
7672             my $transferred_h = render_bytes($transferred, 1);
7673
7674             print "transferred $transferred_h of $total_h ($percent%)\n";
7675         }
7676
7677     };
7678
7679     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7680     my $err = $@;
7681     die "copy failed: $err" if $err;
7682 }
7683
7684 sub qemu_img_format {
7685     my ($scfg, $volname) = @_;
7686
7687     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7688         return $1;
7689     } else {
7690         return "raw";
7691     }
7692 }
7693
7694 sub qemu_drive_mirror {
7695     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7696
7697     $jobs = {} if !$jobs;
7698
7699     my $qemu_target;
7700     my $format;
7701     $jobs->{"drive-$drive"} = {};
7702
7703     if ($dst_volid =~ /^nbd:/) {
7704         $qemu_target = $dst_volid;
7705         $format = "nbd";
7706     } else {
7707         my $storecfg = PVE::Storage::config();
7708         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7709
7710         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7711
7712         $format = qemu_img_format($dst_scfg, $dst_volname);
7713
7714         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7715
7716         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7717     }
7718
7719     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7720     $opts->{format} = $format if $format;
7721
7722     if (defined($src_bitmap)) {
7723         $opts->{sync} = 'incremental';
7724         $opts->{bitmap} = $src_bitmap;
7725         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7726     }
7727
7728     if (defined($bwlimit)) {
7729         $opts->{speed} = $bwlimit * 1024;
7730         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7731     } else {
7732         print "drive mirror is starting for drive-$drive\n";
7733     }
7734
7735     # if a job already runs for this device we get an error, catch it for cleanup
7736     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7737     if (my $err = $@) {
7738         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7739         warn "$@\n" if $@;
7740         die "mirroring error: $err\n";
7741     }
7742
7743     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7744 }
7745
7746 # $completion can be either
7747 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7748 # 'cancel': wait until all jobs are ready, block-job-cancel them
7749 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7750 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7751 sub qemu_drive_mirror_monitor {
7752     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7753
7754     $completion //= 'complete';
7755     $op //= "mirror";
7756
7757     eval {
7758         my $err_complete = 0;
7759
7760         my $starttime = time ();
7761         while (1) {
7762             die "block job ('$op') timed out\n" if $err_complete > 300;
7763
7764             my $stats = mon_cmd($vmid, "query-block-jobs");
7765             my $ctime = time();
7766
7767             my $running_jobs = {};
7768             for my $stat (@$stats) {
7769                 next if $stat->{type} ne $op;
7770                 $running_jobs->{$stat->{device}} = $stat;
7771             }
7772
7773             my $readycounter = 0;
7774
7775             for my $job_id (sort keys %$jobs) {
7776                 my $job = $running_jobs->{$job_id};
7777
7778                 my $vanished = !defined($job);
7779                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7780                 if($complete || ($vanished && $completion eq 'auto')) {
7781                     print "$job_id: $op-job finished\n";
7782                     delete $jobs->{$job_id};
7783                     next;
7784                 }
7785
7786                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7787
7788                 my $busy = $job->{busy};
7789                 my $ready = $job->{ready};
7790                 if (my $total = $job->{len}) {
7791                     my $transferred = $job->{offset} || 0;
7792                     my $remaining = $total - $transferred;
7793                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7794
7795                     my $duration = $ctime - $starttime;
7796                     my $total_h = render_bytes($total, 1);
7797                     my $transferred_h = render_bytes($transferred, 1);
7798
7799                     my $status = sprintf(
7800                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7801                         render_duration($duration),
7802                     );
7803
7804                     if ($ready) {
7805                         if ($busy) {
7806                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7807                         } else {
7808                             $status .= ", ready";
7809                         }
7810                     }
7811                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7812                     $jobs->{$job_id}->{ready} = $ready;
7813                 }
7814
7815                 $readycounter++ if $job->{ready};
7816             }
7817
7818             last if scalar(keys %$jobs) == 0;
7819
7820             if ($readycounter == scalar(keys %$jobs)) {
7821                 print "all '$op' jobs are ready\n";
7822
7823                 # do the complete later (or has already been done)
7824                 last if $completion eq 'skip' || $completion eq 'auto';
7825
7826                 if ($vmiddst && $vmiddst != $vmid) {
7827                     my $agent_running = $qga && qga_check_running($vmid);
7828                     if ($agent_running) {
7829                         print "freeze filesystem\n";
7830                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7831                         warn $@ if $@;
7832                     } else {
7833                         print "suspend vm\n";
7834                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7835                         warn $@ if $@;
7836                     }
7837
7838                     # if we clone a disk for a new target vm, we don't switch the disk
7839                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7840
7841                     if ($agent_running) {
7842                         print "unfreeze filesystem\n";
7843                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7844                         warn $@ if $@;
7845                     } else {
7846                         print "resume vm\n";
7847                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7848                         warn $@ if $@;
7849                     }
7850
7851                     last;
7852                 } else {
7853
7854                     for my $job_id (sort keys %$jobs) {
7855                         # try to switch the disk if source and destination are on the same guest
7856                         print "$job_id: Completing block job_id...\n";
7857
7858                         my $op;
7859                         if ($completion eq 'complete') {
7860                             $op = 'block-job-complete';
7861                         } elsif ($completion eq 'cancel') {
7862                             $op = 'block-job-cancel';
7863                         } else {
7864                             die "invalid completion value: $completion\n";
7865                         }
7866                         eval { mon_cmd($vmid, $op, device => $job_id) };
7867                         if ($@ =~ m/cannot be completed/) {
7868                             print "$job_id: block job cannot be completed, trying again.\n";
7869                             $err_complete++;
7870                         }else {
7871                             print "$job_id: Completed successfully.\n";
7872                             $jobs->{$job_id}->{complete} = 1;
7873                         }
7874                     }
7875                 }
7876             }
7877             sleep 1;
7878         }
7879     };
7880     my $err = $@;
7881
7882     if ($err) {
7883         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7884         die "block job ($op) error: $err";
7885     }
7886 }
7887
7888 sub qemu_blockjobs_cancel {
7889     my ($vmid, $jobs) = @_;
7890
7891     foreach my $job (keys %$jobs) {
7892         print "$job: Cancelling block job\n";
7893         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7894         $jobs->{$job}->{cancel} = 1;
7895     }
7896
7897     while (1) {
7898         my $stats = mon_cmd($vmid, "query-block-jobs");
7899
7900         my $running_jobs = {};
7901         foreach my $stat (@$stats) {
7902             $running_jobs->{$stat->{device}} = $stat;
7903         }
7904
7905         foreach my $job (keys %$jobs) {
7906
7907             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7908                 print "$job: Done.\n";
7909                 delete $jobs->{$job};
7910             }
7911         }
7912
7913         last if scalar(keys %$jobs) == 0;
7914
7915         sleep 1;
7916     }
7917 }
7918
7919 sub clone_disk {
7920     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7921
7922     my ($vmid, $running) = $source->@{qw(vmid running)};
7923     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7924
7925     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7926     my ($storage, $format) = $dest->@{qw(storage format)};
7927
7928     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7929
7930     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7931         die "cloning from/to EFI disk requires EFI disk\n"
7932             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7933         die "cloning from/to TPM state requires TPM state\n"
7934             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7935
7936         # This would lead to two device nodes in QEMU pointing to the same backing image!
7937         die "cannot change drive name when cloning disk from/to the same VM\n"
7938             if $use_drive_mirror && $vmid == $newvmid;
7939     }
7940
7941     die "cannot move TPM state while VM is running\n"
7942         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7943
7944     my $newvolid;
7945
7946     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7947     print "$src_drivename " if $src_drivename;
7948     print "($drive->{file})\n";
7949
7950     if (!$full) {
7951         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7952         push @$newvollist, $newvolid;
7953     } else {
7954
7955         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7956         $storeid = $storage if $storage;
7957
7958         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7959
7960         my $name = undef;
7961         my $size = undef;
7962         if (drive_is_cloudinit($drive)) {
7963             $name = "vm-$newvmid-cloudinit";
7964             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7965             if ($scfg->{path}) {
7966                 $name .= ".$dst_format";
7967             }
7968             $snapname = undef;
7969             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7970         } elsif ($dst_drivename eq 'efidisk0') {
7971             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7972         } elsif ($dst_drivename eq 'tpmstate0') {
7973             $dst_format = 'raw';
7974             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7975         } else {
7976             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7977         }
7978         $newvolid = PVE::Storage::vdisk_alloc(
7979             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7980         );
7981         push @$newvollist, $newvolid;
7982
7983         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7984
7985         if (drive_is_cloudinit($drive)) {
7986             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7987             # if this is the case, we have to complete any block-jobs still there from
7988             # previous drive-mirrors
7989             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7990                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7991             }
7992             goto no_data_clone;
7993         }
7994
7995         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7996         if ($use_drive_mirror) {
7997             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7998                 $completion, $qga, $bwlimit);
7999         } else {
8000             # TODO: handle bwlimits
8001             if ($dst_drivename eq 'efidisk0') {
8002                 # the relevant data on the efidisk may be smaller than the source
8003                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8004                 # that is given by the OVMF_VARS.fd
8005                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8006                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8007
8008                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8009
8010                 # better for Ceph if block size is not too small, see bug #3324
8011                 my $bs = 1024*1024;
8012
8013                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8014
8015                 if ($src_format eq 'qcow2' && $snapname) {
8016                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8017                         if !min_version(kvm_user_version(), 6, 2);
8018                     push $cmd->@*, '-l', $snapname;
8019                 }
8020                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8021                 run_command($cmd);
8022             } else {
8023                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8024             }
8025         }
8026     }
8027
8028 no_data_clone:
8029     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8030
8031     my $disk = dclone($drive);
8032     delete $disk->{format};
8033     $disk->{file} = $newvolid;
8034     $disk->{size} = $size if defined($size);
8035
8036     return $disk;
8037 }
8038
8039 sub get_running_qemu_version {
8040     my ($vmid) = @_;
8041     my $res = mon_cmd($vmid, "query-version");
8042     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8043 }
8044
8045 sub qemu_use_old_bios_files {
8046     my ($machine_type) = @_;
8047
8048     return if !$machine_type;
8049
8050     my $use_old_bios_files = undef;
8051
8052     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8053         $machine_type = $1;
8054         $use_old_bios_files = 1;
8055     } else {
8056         my $version = extract_version($machine_type, kvm_user_version());
8057         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8058         # load new efi bios files on migration. So this hack is required to allow
8059         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8060         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8061         $use_old_bios_files = !min_version($version, 2, 4);
8062     }
8063
8064     return ($use_old_bios_files, $machine_type);
8065 }
8066
8067 sub get_efivars_size {
8068     my ($conf, $efidisk) = @_;
8069
8070     my $arch = get_vm_arch($conf);
8071     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8072     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8073     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8074     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8075     return -s $ovmf_vars;
8076 }
8077
8078 sub update_efidisk_size {
8079     my ($conf) = @_;
8080
8081     return if !defined($conf->{efidisk0});
8082
8083     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8084     $disk->{size} = get_efivars_size($conf);
8085     $conf->{efidisk0} = print_drive($disk);
8086
8087     return;
8088 }
8089
8090 sub update_tpmstate_size {
8091     my ($conf) = @_;
8092
8093     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8094     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8095     $conf->{tpmstate0} = print_drive($disk);
8096 }
8097
8098 sub create_efidisk($$$$$$$) {
8099     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8100
8101     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8102     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8103
8104     my $vars_size_b = -s $ovmf_vars;
8105     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8106     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8107     PVE::Storage::activate_volumes($storecfg, [$volid]);
8108
8109     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8110     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8111
8112     return ($volid, $size/1024);
8113 }
8114
8115 sub vm_iothreads_list {
8116     my ($vmid) = @_;
8117
8118     my $res = mon_cmd($vmid, 'query-iothreads');
8119
8120     my $iothreads = {};
8121     foreach my $iothread (@$res) {
8122         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8123     }
8124
8125     return $iothreads;
8126 }
8127
8128 sub scsihw_infos {
8129     my ($conf, $drive) = @_;
8130
8131     my $maxdev = 0;
8132
8133     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8134         $maxdev = 7;
8135     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8136         $maxdev = 1;
8137     } else {
8138         $maxdev = 256;
8139     }
8140
8141     my $controller = int($drive->{index} / $maxdev);
8142     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8143         ? "virtioscsi"
8144         : "scsihw";
8145
8146     return ($maxdev, $controller, $controller_prefix);
8147 }
8148
8149 sub resolve_dst_disk_format {
8150         my ($storecfg, $storeid, $src_volname, $format) = @_;
8151         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8152
8153         if (!$format) {
8154             # if no target format is specified, use the source disk format as hint
8155             if ($src_volname) {
8156                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8157                 $format = qemu_img_format($scfg, $src_volname);
8158             } else {
8159                 return $defFormat;
8160             }
8161         }
8162
8163         # test if requested format is supported - else use default
8164         my $supported = grep { $_ eq $format } @$validFormats;
8165         $format = $defFormat if !$supported;
8166         return $format;
8167 }
8168
8169 # NOTE: if this logic changes, please update docs & possibly gui logic
8170 sub find_vmstate_storage {
8171     my ($conf, $storecfg) = @_;
8172
8173     # first, return storage from conf if set
8174     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8175
8176     my ($target, $shared, $local);
8177
8178     foreach_storage_used_by_vm($conf, sub {
8179         my ($sid) = @_;
8180         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8181         my $dst = $scfg->{shared} ? \$shared : \$local;
8182         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8183     });
8184
8185     # second, use shared storage where VM has at least one disk
8186     # third, use local storage where VM has at least one disk
8187     # fall back to local storage
8188     $target = $shared // $local // 'local';
8189
8190     return $target;
8191 }
8192
8193 sub generate_uuid {
8194     my ($uuid, $uuid_str);
8195     UUID::generate($uuid);
8196     UUID::unparse($uuid, $uuid_str);
8197     return $uuid_str;
8198 }
8199
8200 sub generate_smbios1_uuid {
8201     return "uuid=".generate_uuid();
8202 }
8203
8204 sub nbd_stop {
8205     my ($vmid) = @_;
8206
8207     mon_cmd($vmid, 'nbd-server-stop');
8208 }
8209
8210 sub create_reboot_request {
8211     my ($vmid) = @_;
8212     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8213         or die "failed to create reboot trigger file: $!\n";
8214     close($fh);
8215 }
8216
8217 sub clear_reboot_request {
8218     my ($vmid) = @_;
8219     my $path = "/run/qemu-server/$vmid.reboot";
8220     my $res = 0;
8221
8222     $res = unlink($path);
8223     die "could not remove reboot request for $vmid: $!"
8224         if !$res && $! != POSIX::ENOENT;
8225
8226     return $res;
8227 }
8228
8229 sub bootorder_from_legacy {
8230     my ($conf, $bootcfg) = @_;
8231
8232     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8233     my $bootindex_hash = {};
8234     my $i = 1;
8235     foreach my $o (split(//, $boot)) {
8236         $bootindex_hash->{$o} = $i*100;
8237         $i++;
8238     }
8239
8240     my $bootorder = {};
8241
8242     PVE::QemuConfig->foreach_volume($conf, sub {
8243         my ($ds, $drive) = @_;
8244
8245         if (drive_is_cdrom ($drive, 1)) {
8246             if ($bootindex_hash->{d}) {
8247                 $bootorder->{$ds} = $bootindex_hash->{d};
8248                 $bootindex_hash->{d} += 1;
8249             }
8250         } elsif ($bootindex_hash->{c}) {
8251             $bootorder->{$ds} = $bootindex_hash->{c}
8252                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8253             $bootindex_hash->{c} += 1;
8254         }
8255     });
8256
8257     if ($bootindex_hash->{n}) {
8258         for (my $i = 0; $i < $MAX_NETS; $i++) {
8259             my $netname = "net$i";
8260             next if !$conf->{$netname};
8261             $bootorder->{$netname} = $bootindex_hash->{n};
8262             $bootindex_hash->{n} += 1;
8263         }
8264     }
8265
8266     return $bootorder;
8267 }
8268
8269 # Generate default device list for 'boot: order=' property. Matches legacy
8270 # default boot order, but with explicit device names. This is important, since
8271 # the fallback for when neither 'order' nor the old format is specified relies
8272 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8273 sub get_default_bootdevices {
8274     my ($conf) = @_;
8275
8276     my @ret = ();
8277
8278     # harddisk
8279     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8280     push @ret, $first if $first;
8281
8282     # cdrom
8283     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8284     push @ret, $first if $first;
8285
8286     # network
8287     for (my $i = 0; $i < $MAX_NETS; $i++) {
8288         my $netname = "net$i";
8289         next if !$conf->{$netname};
8290         push @ret, $netname;
8291         last;
8292     }
8293
8294     return \@ret;
8295 }
8296
8297 sub device_bootorder {
8298     my ($conf) = @_;
8299
8300     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8301
8302     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8303
8304     my $bootorder = {};
8305     if (!defined($boot) || $boot->{legacy}) {
8306         $bootorder = bootorder_from_legacy($conf, $boot);
8307     } elsif ($boot->{order}) {
8308         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8309         for my $dev (PVE::Tools::split_list($boot->{order})) {
8310             $bootorder->{$dev} = $i++;
8311         }
8312     }
8313
8314     return $bootorder;
8315 }
8316
8317 sub register_qmeventd_handle {
8318     my ($vmid) = @_;
8319
8320     my $fh;
8321     my $peer = "/var/run/qmeventd.sock";
8322     my $count = 0;
8323
8324     for (;;) {
8325         $count++;
8326         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8327         last if $fh;
8328         if ($! != EINTR && $! != EAGAIN) {
8329             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8330         }
8331         if ($count > 4) {
8332             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8333               . "after $count retries\n";
8334         }
8335         usleep(25000);
8336     }
8337
8338     # send handshake to mark VM as backing up
8339     print $fh to_json({vzdump => {vmid => "$vmid"}});
8340
8341     # return handle to be closed later when inhibit is no longer required
8342     return $fh;
8343 }
8344
8345 # bash completion helper
8346
8347 sub complete_backup_archives {
8348     my ($cmdname, $pname, $cvalue) = @_;
8349
8350     my $cfg = PVE::Storage::config();
8351
8352     my $storeid;
8353
8354     if ($cvalue =~ m/^([^:]+):/) {
8355         $storeid = $1;
8356     }
8357
8358     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8359
8360     my $res = [];
8361     foreach my $id (keys %$data) {
8362         foreach my $item (@{$data->{$id}}) {
8363             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8364             push @$res, $item->{volid} if defined($item->{volid});
8365         }
8366     }
8367
8368     return $res;
8369 }
8370
8371 my $complete_vmid_full = sub {
8372     my ($running) = @_;
8373
8374     my $idlist = vmstatus();
8375
8376     my $res = [];
8377
8378     foreach my $id (keys %$idlist) {
8379         my $d = $idlist->{$id};
8380         if (defined($running)) {
8381             next if $d->{template};
8382             next if $running && $d->{status} ne 'running';
8383             next if !$running && $d->{status} eq 'running';
8384         }
8385         push @$res, $id;
8386
8387     }
8388     return $res;
8389 };
8390
8391 sub complete_vmid {
8392     return &$complete_vmid_full();
8393 }
8394
8395 sub complete_vmid_stopped {
8396     return &$complete_vmid_full(0);
8397 }
8398
8399 sub complete_vmid_running {
8400     return &$complete_vmid_full(1);
8401 }
8402
8403 sub complete_storage {
8404
8405     my $cfg = PVE::Storage::config();
8406     my $ids = $cfg->{ids};
8407
8408     my $res = [];
8409     foreach my $sid (keys %$ids) {
8410         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8411         next if !$ids->{$sid}->{content}->{images};
8412         push @$res, $sid;
8413     }
8414
8415     return $res;
8416 }
8417
8418 sub complete_migration_storage {
8419     my ($cmd, $param, $current_value, $all_args) = @_;
8420
8421     my $targetnode = @$all_args[1];
8422
8423     my $cfg = PVE::Storage::config();
8424     my $ids = $cfg->{ids};
8425
8426     my $res = [];
8427     foreach my $sid (keys %$ids) {
8428         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8429         next if !$ids->{$sid}->{content}->{images};
8430         push @$res, $sid;
8431     }
8432
8433     return $res;
8434 }
8435
8436 sub vm_is_paused {
8437     my ($vmid) = @_;
8438     my $qmpstatus = eval {
8439         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8440         mon_cmd($vmid, "query-status");
8441     };
8442     warn "$@\n" if $@;
8443     return $qmpstatus && $qmpstatus->{status} eq "paused";
8444 }
8445
8446 sub check_volume_storage_type {
8447     my ($storecfg, $vol) = @_;
8448
8449     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8450     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8451     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8452
8453     die "storage '$storeid' does not support content-type '$vtype'\n"
8454         if !$scfg->{content}->{$vtype};
8455
8456     return 1;
8457 }
8458
8459 sub add_nets_bridge_fdb {
8460     my ($conf, $vmid) = @_;
8461
8462     for my $opt (keys %$conf) {
8463         next if $opt !~ m/^net(\d+)$/;
8464         my $iface = "tap${vmid}i$1";
8465         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8466         my $net = parse_net($conf->{$opt}, 1) or next;
8467
8468         my $mac = $net->{macaddr};
8469         if (!$mac) {
8470             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8471                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8472             next;
8473         }
8474
8475         my $bridge = $net->{bridge};
8476         if ($have_sdn) {
8477             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8478         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8479             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8480         }
8481     }
8482 }
8483
8484 sub del_nets_bridge_fdb {
8485     my ($conf, $vmid) = @_;
8486
8487     for my $opt (keys %$conf) {
8488         next if $opt !~ m/^net(\d+)$/;
8489         my $iface = "tap${vmid}i$1";
8490
8491         my $net = parse_net($conf->{$opt}) or next;
8492         my $mac = $net->{macaddr} or next;
8493
8494         my $bridge = $net->{bridge};
8495         if ($have_sdn) {
8496             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8497         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8498             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8499         }
8500     }
8501 }
8502
8503 1;