PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2345
2346     if ($conf->{template}) {
2347         # check if any base image is still used by a linked clone
2348         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2349                 my ($ds, $drive) = @_;
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353                 return if !$volid || $volid =~ m|^/|;
2354
2355                 die "base volume '$volid' is still in use by linked cloned\n"
2356                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2357
2358         });
2359     }
2360
2361     my $volids = {};
2362     my $remove_owned_drive = sub {
2363         my ($ds, $drive) = @_;
2364         return if drive_is_cdrom($drive, 1);
2365
2366         my $volid = $drive->{file};
2367         return if !$volid || $volid =~ m|^/|;
2368         return if $volids->{$volid};
2369
2370         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2371         return if !$path || !$owner || ($owner != $vmid);
2372
2373         $volids->{$volid} = 1;
2374         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2375         warn "Could not remove disk '$volid', check manually: $@" if $@;
2376     };
2377
2378     # only remove disks owned by this VM (referenced in the config)
2379     my $include_opts = {
2380         include_unused => 1,
2381         extra_keys => ['vmstate'],
2382     };
2383     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2384
2385     for my $snap (values %{$conf->{snapshots}}) {
2386         next if !defined($snap->{vmstate});
2387         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2388         next if !defined($drive);
2389         $remove_owned_drive->('vmstate', $drive);
2390     }
2391
2392     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2393
2394     if ($purge_unreferenced) { # also remove unreferenced disk
2395         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2396         PVE::Storage::foreach_volid($vmdisks, sub {
2397             my ($volid, $sid, $volname, $d) = @_;
2398             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2399             warn $@ if $@;
2400         });
2401     }
2402
2403     if (defined $replacement_conf) {
2404         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2405     } else {
2406         PVE::QemuConfig->destroy_config($vmid);
2407     }
2408 }
2409
2410 sub parse_vm_config {
2411     my ($filename, $raw, $strict) = @_;
2412
2413     return if !defined($raw);
2414
2415     my $res = {
2416         digest => Digest::SHA::sha1_hex($raw),
2417         snapshots => {},
2418         pending => {},
2419         cloudinit => {},
2420     };
2421
2422     my $handle_error = sub {
2423         my ($msg) = @_;
2424
2425         if ($strict) {
2426             die $msg;
2427         } else {
2428             warn $msg;
2429         }
2430     };
2431
2432     $filename =~ m|/qemu-server/(\d+)\.conf$|
2433         || die "got strange filename '$filename'";
2434
2435     my $vmid = $1;
2436
2437     my $conf = $res;
2438     my $descr;
2439     my $finish_description = sub {
2440         if (defined($descr)) {
2441             $descr =~ s/\s+$//;
2442             $conf->{description} = $descr;
2443         }
2444         $descr = undef;
2445     };
2446     my $section = '';
2447
2448     my @lines = split(/\n/, $raw);
2449     foreach my $line (@lines) {
2450         next if $line =~ m/^\s*$/;
2451
2452         if ($line =~ m/^\[PENDING\]\s*$/i) {
2453             $section = 'pending';
2454             $finish_description->();
2455             $conf = $res->{$section} = {};
2456             next;
2457         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458             $section = 'cloudinit';
2459             $finish_description->();
2460             $conf = $res->{$section} = {};
2461             next;
2462
2463         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2464             $section = $1;
2465             $finish_description->();
2466             $conf = $res->{snapshots}->{$section} = {};
2467             next;
2468         }
2469
2470         if ($line =~ m/^\#(.*)$/) {
2471             $descr = '' if !defined($descr);
2472             $descr .= PVE::Tools::decode_text($1) . "\n";
2473             next;
2474         }
2475
2476         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477             $descr = '' if !defined($descr);
2478             $descr .= PVE::Tools::decode_text($2);
2479         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480             $conf->{snapstate} = $1;
2481         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2482             my $key = $1;
2483             my $value = $2;
2484             $conf->{$key} = $value;
2485         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2486             my $value = $1;
2487             if ($section eq 'pending') {
2488                 $conf->{delete} = $value; # we parse this later
2489             } else {
2490                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2491             }
2492         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             if ($section eq 'cloudinit') {
2496                 # ignore validation only used for informative purpose
2497                 $conf->{$key} = $value;
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             if ($@) {
2502                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2503             } else {
2504                 $key = 'ide2' if $key eq 'cdrom';
2505                 my $fmt = $confdesc->{$key}->{format};
2506                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507                     my $v = parse_drive($key, $value);
2508                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2509                         $v->{file} = $volid;
2510                         $value = print_drive($v);
2511                     } else {
2512                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513                         next;
2514                     }
2515                 }
2516
2517                 $conf->{$key} = $value;
2518             }
2519         } else {
2520             $handle_error->("vm $vmid - unable to parse config: $line\n");
2521         }
2522     }
2523
2524     $finish_description->();
2525     delete $res->{snapstate}; # just to be sure
2526
2527     return $res;
2528 }
2529
2530 sub write_vm_config {
2531     my ($filename, $conf) = @_;
2532
2533     delete $conf->{snapstate}; # just to be sure
2534
2535     if ($conf->{cdrom}) {
2536         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2537         $conf->{ide2} = $conf->{cdrom};
2538         delete $conf->{cdrom};
2539     }
2540
2541     # we do not use 'smp' any longer
2542     if ($conf->{sockets}) {
2543         delete $conf->{smp};
2544     } elsif ($conf->{smp}) {
2545         $conf->{sockets} = $conf->{smp};
2546         delete $conf->{cores};
2547         delete $conf->{smp};
2548     }
2549
2550     my $used_volids = {};
2551
2552     my $cleanup_config = sub {
2553         my ($cref, $pending, $snapname) = @_;
2554
2555         foreach my $key (keys %$cref) {
2556             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558             my $value = $cref->{$key};
2559             if ($key eq 'delete') {
2560                 die "propertry 'delete' is only allowed in [PENDING]\n"
2561                     if !$pending;
2562                 # fixme: check syntax?
2563                 next;
2564             }
2565             eval { $value = check_type($key, $value); };
2566             die "unable to parse value of '$key' - $@" if $@;
2567
2568             $cref->{$key} = $value;
2569
2570             if (!$snapname && is_valid_drivename($key)) {
2571                 my $drive = parse_drive($key, $value);
2572                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2573             }
2574         }
2575     };
2576
2577     &$cleanup_config($conf);
2578
2579     &$cleanup_config($conf->{pending}, 1);
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2584     }
2585
2586     # remove 'unusedX' settings if we re-add a volume
2587     foreach my $key (keys %$conf) {
2588         my $value = $conf->{$key};
2589         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590             delete $conf->{$key};
2591         }
2592     }
2593
2594     my $generate_raw_config = sub {
2595         my ($conf, $pending) = @_;
2596
2597         my $raw = '';
2598
2599         # add description as comment to top of file
2600         if (defined(my $descr = $conf->{description})) {
2601             if ($descr) {
2602                 foreach my $cl (split(/\n/, $descr)) {
2603                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2604                 }
2605             } else {
2606                 $raw .= "#\n" if $pending;
2607             }
2608         }
2609
2610         foreach my $key (sort keys %$conf) {
2611             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612             $raw .= "$key: $conf->{$key}\n";
2613         }
2614         return $raw;
2615     };
2616
2617     my $raw = &$generate_raw_config($conf);
2618
2619     if (scalar(keys %{$conf->{pending}})){
2620         $raw .= "\n[PENDING]\n";
2621         $raw .= &$generate_raw_config($conf->{pending}, 1);
2622     }
2623
2624     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2625         $raw .= "\n[special:cloudinit]\n";
2626         $raw .= &$generate_raw_config($conf->{cloudinit});
2627     }
2628
2629     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2630         $raw .= "\n[$snapname]\n";
2631         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2632     }
2633
2634     return $raw;
2635 }
2636
2637 sub load_defaults {
2638
2639     my $res = {};
2640
2641     # we use static defaults from our JSON schema configuration
2642     foreach my $key (keys %$confdesc) {
2643         if (defined(my $default = $confdesc->{$key}->{default})) {
2644             $res->{$key} = $default;
2645         }
2646     }
2647
2648     return $res;
2649 }
2650
2651 sub config_list {
2652     my $vmlist = PVE::Cluster::get_vmlist();
2653     my $res = {};
2654     return $res if !$vmlist || !$vmlist->{ids};
2655     my $ids = $vmlist->{ids};
2656     my $nodename = nodename();
2657
2658     foreach my $vmid (keys %$ids) {
2659         my $d = $ids->{$vmid};
2660         next if !$d->{node} || $d->{node} ne $nodename;
2661         next if !$d->{type} || $d->{type} ne 'qemu';
2662         $res->{$vmid}->{exists} = 1;
2663     }
2664     return $res;
2665 }
2666
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources {
2669     my ($conf, $noerr) = @_;
2670
2671     my @loc_res = ();
2672
2673     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2674     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2675
2676     push @loc_res, "ivshmem" if $conf->{ivshmem};
2677
2678     foreach my $k (keys %$conf) {
2679         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680         # sockets are safe: they will recreated be on the target side post-migrate
2681         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2683     }
2684
2685     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2686
2687     return \@loc_res;
2688 }
2689
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability {
2692     my ($storecfg, $conf, $node) = @_;
2693
2694     PVE::QemuConfig->foreach_volume($conf, sub {
2695         my ($ds, $drive) = @_;
2696
2697         my $volid = $drive->{file};
2698         return if !$volid;
2699
2700         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2701         return if !$sid;
2702
2703         # check if storage is available on both nodes
2704         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2705         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2706
2707         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2708
2709         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710             if !$scfg->{content}->{$vtype};
2711    });
2712 }
2713
2714 # list nodes where all VM images are available (used by has_feature API)
2715 sub shared_nodes {
2716     my ($conf, $storecfg) = @_;
2717
2718     my $nodelist = PVE::Cluster::get_nodelist();
2719     my $nodehash = { map { $_ => 1 } @$nodelist };
2720     my $nodename = nodename();
2721
2722     PVE::QemuConfig->foreach_volume($conf, sub {
2723         my ($ds, $drive) = @_;
2724
2725         my $volid = $drive->{file};
2726         return if !$volid;
2727
2728         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2729         if ($storeid) {
2730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2731             if ($scfg->{disable}) {
2732                 $nodehash = {};
2733             } elsif (my $avail = $scfg->{nodes}) {
2734                 foreach my $node (keys %$nodehash) {
2735                     delete $nodehash->{$node} if !$avail->{$node};
2736                 }
2737             } elsif (!$scfg->{shared}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     delete $nodehash->{$node} if $node ne $nodename
2740                 }
2741             }
2742         }
2743     });
2744
2745     return $nodehash
2746 }
2747
2748 sub check_local_storage_availability {
2749     my ($conf, $storecfg) = @_;
2750
2751     my $nodelist = PVE::Cluster::get_nodelist();
2752     my $nodehash = { map { $_ => {} } @$nodelist };
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763
2764             if ($scfg->{disable}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2767                 }
2768             } elsif (my $avail = $scfg->{nodes}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     if (!$avail->{$node}) {
2771                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2772                     }
2773                 }
2774             }
2775         }
2776     });
2777
2778     foreach my $node (values %$nodehash) {
2779         if (my $unavail = $node->{unavailable_storages}) {
2780             $node->{unavailable_storages} = [ sort keys %$unavail ];
2781         }
2782     }
2783
2784     return $nodehash
2785 }
2786
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2788 sub check_running {
2789     my ($vmid, $nocheck, $node) = @_;
2790
2791     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2792     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2793 }
2794
2795 sub vzlist {
2796
2797     my $vzlist = config_list();
2798
2799     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2800
2801     while (defined(my $de = $fd->read)) {
2802         next if $de !~ m/^(\d+)\.pid$/;
2803         my $vmid = $1;
2804         next if !defined($vzlist->{$vmid});
2805         if (my $pid = check_running($vmid)) {
2806             $vzlist->{$vmid}->{pid} = $pid;
2807         }
2808     }
2809
2810     return $vzlist;
2811 }
2812
2813 our $vmstatus_return_properties = {
2814     vmid => get_standard_option('pve-vmid'),
2815     status => {
2816         description => "Qemu process status.",
2817         type => 'string',
2818         enum => ['stopped', 'running'],
2819     },
2820     maxmem => {
2821         description => "Maximum memory in bytes.",
2822         type => 'integer',
2823         optional => 1,
2824         renderer => 'bytes',
2825     },
2826     maxdisk => {
2827         description => "Root disk size in bytes.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'bytes',
2831     },
2832     name => {
2833         description => "VM name.",
2834         type => 'string',
2835         optional => 1,
2836     },
2837     qmpstatus => {
2838         description => "Qemu QMP agent status.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     pid => {
2843         description => "PID of running qemu process.",
2844         type => 'integer',
2845         optional => 1,
2846     },
2847     uptime => {
2848         description => "Uptime.",
2849         type => 'integer',
2850         optional => 1,
2851         renderer => 'duration',
2852     },
2853     cpus => {
2854         description => "Maximum usable CPUs.",
2855         type => 'number',
2856         optional => 1,
2857     },
2858     lock => {
2859         description => "The current config lock, if any.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     tags => {
2864         description  => "The current configured tags, if any",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     'running-machine' => {
2869         description => "The currently running machine type (if running).",
2870         type => 'string',
2871         optional => 1,
2872     },
2873     'running-qemu' => {
2874         description => "The currently running QEMU version (if running).",
2875         type => 'string',
2876         optional => 1,
2877     },
2878 };
2879
2880 my $last_proc_pid_stat;
2881
2882 # get VM status information
2883 # This must be fast and should not block ($full == false)
2884 # We only query KVM using QMP if $full == true (this can be slow)
2885 sub vmstatus {
2886     my ($opt_vmid, $full) = @_;
2887
2888     my $res = {};
2889
2890     my $storecfg = PVE::Storage::config();
2891
2892     my $list = vzlist();
2893     my $defaults = load_defaults();
2894
2895     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2896
2897     my $cpucount = $cpuinfo->{cpus} || 1;
2898
2899     foreach my $vmid (keys %$list) {
2900         next if $opt_vmid && ($vmid ne $opt_vmid);
2901
2902         my $conf = PVE::QemuConfig->load_config($vmid);
2903
2904         my $d = { vmid => int($vmid) };
2905         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2906
2907         # fixme: better status?
2908         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2909
2910         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2911         if (defined($size)) {
2912             $d->{disk} = 0; # no info available
2913             $d->{maxdisk} = $size;
2914         } else {
2915             $d->{disk} = 0;
2916             $d->{maxdisk} = 0;
2917         }
2918
2919         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2920             * ($conf->{cores} || $defaults->{cores});
2921         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2922         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2923
2924         $d->{name} = $conf->{name} || "VM $vmid";
2925         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2926             : $defaults->{memory}*(1024*1024);
2927
2928         if ($conf->{balloon}) {
2929             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2930             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2931                 : $defaults->{shares};
2932         }
2933
2934         $d->{uptime} = 0;
2935         $d->{cpu} = 0;
2936         $d->{mem} = 0;
2937
2938         $d->{netout} = 0;
2939         $d->{netin} = 0;
2940
2941         $d->{diskread} = 0;
2942         $d->{diskwrite} = 0;
2943
2944         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2945
2946         $d->{serial} = 1 if conf_has_serial($conf);
2947         $d->{lock} = $conf->{lock} if $conf->{lock};
2948         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2949
2950         $res->{$vmid} = $d;
2951     }
2952
2953     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2954     foreach my $dev (keys %$netdev) {
2955         next if $dev !~ m/^tap([1-9]\d*)i/;
2956         my $vmid = $1;
2957         my $d = $res->{$vmid};
2958         next if !$d;
2959
2960         $d->{netout} += $netdev->{$dev}->{receive};
2961         $d->{netin} += $netdev->{$dev}->{transmit};
2962
2963         if ($full) {
2964             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2965             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2966         }
2967
2968     }
2969
2970     my $ctime = gettimeofday;
2971
2972     foreach my $vmid (keys %$list) {
2973
2974         my $d = $res->{$vmid};
2975         my $pid = $d->{pid};
2976         next if !$pid;
2977
2978         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2979         next if !$pstat; # not running
2980
2981         my $used = $pstat->{utime} + $pstat->{stime};
2982
2983         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2984
2985         if ($pstat->{vsize}) {
2986             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2987         }
2988
2989         my $old = $last_proc_pid_stat->{$pid};
2990         if (!$old) {
2991             $last_proc_pid_stat->{$pid} = {
2992                 time => $ctime,
2993                 used => $used,
2994                 cpu => 0,
2995             };
2996             next;
2997         }
2998
2999         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3000
3001         if ($dtime > 1000) {
3002             my $dutime = $used -  $old->{used};
3003
3004             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3005             $last_proc_pid_stat->{$pid} = {
3006                 time => $ctime,
3007                 used => $used,
3008                 cpu => $d->{cpu},
3009             };
3010         } else {
3011             $d->{cpu} = $old->{cpu};
3012         }
3013     }
3014
3015     return $res if !$full;
3016
3017     my $qmpclient = PVE::QMPClient->new();
3018
3019     my $ballooncb = sub {
3020         my ($vmid, $resp) = @_;
3021
3022         my $info = $resp->{'return'};
3023         return if !$info->{max_mem};
3024
3025         my $d = $res->{$vmid};
3026
3027         # use memory assigned to VM
3028         $d->{maxmem} = $info->{max_mem};
3029         $d->{balloon} = $info->{actual};
3030
3031         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3032             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3033             $d->{freemem} = $info->{free_mem};
3034         }
3035
3036         $d->{ballooninfo} = $info;
3037     };
3038
3039     my $blockstatscb = sub {
3040         my ($vmid, $resp) = @_;
3041         my $data = $resp->{'return'} || [];
3042         my $totalrdbytes = 0;
3043         my $totalwrbytes = 0;
3044
3045         for my $blockstat (@$data) {
3046             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3047             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3048
3049             $blockstat->{device} =~ s/drive-//;
3050             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3051         }
3052         $res->{$vmid}->{diskread} = $totalrdbytes;
3053         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3054     };
3055
3056     my $machinecb = sub {
3057         my ($vmid, $resp) = @_;
3058         my $data = $resp->{'return'} || [];
3059
3060         $res->{$vmid}->{'running-machine'} =
3061             PVE::QemuServer::Machine::current_from_query_machines($data);
3062     };
3063
3064     my $versioncb = sub {
3065         my ($vmid, $resp) = @_;
3066         my $data = $resp->{'return'} // {};
3067         my $version = 'unknown';
3068
3069         if (my $v = $data->{qemu}) {
3070             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3071         }
3072
3073         $res->{$vmid}->{'running-qemu'} = $version;
3074     };
3075
3076     my $statuscb = sub {
3077         my ($vmid, $resp) = @_;
3078
3079         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3080         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3081         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3082         # this fails if ballon driver is not loaded, so this must be
3083         # the last commnand (following command are aborted if this fails).
3084         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3085
3086         my $status = 'unknown';
3087         if (!defined($status = $resp->{'return'}->{status})) {
3088             warn "unable to get VM status\n";
3089             return;
3090         }
3091
3092         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3093     };
3094
3095     foreach my $vmid (keys %$list) {
3096         next if $opt_vmid && ($vmid ne $opt_vmid);
3097         next if !$res->{$vmid}->{pid}; # not running
3098         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3099     }
3100
3101     $qmpclient->queue_execute(undef, 2);
3102
3103     foreach my $vmid (keys %$list) {
3104         next if $opt_vmid && ($vmid ne $opt_vmid);
3105         next if !$res->{$vmid}->{pid}; #not running
3106
3107         # we can't use the $qmpclient since it might have already aborted on
3108         # 'query-balloon', but this might also fail for older versions...
3109         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3110         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3111     }
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3116     }
3117
3118     return $res;
3119 }
3120
3121 sub conf_has_serial {
3122     my ($conf) = @_;
3123
3124     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3125         if ($conf->{"serial$i"}) {
3126             return 1;
3127         }
3128     }
3129
3130     return 0;
3131 }
3132
3133 sub conf_has_audio {
3134     my ($conf, $id) = @_;
3135
3136     $id //= 0;
3137     my $audio = $conf->{"audio$id"};
3138     return if !defined($audio);
3139
3140     my $audioproperties = parse_property_string($audio_fmt, $audio);
3141     my $audiodriver = $audioproperties->{driver} // 'spice';
3142
3143     return {
3144         dev => $audioproperties->{device},
3145         dev_id => "audiodev$id",
3146         backend => $audiodriver,
3147         backend_id => "$audiodriver-backend${id}",
3148     };
3149 }
3150
3151 sub audio_devs {
3152     my ($audio, $audiopciaddr, $machine_version) = @_;
3153
3154     my $devs = [];
3155
3156     my $id = $audio->{dev_id};
3157     my $audiodev = "";
3158     if (min_version($machine_version, 4, 2)) {
3159         $audiodev = ",audiodev=$audio->{backend_id}";
3160     }
3161
3162     if ($audio->{dev} eq 'AC97') {
3163         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3164     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3165         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3166         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3167         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3168     } else {
3169         die "unkown audio device '$audio->{dev}', implement me!";
3170     }
3171
3172     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3173
3174     return $devs;
3175 }
3176
3177 sub get_tpm_paths {
3178     my ($vmid) = @_;
3179     return {
3180         socket => "/var/run/qemu-server/$vmid.swtpm",
3181         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3182     };
3183 }
3184
3185 sub add_tpm_device {
3186     my ($vmid, $devices, $conf) = @_;
3187
3188     return if !$conf->{tpmstate0};
3189
3190     my $paths = get_tpm_paths($vmid);
3191
3192     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3193     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3194     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3195 }
3196
3197 sub start_swtpm {
3198     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3199
3200     return if !$tpmdrive;
3201
3202     my $state;
3203     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3204     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3205     if ($storeid) {
3206         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3207     } else {
3208         $state = $tpm->{file};
3209     }
3210
3211     my $paths = get_tpm_paths($vmid);
3212
3213     # during migration, we will get state from remote
3214     #
3215     if (!$migration) {
3216         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3217         my $setup_cmd = [
3218             "swtpm_setup",
3219             "--tpmstate",
3220             "file://$state",
3221             "--createek",
3222             "--create-ek-cert",
3223             "--create-platform-cert",
3224             "--lock-nvram",
3225             "--config",
3226             "/etc/swtpm_setup.conf", # do not use XDG configs
3227             "--runas",
3228             "0", # force creation as root, error if not possible
3229             "--not-overwrite", # ignore existing state, do not modify
3230         ];
3231
3232         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3233         # TPM 2.0 supports ECC crypto, use if possible
3234         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3235
3236         run_command($setup_cmd, outfunc => sub {
3237             print "swtpm_setup: $1\n";
3238         });
3239     }
3240
3241     my $emulator_cmd = [
3242         "swtpm",
3243         "socket",
3244         "--tpmstate",
3245         "backend-uri=file://$state,mode=0600",
3246         "--ctrl",
3247         "type=unixio,path=$paths->{socket},mode=0600",
3248         "--pid",
3249         "file=$paths->{pid}",
3250         "--terminate", # terminate on QEMU disconnect
3251         "--daemon",
3252     ];
3253     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3254     run_command($emulator_cmd, outfunc => sub { print $1; });
3255
3256     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3257     while (! -e $paths->{pid}) {
3258         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3259         usleep(50_000);
3260     }
3261
3262     # return untainted PID of swtpm daemon so it can be killed on error
3263     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3264     return $1;
3265 }
3266
3267 sub vga_conf_has_spice {
3268     my ($vga) = @_;
3269
3270     my $vgaconf = parse_vga($vga);
3271     my $vgatype = $vgaconf->{type};
3272     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3273
3274     return $1 || 1;
3275 }
3276
3277 sub is_native($) {
3278     my ($arch) = @_;
3279     return get_host_arch() eq $arch;
3280 }
3281
3282 sub get_vm_arch {
3283     my ($conf) = @_;
3284     return $conf->{arch} // get_host_arch();
3285 }
3286
3287 my $default_machines = {
3288     x86_64 => 'pc',
3289     aarch64 => 'virt',
3290 };
3291
3292 sub get_installed_machine_version {
3293     my ($kvmversion) = @_;
3294     $kvmversion = kvm_user_version() if !defined($kvmversion);
3295     $kvmversion =~ m/^(\d+\.\d+)/;
3296     return $1;
3297 }
3298
3299 sub windows_get_pinned_machine_version {
3300     my ($machine, $base_version, $kvmversion) = @_;
3301
3302     my $pin_version = $base_version;
3303     if (!defined($base_version) ||
3304         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3305     ) {
3306         $pin_version = get_installed_machine_version($kvmversion);
3307     }
3308     if (!$machine || $machine eq 'pc') {
3309         $machine = "pc-i440fx-$pin_version";
3310     } elsif ($machine eq 'q35') {
3311         $machine = "pc-q35-$pin_version";
3312     } elsif ($machine eq 'virt') {
3313         $machine = "virt-$pin_version";
3314     } else {
3315         warn "unknown machine type '$machine', not touching that!\n";
3316     }
3317
3318     return $machine;
3319 }
3320
3321 sub get_vm_machine {
3322     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3323
3324     my $machine = $forcemachine || $conf->{machine};
3325
3326     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3327         $kvmversion //= kvm_user_version();
3328         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3329         # layout which confuses windows quite a bit and may result in various regressions..
3330         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3331         if (windows_version($conf->{ostype})) {
3332             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3333         }
3334         $arch //= 'x86_64';
3335         $machine ||= $default_machines->{$arch};
3336         if ($add_pve_version) {
3337             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3338             $machine .= "+pve$pvever";
3339         }
3340     }
3341
3342     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3343         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3344         $machine = $1 if $is_pxe;
3345
3346         # for version-pinned machines that do not include a pve-version (e.g.
3347         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3348         $machine .= '+pve0';
3349
3350         $machine .= '.pxe' if $is_pxe;
3351     }
3352
3353     return $machine;
3354 }
3355
3356 sub get_ovmf_files($$$) {
3357     my ($arch, $efidisk, $smm) = @_;
3358
3359     my $types = $OVMF->{$arch}
3360         or die "no OVMF images known for architecture '$arch'\n";
3361
3362     my $type = 'default';
3363     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3364         $type = $smm ? "4m" : "4m-no-smm";
3365         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3366     }
3367
3368     return $types->{$type}->@*;
3369 }
3370
3371 my $Arch2Qemu = {
3372     aarch64 => '/usr/bin/qemu-system-aarch64',
3373     x86_64 => '/usr/bin/qemu-system-x86_64',
3374 };
3375 sub get_command_for_arch($) {
3376     my ($arch) = @_;
3377     return '/usr/bin/kvm' if is_native($arch);
3378
3379     my $cmd = $Arch2Qemu->{$arch}
3380         or die "don't know how to emulate architecture '$arch'\n";
3381     return $cmd;
3382 }
3383
3384 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3385 # to use in a QEMU command line (-cpu element), first array_intersect the result
3386 # of query_supported_ with query_understood_. This is necessary because:
3387 #
3388 # a) query_understood_ returns flags the host cannot use and
3389 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3390 #    flags, but CPU settings - with most of them being flags. Those settings
3391 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3392 #
3393 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3394 # expensive. If you need the value returned from this, you can get it much
3395 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3396 # $accel being 'kvm' or 'tcg'.
3397 #
3398 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3399 # changes, automatically populating pmxcfs.
3400 #
3401 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3402 # since kvm and tcg machines support different flags
3403 #
3404 sub query_supported_cpu_flags {
3405     my ($arch) = @_;
3406
3407     $arch //= get_host_arch();
3408     my $default_machine = $default_machines->{$arch};
3409
3410     my $flags = {};
3411
3412     # FIXME: Once this is merged, the code below should work for ARM as well:
3413     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3414     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3415         $arch eq "aarch64";
3416
3417     my $kvm_supported = defined(kvm_version());
3418     my $qemu_cmd = get_command_for_arch($arch);
3419     my $fakevmid = -1;
3420     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3421
3422     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3423     my $query_supported_run_qemu = sub {
3424         my ($kvm) = @_;
3425
3426         my $flags = {};
3427         my $cmd = [
3428             $qemu_cmd,
3429             '-machine', $default_machine,
3430             '-display', 'none',
3431             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3432             '-mon', 'chardev=qmp,mode=control',
3433             '-pidfile', $pidfile,
3434             '-S', '-daemonize'
3435         ];
3436
3437         if (!$kvm) {
3438             push @$cmd, '-accel', 'tcg';
3439         }
3440
3441         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3442         die "QEMU flag querying VM exited with code " . $rc if $rc;
3443
3444         eval {
3445             my $cmd_result = mon_cmd(
3446                 $fakevmid,
3447                 'query-cpu-model-expansion',
3448                 type => 'full',
3449                 model => { name => 'host' }
3450             );
3451
3452             my $props = $cmd_result->{model}->{props};
3453             foreach my $prop (keys %$props) {
3454                 next if $props->{$prop} ne '1';
3455                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3456                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3457                 # We only keep those with underscores, to match /proc/cpuinfo
3458                 $prop =~ s/\.|-/_/g;
3459                 $flags->{$prop} = 1;
3460             }
3461         };
3462         my $err = $@;
3463
3464         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3465         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3466
3467         die $err if $err;
3468
3469         return [ sort keys %$flags ];
3470     };
3471
3472     # We need to query QEMU twice, since KVM and TCG have different supported flags
3473     PVE::QemuConfig->lock_config($fakevmid, sub {
3474         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3475         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3476
3477         if ($kvm_supported) {
3478             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3479             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3480         }
3481     });
3482
3483     return $flags;
3484 }
3485
3486 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3487 my $understood_cpu_flag_dir = "/usr/share/kvm";
3488 sub query_understood_cpu_flags {
3489     my $arch = get_host_arch();
3490     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3491
3492     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3493         if ! -e $filepath;
3494
3495     my $raw = file_get_contents($filepath);
3496     $raw =~ s/^\s+|\s+$//g;
3497     my @flags = split(/\s+/, $raw);
3498
3499     return \@flags;
3500 }
3501
3502 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3503 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3504 my sub should_disable_smm {
3505     my ($conf, $vga) = @_;
3506
3507     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3508         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3509 }
3510
3511 sub config_to_command {
3512     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3513         $pbs_backing) = @_;
3514
3515     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3516     my $devices = [];
3517     my $bridges = {};
3518     my $ostype = $conf->{ostype};
3519     my $winversion = windows_version($ostype);
3520     my $kvm = $conf->{kvm};
3521     my $nodename = nodename();
3522
3523     my $arch = get_vm_arch($conf);
3524     my $kvm_binary = get_command_for_arch($arch);
3525     my $kvmver = kvm_user_version($kvm_binary);
3526
3527     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3528         $kvmver //= "undefined";
3529         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3530     }
3531
3532     my $add_pve_version = min_version($kvmver, 4, 1);
3533
3534     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3535     my $machine_version = extract_version($machine_type, $kvmver);
3536     $kvm //= 1 if is_native($arch);
3537
3538     $machine_version =~ m/(\d+)\.(\d+)/;
3539     my ($machine_major, $machine_minor) = ($1, $2);
3540
3541     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3542         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3543     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3544         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3545             ." please upgrade node '$nodename'\n"
3546     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3547         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3548         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3549             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3550             ." node '$nodename'\n";
3551     }
3552
3553     # if a specific +pve version is required for a feature, use $version_guard
3554     # instead of min_version to allow machines to be run with the minimum
3555     # required version
3556     my $required_pve_version = 0;
3557     my $version_guard = sub {
3558         my ($major, $minor, $pve) = @_;
3559         return 0 if !min_version($machine_version, $major, $minor, $pve);
3560         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3561         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3562         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3563         return 1;
3564     };
3565
3566     if ($kvm && !defined kvm_version()) {
3567         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3568             ." or enable in BIOS.\n";
3569     }
3570
3571     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3572     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3573     my $use_old_bios_files = undef;
3574     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3575
3576     my $cmd = [];
3577     if ($conf->{affinity}) {
3578         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3579     }
3580
3581     push @$cmd, $kvm_binary;
3582
3583     push @$cmd, '-id', $vmid;
3584
3585     my $vmname = $conf->{name} || "vm$vmid";
3586
3587     push @$cmd, '-name', "$vmname,debug-threads=on";
3588
3589     push @$cmd, '-no-shutdown';
3590
3591     my $use_virtio = 0;
3592
3593     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3594     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3595     push @$cmd, '-mon', "chardev=qmp,mode=control";
3596
3597     if (min_version($machine_version, 2, 12)) {
3598         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3599         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3600     }
3601
3602     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3603
3604     push @$cmd, '-daemonize';
3605
3606     if ($conf->{smbios1}) {
3607         my $smbios_conf = parse_smbios1($conf->{smbios1});
3608         if ($smbios_conf->{base64}) {
3609             # Do not pass base64 flag to qemu
3610             delete $smbios_conf->{base64};
3611             my $smbios_string = "";
3612             foreach my $key (keys %$smbios_conf) {
3613                 my $value;
3614                 if ($key eq "uuid") {
3615                     $value = $smbios_conf->{uuid}
3616                 } else {
3617                     $value = decode_base64($smbios_conf->{$key});
3618                 }
3619                 # qemu accepts any binary data, only commas need escaping by double comma
3620                 $value =~ s/,/,,/g;
3621                 $smbios_string .= "," . $key . "=" . $value if $value;
3622             }
3623             push @$cmd, '-smbios', "type=1" . $smbios_string;
3624         } else {
3625             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3626         }
3627     }
3628
3629     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3630         my $d;
3631         if (my $efidisk = $conf->{efidisk0}) {
3632             $d = parse_drive('efidisk0', $efidisk);
3633         }
3634
3635         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3636         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3637
3638         my ($path, $format);
3639         my $read_only_str = '';
3640         if ($d) {
3641             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3642             $format = $d->{format};
3643             if ($storeid) {
3644                 $path = PVE::Storage::path($storecfg, $d->{file});
3645                 if (!defined($format)) {
3646                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3647                     $format = qemu_img_format($scfg, $volname);
3648                 }
3649             } else {
3650                 $path = $d->{file};
3651                 die "efidisk format must be specified\n"
3652                     if !defined($format);
3653             }
3654
3655             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3656         } else {
3657             log_warn("no efidisk configured! Using temporary efivars disk.");
3658             $path = "/tmp/$vmid-ovmf.fd";
3659             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3660             $format = 'raw';
3661         }
3662
3663         my $size_str = "";
3664
3665         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3666             $size_str = ",size=" . (-s $ovmf_vars);
3667         }
3668
3669         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3670         my $cache = "";
3671         if ($path =~ m/^rbd:/) {
3672                 $cache = ',cache=writeback';
3673                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3674         }
3675
3676         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3677         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3678     }
3679
3680     if ($q35) { # tell QEMU to load q35 config early
3681         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3682         if (min_version($machine_version, 4, 0)) {
3683             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3684         } else {
3685             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3686         }
3687     }
3688
3689     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3690         push @$cmd, $fixups->@*;
3691     }
3692
3693     if ($conf->{vmgenid}) {
3694         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3695     }
3696
3697     # add usb controllers
3698     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3699         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3700     push @$devices, @usbcontrollers if @usbcontrollers;
3701     my $vga = parse_vga($conf->{vga});
3702
3703     my $qxlnum = vga_conf_has_spice($conf->{vga});
3704     $vga->{type} = 'qxl' if $qxlnum;
3705
3706     if (!$vga->{type}) {
3707         if ($arch eq 'aarch64') {
3708             $vga->{type} = 'virtio';
3709         } elsif (min_version($machine_version, 2, 9)) {
3710             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3711         } else {
3712             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3713         }
3714     }
3715
3716     # enable absolute mouse coordinates (needed by vnc)
3717     my $tablet = $conf->{tablet};
3718     if (!defined($tablet)) {
3719         $tablet = $defaults->{tablet};
3720         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3721         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3722     }
3723
3724     if ($tablet) {
3725         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3726         my $kbd = print_keyboarddevice_full($conf, $arch);
3727         push @$devices, '-device', $kbd if defined($kbd);
3728     }
3729
3730     my $bootorder = device_bootorder($conf);
3731
3732     # host pci device passthrough
3733     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3734         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3735
3736     # usb devices
3737     my $usb_dev_features = {};
3738     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3739
3740     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3741         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3742     push @$devices, @usbdevices if @usbdevices;
3743
3744     # serial devices
3745     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3746         my $path = $conf->{"serial$i"} or next;
3747         if ($path eq 'socket') {
3748             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3749             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3750             # On aarch64, serial0 is the UART device. Qemu only allows
3751             # connecting UART devices via the '-serial' command line, as
3752             # the device has a fixed slot on the hardware...
3753             if ($arch eq 'aarch64' && $i == 0) {
3754                 push @$devices, '-serial', "chardev:serial$i";
3755             } else {
3756                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3757             }
3758         } else {
3759             die "no such serial device\n" if ! -c $path;
3760             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3761             push @$devices, '-device', "isa-serial,chardev=serial$i";
3762         }
3763     }
3764
3765     # parallel devices
3766     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3767         if (my $path = $conf->{"parallel$i"}) {
3768             die "no such parallel device\n" if ! -c $path;
3769             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3770             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3771             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3772         }
3773     }
3774
3775     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3776         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3777         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3778         push @$devices, @$audio_devs;
3779     }
3780
3781     add_tpm_device($vmid, $devices, $conf);
3782
3783     my $sockets = 1;
3784     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3785     $sockets = $conf->{sockets} if  $conf->{sockets};
3786
3787     my $cores = $conf->{cores} || 1;
3788
3789     my $maxcpus = $sockets * $cores;
3790
3791     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3792
3793     my $allowed_vcpus = $cpuinfo->{cpus};
3794
3795     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3796
3797     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3798         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3799         for (my $i = 2; $i <= $vcpus; $i++)  {
3800             my $cpustr = print_cpu_device($conf,$i);
3801             push @$cmd, '-device', $cpustr;
3802         }
3803
3804     } else {
3805
3806         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3807     }
3808     push @$cmd, '-nodefaults';
3809
3810     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3811
3812     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3813
3814     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3815
3816     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3817         push @$devices, '-device', print_vga_device(
3818             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3819
3820         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3821
3822         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3823         push @$cmd,  '-vnc', "unix:$socket,password=on";
3824     } else {
3825         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3826         push @$cmd, '-nographic';
3827     }
3828
3829     # time drift fix
3830     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3831     my $useLocaltime = $conf->{localtime};
3832
3833     if ($winversion >= 5) { # windows
3834         $useLocaltime = 1 if !defined($conf->{localtime});
3835
3836         # use time drift fix when acpi is enabled
3837         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3838             $tdf = 1 if !defined($conf->{tdf});
3839         }
3840     }
3841
3842     if ($winversion >= 6) {
3843         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3844         push @$cmd, '-no-hpet';
3845     }
3846
3847     push @$rtcFlags, 'driftfix=slew' if $tdf;
3848
3849     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3850         push @$rtcFlags, "base=$conf->{startdate}";
3851     } elsif ($useLocaltime) {
3852         push @$rtcFlags, 'base=localtime';
3853     }
3854
3855     if ($forcecpu) {
3856         push @$cmd, '-cpu', $forcecpu;
3857     } else {
3858         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3859     }
3860
3861     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3862
3863     push @$cmd, '-S' if $conf->{freeze};
3864
3865     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3866
3867     my $guest_agent = parse_guest_agent($conf);
3868
3869     if ($guest_agent->{enabled}) {
3870         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3871         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3872
3873         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3874             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3875             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3876             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3877         } elsif ($guest_agent->{type} eq 'isa') {
3878             push @$devices, '-device', "isa-serial,chardev=qga0";
3879         }
3880     }
3881
3882     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3883     if ($rng && $version_guard->(4, 1, 2)) {
3884         check_rng_source($rng->{source});
3885
3886         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3887         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3888         my $limiter_str = "";
3889         if ($max_bytes) {
3890             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3891         }
3892
3893         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3894         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3895         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3896     }
3897
3898     my $spice_port;
3899
3900     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3901         if ($qxlnum > 1) {
3902             if ($winversion){
3903                 for (my $i = 1; $i < $qxlnum; $i++){
3904                     push @$devices, '-device', print_vga_device(
3905                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3906                 }
3907             } else {
3908                 # assume other OS works like Linux
3909                 my ($ram, $vram) = ("134217728", "67108864");
3910                 if ($vga->{memory}) {
3911                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3912                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3913                 }
3914                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3915                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3916             }
3917         }
3918
3919         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3920
3921         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3922         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3923         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3924
3925         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3926         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3927         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3928
3929         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3930         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3931
3932         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3933         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3934         if ($spice_enhancement->{foldersharing}) {
3935             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3936             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3937         }
3938
3939         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3940         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3941             if $spice_enhancement->{videostreaming};
3942
3943         push @$devices, '-spice', "$spice_opts";
3944     }
3945
3946     # enable balloon by default, unless explicitly disabled
3947     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3948         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3949         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3950         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3951         push @$devices, '-device', $ballooncmd;
3952     }
3953
3954     if ($conf->{watchdog}) {
3955         my $wdopts = parse_watchdog($conf->{watchdog});
3956         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3957         my $watchdog = $wdopts->{model} || 'i6300esb';
3958         push @$devices, '-device', "$watchdog$pciaddr";
3959         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3960     }
3961
3962     my $vollist = [];
3963     my $scsicontroller = {};
3964     my $ahcicontroller = {};
3965     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3966
3967     # Add iscsi initiator name if available
3968     if (my $initiator = get_initiator_name()) {
3969         push @$devices, '-iscsi', "initiator-name=$initiator";
3970     }
3971
3972     PVE::QemuConfig->foreach_volume($conf, sub {
3973         my ($ds, $drive) = @_;
3974
3975         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3976             check_volume_storage_type($storecfg, $drive->{file});
3977             push @$vollist, $drive->{file};
3978         }
3979
3980         # ignore efidisk here, already added in bios/fw handling code above
3981         return if $drive->{interface} eq 'efidisk';
3982         # similar for TPM
3983         return if $drive->{interface} eq 'tpmstate';
3984
3985         $use_virtio = 1 if $ds =~ m/^virtio/;
3986
3987         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3988
3989         if ($drive->{interface} eq 'virtio'){
3990            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3991         }
3992
3993         if ($drive->{interface} eq 'scsi') {
3994
3995             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3996
3997             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3998                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3999
4000             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4001             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4002
4003             my $iothread = '';
4004             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4005                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4006                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4007             } elsif ($drive->{iothread}) {
4008                 log_warn(
4009                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4010                 );
4011             }
4012
4013             my $queues = '';
4014             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4015                 $queues = ",num_queues=$drive->{queues}";
4016             }
4017
4018             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4019                 if !$scsicontroller->{$controller};
4020             $scsicontroller->{$controller}=1;
4021         }
4022
4023         if ($drive->{interface} eq 'sata') {
4024             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4025             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4026             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4027                 if !$ahcicontroller->{$controller};
4028             $ahcicontroller->{$controller}=1;
4029         }
4030
4031         my $pbs_conf = $pbs_backing->{$ds};
4032         my $pbs_name = undef;
4033         if ($pbs_conf) {
4034             $pbs_name = "drive-$ds-pbs";
4035             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4036         }
4037
4038         my $drive_cmd = print_drive_commandline_full(
4039             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4040
4041         # extra protection for templates, but SATA and IDE don't support it..
4042         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4043
4044         push @$devices, '-drive',$drive_cmd;
4045         push @$devices, '-device', print_drivedevice_full(
4046             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4047     });
4048
4049     for (my $i = 0; $i < $MAX_NETS; $i++) {
4050         my $netname = "net$i";
4051
4052         next if !$conf->{$netname};
4053         my $d = parse_net($conf->{$netname});
4054         next if !$d;
4055         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4056
4057         $use_virtio = 1 if $d->{model} eq 'virtio';
4058
4059         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4060
4061         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4062         push @$devices, '-netdev', $netdevfull;
4063
4064         my $netdevicefull = print_netdevice_full(
4065             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4066
4067         push @$devices, '-device', $netdevicefull;
4068     }
4069
4070     if ($conf->{ivshmem}) {
4071         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4072
4073         my $bus;
4074         if ($q35) {
4075             $bus = print_pcie_addr("ivshmem");
4076         } else {
4077             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4078         }
4079
4080         my $ivshmem_name = $ivshmem->{name} // $vmid;
4081         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4082
4083         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4084         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4085             .",size=$ivshmem->{size}M";
4086     }
4087
4088     # pci.4 is nested in pci.1
4089     $bridges->{1} = 1 if $bridges->{4};
4090
4091     if (!$q35) { # add pci bridges
4092         if (min_version($machine_version, 2, 3)) {
4093            $bridges->{1} = 1;
4094            $bridges->{2} = 1;
4095         }
4096         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4097     }
4098
4099     for my $k (sort {$b cmp $a} keys %$bridges) {
4100         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4101
4102         my $k_name = $k;
4103         if ($k == 2 && $legacy_igd) {
4104             $k_name = "$k-igd";
4105         }
4106         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4107         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4108
4109         if ($q35) { # add after -readconfig pve-q35.cfg
4110             splice @$devices, 2, 0, '-device', $devstr;
4111         } else {
4112             unshift @$devices, '-device', $devstr if $k > 0;
4113         }
4114     }
4115
4116     if (!$kvm) {
4117         push @$machineFlags, 'accel=tcg';
4118     }
4119
4120     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4121
4122     my $machine_type_min = $machine_type;
4123     if ($add_pve_version) {
4124         $machine_type_min =~ s/\+pve\d+$//;
4125         $machine_type_min .= "+pve$required_pve_version";
4126     }
4127     push @$machineFlags, "type=${machine_type_min}";
4128
4129     push @$cmd, @$devices;
4130     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4131     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4132     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4133
4134     if (my $vmstate = $conf->{vmstate}) {
4135         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4136         push @$vollist, $vmstate;
4137         push @$cmd, '-loadstate', $statepath;
4138         print "activating and using '$vmstate' as vmstate\n";
4139     }
4140
4141     if (PVE::QemuConfig->is_template($conf)) {
4142         # needed to workaround base volumes being read-only
4143         push @$cmd, '-snapshot';
4144     }
4145
4146     # add custom args
4147     if ($conf->{args}) {
4148         my $aa = PVE::Tools::split_args($conf->{args});
4149         push @$cmd, @$aa;
4150     }
4151
4152     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4153 }
4154
4155 sub check_rng_source {
4156     my ($source) = @_;
4157
4158     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4159     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4160         if ! -e $source;
4161
4162     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4163     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4164         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4165         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4166         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4167             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4168             ." to the host.\n";
4169     }
4170 }
4171
4172 sub spice_port {
4173     my ($vmid) = @_;
4174
4175     my $res = mon_cmd($vmid, 'query-spice');
4176
4177     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4178 }
4179
4180 sub vm_devices_list {
4181     my ($vmid) = @_;
4182
4183     my $res = mon_cmd($vmid, 'query-pci');
4184     my $devices_to_check = [];
4185     my $devices = {};
4186     foreach my $pcibus (@$res) {
4187         push @$devices_to_check, @{$pcibus->{devices}},
4188     }
4189
4190     while (@$devices_to_check) {
4191         my $to_check = [];
4192         for my $d (@$devices_to_check) {
4193             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4194             next if !$d->{'pci_bridge'};
4195
4196             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4197             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4198         }
4199         $devices_to_check = $to_check;
4200     }
4201
4202     my $resblock = mon_cmd($vmid, 'query-block');
4203     foreach my $block (@$resblock) {
4204         if($block->{device} =~ m/^drive-(\S+)/){
4205                 $devices->{$1} = 1;
4206         }
4207     }
4208
4209     my $resmice = mon_cmd($vmid, 'query-mice');
4210     foreach my $mice (@$resmice) {
4211         if ($mice->{name} eq 'QEMU HID Tablet') {
4212             $devices->{tablet} = 1;
4213             last;
4214         }
4215     }
4216
4217     # for usb devices there is no query-usb
4218     # but we can iterate over the entries in
4219     # qom-list path=/machine/peripheral
4220     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4221     foreach my $per (@$resperipheral) {
4222         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4223             $devices->{$per->{name}} = 1;
4224         }
4225     }
4226
4227     return $devices;
4228 }
4229
4230 sub vm_deviceplug {
4231     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4232
4233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4234
4235     my $devices_list = vm_devices_list($vmid);
4236     return 1 if defined($devices_list->{$deviceid});
4237
4238     # add PCI bridge if we need it for the device
4239     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4240
4241     if ($deviceid eq 'tablet') {
4242         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4243     } elsif ($deviceid eq 'keyboard') {
4244         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4245     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4246         my $id = $1;
4247         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4248         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4249     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4250         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4251     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4252         qemu_iothread_add($vmid, $deviceid, $device);
4253
4254         qemu_driveadd($storecfg, $vmid, $device);
4255         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4256
4257         qemu_deviceadd($vmid, $devicefull);
4258         eval { qemu_deviceaddverify($vmid, $deviceid); };
4259         if (my $err = $@) {
4260             eval { qemu_drivedel($vmid, $deviceid); };
4261             warn $@ if $@;
4262             die $err;
4263         }
4264     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4265         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4266         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4267         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4268
4269         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4270
4271         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4272             qemu_iothread_add($vmid, $deviceid, $device);
4273             $devicefull .= ",iothread=iothread-$deviceid";
4274         }
4275
4276         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4277             $devicefull .= ",num_queues=$device->{queues}";
4278         }
4279
4280         qemu_deviceadd($vmid, $devicefull);
4281         qemu_deviceaddverify($vmid, $deviceid);
4282     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4283         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4284         qemu_driveadd($storecfg, $vmid, $device);
4285
4286         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4287         eval { qemu_deviceadd($vmid, $devicefull); };
4288         if (my $err = $@) {
4289             eval { qemu_drivedel($vmid, $deviceid); };
4290             warn $@ if $@;
4291             die $err;
4292         }
4293     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4294         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4295
4296         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4297         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4298         my $use_old_bios_files = undef;
4299         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4300
4301         my $netdevicefull = print_netdevice_full(
4302             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4303         qemu_deviceadd($vmid, $netdevicefull);
4304         eval {
4305             qemu_deviceaddverify($vmid, $deviceid);
4306             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4307         };
4308         if (my $err = $@) {
4309             eval { qemu_netdevdel($vmid, $deviceid); };
4310             warn $@ if $@;
4311             die $err;
4312         }
4313     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4314         my $bridgeid = $2;
4315         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4316         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4317
4318         qemu_deviceadd($vmid, $devicefull);
4319         qemu_deviceaddverify($vmid, $deviceid);
4320     } else {
4321         die "can't hotplug device '$deviceid'\n";
4322     }
4323
4324     return 1;
4325 }
4326
4327 # fixme: this should raise exceptions on error!
4328 sub vm_deviceunplug {
4329     my ($vmid, $conf, $deviceid) = @_;
4330
4331     my $devices_list = vm_devices_list($vmid);
4332     return 1 if !defined($devices_list->{$deviceid});
4333
4334     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4335     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4336
4337     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4338         qemu_devicedel($vmid, $deviceid);
4339     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4340         qemu_devicedel($vmid, $deviceid);
4341         qemu_devicedelverify($vmid, $deviceid);
4342     } elsif ($deviceid =~ m/^usb\d+$/) {
4343         qemu_devicedel($vmid, $deviceid);
4344         qemu_devicedelverify($vmid, $deviceid);
4345     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4346         my $device = parse_drive($deviceid, $conf->{$deviceid});
4347
4348         qemu_devicedel($vmid, $deviceid);
4349         qemu_devicedelverify($vmid, $deviceid);
4350         qemu_drivedel($vmid, $deviceid);
4351         qemu_iothread_del($vmid, $deviceid, $device);
4352     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4353         qemu_devicedel($vmid, $deviceid);
4354         qemu_devicedelverify($vmid, $deviceid);
4355     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4356         my $device = parse_drive($deviceid, $conf->{$deviceid});
4357
4358         qemu_devicedel($vmid, $deviceid);
4359         qemu_devicedelverify($vmid, $deviceid);
4360         qemu_drivedel($vmid, $deviceid);
4361         qemu_deletescsihw($conf, $vmid, $deviceid);
4362
4363         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4364             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4365     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4366         qemu_devicedel($vmid, $deviceid);
4367         qemu_devicedelverify($vmid, $deviceid);
4368         qemu_netdevdel($vmid, $deviceid);
4369     } else {
4370         die "can't unplug device '$deviceid'\n";
4371     }
4372
4373     return 1;
4374 }
4375
4376 sub qemu_spice_usbredir_chardev_add {
4377     my ($vmid, $id) = @_;
4378
4379     mon_cmd($vmid, "chardev-add" , (
4380         id => $id,
4381         backend => {
4382             type => 'spicevmc',
4383             data => {
4384                 type => "usbredir",
4385             },
4386         },
4387     ));
4388 }
4389
4390 sub qemu_deviceadd {
4391     my ($vmid, $devicefull) = @_;
4392
4393     $devicefull = "driver=".$devicefull;
4394     my %options =  split(/[=,]/, $devicefull);
4395
4396     mon_cmd($vmid, "device_add" , %options);
4397 }
4398
4399 sub qemu_devicedel {
4400     my ($vmid, $deviceid) = @_;
4401
4402     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4403 }
4404
4405 sub qemu_iothread_add {
4406     my ($vmid, $deviceid, $device) = @_;
4407
4408     if ($device->{iothread}) {
4409         my $iothreads = vm_iothreads_list($vmid);
4410         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4411     }
4412 }
4413
4414 sub qemu_iothread_del {
4415     my ($vmid, $deviceid, $device) = @_;
4416
4417     if ($device->{iothread}) {
4418         my $iothreads = vm_iothreads_list($vmid);
4419         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4420     }
4421 }
4422
4423 sub qemu_objectadd {
4424     my ($vmid, $objectid, $qomtype) = @_;
4425
4426     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4427
4428     return 1;
4429 }
4430
4431 sub qemu_objectdel {
4432     my ($vmid, $objectid) = @_;
4433
4434     mon_cmd($vmid, "object-del", id => $objectid);
4435
4436     return 1;
4437 }
4438
4439 sub qemu_driveadd {
4440     my ($storecfg, $vmid, $device) = @_;
4441
4442     my $kvmver = get_running_qemu_version($vmid);
4443     my $io_uring = min_version($kvmver, 6, 0);
4444     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4445     $drive =~ s/\\/\\\\/g;
4446     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4447
4448     # If the command succeeds qemu prints: "OK"
4449     return 1 if $ret =~ m/OK/s;
4450
4451     die "adding drive failed: $ret\n";
4452 }
4453
4454 sub qemu_drivedel {
4455     my ($vmid, $deviceid) = @_;
4456
4457     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4458     $ret =~ s/^\s+//;
4459
4460     return 1 if $ret eq "";
4461
4462     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4463     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4464
4465     die "deleting drive $deviceid failed : $ret\n";
4466 }
4467
4468 sub qemu_deviceaddverify {
4469     my ($vmid, $deviceid) = @_;
4470
4471     for (my $i = 0; $i <= 5; $i++) {
4472          my $devices_list = vm_devices_list($vmid);
4473          return 1 if defined($devices_list->{$deviceid});
4474          sleep 1;
4475     }
4476
4477     die "error on hotplug device '$deviceid'\n";
4478 }
4479
4480
4481 sub qemu_devicedelverify {
4482     my ($vmid, $deviceid) = @_;
4483
4484     # need to verify that the device is correctly removed as device_del
4485     # is async and empty return is not reliable
4486
4487     for (my $i = 0; $i <= 5; $i++) {
4488          my $devices_list = vm_devices_list($vmid);
4489          return 1 if !defined($devices_list->{$deviceid});
4490          sleep 1;
4491     }
4492
4493     die "error on hot-unplugging device '$deviceid'\n";
4494 }
4495
4496 sub qemu_findorcreatescsihw {
4497     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4498
4499     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4500
4501     my $scsihwid="$controller_prefix$controller";
4502     my $devices_list = vm_devices_list($vmid);
4503
4504     if (!defined($devices_list->{$scsihwid})) {
4505         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4506     }
4507
4508     return 1;
4509 }
4510
4511 sub qemu_deletescsihw {
4512     my ($conf, $vmid, $opt) = @_;
4513
4514     my $device = parse_drive($opt, $conf->{$opt});
4515
4516     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4517         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4518         return 1;
4519     }
4520
4521     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4522
4523     my $devices_list = vm_devices_list($vmid);
4524     foreach my $opt (keys %{$devices_list}) {
4525         if (is_valid_drivename($opt)) {
4526             my $drive = parse_drive($opt, $conf->{$opt});
4527             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4528                 return 1;
4529             }
4530         }
4531     }
4532
4533     my $scsihwid="scsihw$controller";
4534
4535     vm_deviceunplug($vmid, $conf, $scsihwid);
4536
4537     return 1;
4538 }
4539
4540 sub qemu_add_pci_bridge {
4541     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4542
4543     my $bridges = {};
4544
4545     my $bridgeid;
4546
4547     print_pci_addr($device, $bridges, $arch, $machine_type);
4548
4549     while (my ($k, $v) = each %$bridges) {
4550         $bridgeid = $k;
4551     }
4552     return 1 if !defined($bridgeid) || $bridgeid < 1;
4553
4554     my $bridge = "pci.$bridgeid";
4555     my $devices_list = vm_devices_list($vmid);
4556
4557     if (!defined($devices_list->{$bridge})) {
4558         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4559     }
4560
4561     return 1;
4562 }
4563
4564 sub qemu_set_link_status {
4565     my ($vmid, $device, $up) = @_;
4566
4567     mon_cmd($vmid, "set_link", name => $device,
4568                up => $up ? JSON::true : JSON::false);
4569 }
4570
4571 sub qemu_netdevadd {
4572     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4573
4574     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4575     my %options =  split(/[=,]/, $netdev);
4576
4577     if (defined(my $vhost = $options{vhost})) {
4578         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4579     }
4580
4581     if (defined(my $queues = $options{queues})) {
4582         $options{queues} = $queues + 0;
4583     }
4584
4585     mon_cmd($vmid, "netdev_add",  %options);
4586     return 1;
4587 }
4588
4589 sub qemu_netdevdel {
4590     my ($vmid, $deviceid) = @_;
4591
4592     mon_cmd($vmid, "netdev_del", id => $deviceid);
4593 }
4594
4595 sub qemu_usb_hotplug {
4596     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4597
4598     return if !$device;
4599
4600     # remove the old one first
4601     vm_deviceunplug($vmid, $conf, $deviceid);
4602
4603     # check if xhci controller is necessary and available
4604     my $devicelist = vm_devices_list($vmid);
4605
4606     if (!$devicelist->{xhci}) {
4607         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4608         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4609     }
4610
4611     # print_usbdevice_full expects the parsed device
4612     my $d = parse_usb_device($device->{host});
4613     $d->{usb3} = $device->{usb3};
4614
4615     # add the new one
4616     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4617 }
4618
4619 sub qemu_cpu_hotplug {
4620     my ($vmid, $conf, $vcpus) = @_;
4621
4622     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4623
4624     my $sockets = 1;
4625     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4626     $sockets = $conf->{sockets} if  $conf->{sockets};
4627     my $cores = $conf->{cores} || 1;
4628     my $maxcpus = $sockets * $cores;
4629
4630     $vcpus = $maxcpus if !$vcpus;
4631
4632     die "you can't add more vcpus than maxcpus\n"
4633         if $vcpus > $maxcpus;
4634
4635     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4636
4637     if ($vcpus < $currentvcpus) {
4638
4639         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4640
4641             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4642                 qemu_devicedel($vmid, "cpu$i");
4643                 my $retry = 0;
4644                 my $currentrunningvcpus = undef;
4645                 while (1) {
4646                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4647                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4648                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4649                     $retry++;
4650                     sleep 1;
4651                 }
4652                 #update conf after each succesfull cpu unplug
4653                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4654                 PVE::QemuConfig->write_config($vmid, $conf);
4655             }
4656         } else {
4657             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4658         }
4659
4660         return;
4661     }
4662
4663     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4664     die "vcpus in running vm does not match its configuration\n"
4665         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4666
4667     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4668
4669         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4670             my $cpustr = print_cpu_device($conf, $i);
4671             qemu_deviceadd($vmid, $cpustr);
4672
4673             my $retry = 0;
4674             my $currentrunningvcpus = undef;
4675             while (1) {
4676                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677                 last if scalar(@{$currentrunningvcpus}) == $i;
4678                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4679                 sleep 1;
4680                 $retry++;
4681             }
4682             #update conf after each succesfull cpu hotplug
4683             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684             PVE::QemuConfig->write_config($vmid, $conf);
4685         }
4686     } else {
4687
4688         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4689             mon_cmd($vmid, "cpu-add", id => int($i));
4690         }
4691     }
4692 }
4693
4694 sub qemu_block_set_io_throttle {
4695     my ($vmid, $deviceid,
4696         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4697         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4698         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4699         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4700
4701     return if !check_running($vmid) ;
4702
4703     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4704         bps => int($bps),
4705         bps_rd => int($bps_rd),
4706         bps_wr => int($bps_wr),
4707         iops => int($iops),
4708         iops_rd => int($iops_rd),
4709         iops_wr => int($iops_wr),
4710         bps_max => int($bps_max),
4711         bps_rd_max => int($bps_rd_max),
4712         bps_wr_max => int($bps_wr_max),
4713         iops_max => int($iops_max),
4714         iops_rd_max => int($iops_rd_max),
4715         iops_wr_max => int($iops_wr_max),
4716         bps_max_length => int($bps_max_length),
4717         bps_rd_max_length => int($bps_rd_max_length),
4718         bps_wr_max_length => int($bps_wr_max_length),
4719         iops_max_length => int($iops_max_length),
4720         iops_rd_max_length => int($iops_rd_max_length),
4721         iops_wr_max_length => int($iops_wr_max_length),
4722     );
4723
4724 }
4725
4726 sub qemu_block_resize {
4727     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4728
4729     my $running = check_running($vmid);
4730
4731     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4732
4733     return if !$running;
4734
4735     my $padding = (1024 - $size % 1024) % 1024;
4736     $size = $size + $padding;
4737
4738     mon_cmd(
4739         $vmid,
4740         "block_resize",
4741         device => $deviceid,
4742         size => int($size),
4743         timeout => 60,
4744     );
4745 }
4746
4747 sub qemu_volume_snapshot {
4748     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4749
4750     my $running = check_running($vmid);
4751
4752     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4753         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4754     } else {
4755         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4756     }
4757 }
4758
4759 sub qemu_volume_snapshot_delete {
4760     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4761
4762     my $running = check_running($vmid);
4763
4764     if($running) {
4765
4766         $running = undef;
4767         my $conf = PVE::QemuConfig->load_config($vmid);
4768         PVE::QemuConfig->foreach_volume($conf, sub {
4769             my ($ds, $drive) = @_;
4770             $running = 1 if $drive->{file} eq $volid;
4771         });
4772     }
4773
4774     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4775         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4776     } else {
4777         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4778     }
4779 }
4780
4781 sub set_migration_caps {
4782     my ($vmid, $savevm) = @_;
4783
4784     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4785
4786     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4787     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4788
4789     my $cap_ref = [];
4790
4791     my $enabled_cap = {
4792         "auto-converge" => 1,
4793         "xbzrle" => 1,
4794         "x-rdma-pin-all" => 0,
4795         "zero-blocks" => 0,
4796         "compress" => 0,
4797         "dirty-bitmaps" => $dirty_bitmaps,
4798     };
4799
4800     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4801
4802     for my $supported_capability (@$supported_capabilities) {
4803         push @$cap_ref, {
4804             capability => $supported_capability->{capability},
4805             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4806         };
4807     }
4808
4809     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4810 }
4811
4812 sub foreach_volid {
4813     my ($conf, $func, @param) = @_;
4814
4815     my $volhash = {};
4816
4817     my $test_volid = sub {
4818         my ($key, $drive, $snapname) = @_;
4819
4820         my $volid = $drive->{file};
4821         return if !$volid;
4822
4823         $volhash->{$volid}->{cdrom} //= 1;
4824         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4825
4826         my $replicate = $drive->{replicate} // 1;
4827         $volhash->{$volid}->{replicate} //= 0;
4828         $volhash->{$volid}->{replicate} = 1 if $replicate;
4829
4830         $volhash->{$volid}->{shared} //= 0;
4831         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4832
4833         $volhash->{$volid}->{referenced_in_config} //= 0;
4834         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4835
4836         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4837             if defined($snapname);
4838
4839         my $size = $drive->{size};
4840         $volhash->{$volid}->{size} //= $size if $size;
4841
4842         $volhash->{$volid}->{is_vmstate} //= 0;
4843         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4844
4845         $volhash->{$volid}->{is_tpmstate} //= 0;
4846         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4847
4848         $volhash->{$volid}->{is_unused} //= 0;
4849         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4850
4851         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4852     };
4853
4854     my $include_opts = {
4855         extra_keys => ['vmstate'],
4856         include_unused => 1,
4857     };
4858
4859     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4860     foreach my $snapname (keys %{$conf->{snapshots}}) {
4861         my $snap = $conf->{snapshots}->{$snapname};
4862         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4863     }
4864
4865     foreach my $volid (keys %$volhash) {
4866         &$func($volid, $volhash->{$volid}, @param);
4867     }
4868 }
4869
4870 my $fast_plug_option = {
4871     'lock' => 1,
4872     'name' => 1,
4873     'onboot' => 1,
4874     'shares' => 1,
4875     'startup' => 1,
4876     'description' => 1,
4877     'protection' => 1,
4878     'vmstatestorage' => 1,
4879     'hookscript' => 1,
4880     'tags' => 1,
4881 };
4882
4883 for my $opt (keys %$confdesc_cloudinit) {
4884     $fast_plug_option->{$opt} = 1;
4885 };
4886
4887 # hotplug changes in [PENDING]
4888 # $selection hash can be used to only apply specified options, for
4889 # example: { cores => 1 } (only apply changed 'cores')
4890 # $errors ref is used to return error messages
4891 sub vmconfig_hotplug_pending {
4892     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4893
4894     my $defaults = load_defaults();
4895     my $arch = get_vm_arch($conf);
4896     my $machine_type = get_vm_machine($conf, undef, $arch);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4908
4909     my $cloudinit_record_changed = sub {
4910         my ($conf, $opt, $old, $new) = @_;
4911         return if !$cloudinit_pending_properties->{$opt};
4912
4913         my $ci = ($conf->{cloudinit} //= {});
4914
4915         my $recorded = $ci->{$opt};
4916         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4917
4918         if (defined($new)) {
4919             if (defined($old)) {
4920                 # an existing value is being modified
4921                 if (defined($recorded)) {
4922                     # the value was already not in sync
4923                     if ($new eq $recorded) {
4924                         # a value is being reverted to the cloud-init state:
4925                         delete $ci->{$opt};
4926                         delete $added{$opt};
4927                     } else {
4928                         # the value was changed multiple times, do nothing
4929                     }
4930                 } elsif ($added{$opt}) {
4931                     # the value had been marked as added and is being changed, do nothing
4932                 } else {
4933                     # the value is new, record it:
4934                     $ci->{$opt} = $old;
4935                 }
4936             } else {
4937                 # a new value is being added
4938                 if (defined($recorded)) {
4939                     # it was already not in sync
4940                     if ($new eq $recorded) {
4941                         # a value is being reverted to the cloud-init state:
4942                         delete $ci->{$opt};
4943                         delete $added{$opt};
4944                     } else {
4945                         # the value had temporarily been removed, do nothing
4946                     }
4947                 } elsif ($added{$opt}) {
4948                     # the value had been marked as added already, do nothing
4949                 } else {
4950                     # the value is new, add it
4951                     $added{$opt} = 1;
4952                 }
4953             }
4954         } elsif (!defined($old)) {
4955             # a non-existent value is being removed? ignore...
4956         } else {
4957             # a value is being deleted
4958             if (defined($recorded)) {
4959                 # a value was already recorded, just keep it
4960             } elsif ($added{$opt}) {
4961                 # the value was marked as added, remove it
4962                 delete $added{$opt};
4963             } else {
4964                 # a previously unrecorded value is being removed, record the old value:
4965                 $ci->{$opt} = $old;
4966             }
4967         }
4968
4969         my $added = join(',', sort keys %added);
4970         $ci->{added} = $added if length($added);
4971     };
4972
4973     my $changes = 0;
4974     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4975         if ($fast_plug_option->{$opt}) {
4976             my $new = delete $conf->{pending}->{$opt};
4977             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4978             $conf->{$opt} = $new;
4979             $changes = 1;
4980         }
4981     }
4982
4983     if ($changes) {
4984         PVE::QemuConfig->write_config($vmid, $conf);
4985     }
4986
4987     my $ostype = $conf->{ostype};
4988     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4989     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4990     my $usb_hotplug = $hotplug_features->{usb}
4991         && min_version($version, 7, 1)
4992         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4993
4994     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4995     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4996
4997     foreach my $opt (sort keys %$pending_delete_hash) {
4998         next if $selection && !$selection->{$opt};
4999         my $force = $pending_delete_hash->{$opt}->{force};
5000         eval {
5001             if ($opt eq 'hotplug') {
5002                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5003             } elsif ($opt eq 'tablet') {
5004                 die "skip\n" if !$hotplug_features->{usb};
5005                 if ($defaults->{tablet}) {
5006                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5008                         if $arch eq 'aarch64';
5009                 } else {
5010                     vm_deviceunplug($vmid, $conf, 'tablet');
5011                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5012                 }
5013             } elsif ($opt =~ m/^usb(\d+)$/) {
5014                 my $index = $1;
5015                 die "skip\n" if !$usb_hotplug;
5016                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5017                 vm_deviceunplug($vmid, $conf, $opt);
5018             } elsif ($opt eq 'vcpus') {
5019                 die "skip\n" if !$hotplug_features->{cpu};
5020                 qemu_cpu_hotplug($vmid, $conf, undef);
5021             } elsif ($opt eq 'balloon') {
5022                 # enable balloon device is not hotpluggable
5023                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5024                 # here we reset the ballooning value to memory
5025                 my $balloon = $conf->{memory} || $defaults->{memory};
5026                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5027             } elsif ($fast_plug_option->{$opt}) {
5028                 # do nothing
5029             } elsif ($opt =~ m/^net(\d+)$/) {
5030                 die "skip\n" if !$hotplug_features->{network};
5031                 vm_deviceunplug($vmid, $conf, $opt);
5032             } elsif (is_valid_drivename($opt)) {
5033                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5034                 vm_deviceunplug($vmid, $conf, $opt);
5035                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5036             } elsif ($opt =~ m/^memory$/) {
5037                 die "skip\n" if !$hotplug_features->{memory};
5038                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5039             } elsif ($opt eq 'cpuunits') {
5040                 $cgroup->change_cpu_shares(undef);
5041             } elsif ($opt eq 'cpulimit') {
5042                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5043             } else {
5044                 die "skip\n";
5045             }
5046         };
5047         if (my $err = $@) {
5048             &$add_error($opt, $err) if $err ne "skip\n";
5049         } else {
5050             my $old = delete $conf->{$opt};
5051             $cloudinit_record_changed->($conf, $opt, $old, undef);
5052             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5053         }
5054     }
5055
5056     my $cloudinit_opt;
5057     foreach my $opt (keys %{$conf->{pending}}) {
5058         next if $selection && !$selection->{$opt};
5059         my $value = $conf->{pending}->{$opt};
5060         eval {
5061             if ($opt eq 'hotplug') {
5062                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5063             } elsif ($opt eq 'tablet') {
5064                 die "skip\n" if !$hotplug_features->{usb};
5065                 if ($value == 1) {
5066                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5067                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5068                         if $arch eq 'aarch64';
5069                 } elsif ($value == 0) {
5070                     vm_deviceunplug($vmid, $conf, 'tablet');
5071                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5072                 }
5073             } elsif ($opt =~ m/^usb(\d+)$/) {
5074                 my $index = $1;
5075                 die "skip\n" if !$usb_hotplug;
5076                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5077                 my $id = $opt;
5078                 if ($d->{host} eq 'spice')  {
5079                     $id = "usbredirdev$index";
5080                 }
5081                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5082             } elsif ($opt eq 'vcpus') {
5083                 die "skip\n" if !$hotplug_features->{cpu};
5084                 qemu_cpu_hotplug($vmid, $conf, $value);
5085             } elsif ($opt eq 'balloon') {
5086                 # enable/disable balloning device is not hotpluggable
5087                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5088                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5089                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5090
5091                 # allow manual ballooning if shares is set to zero
5092                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5093                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5094                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5095                 }
5096             } elsif ($opt =~ m/^net(\d+)$/) {
5097                 # some changes can be done without hotplug
5098                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5099                                     $vmid, $opt, $value, $arch, $machine_type);
5100             } elsif (is_valid_drivename($opt)) {
5101                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5102                 # some changes can be done without hotplug
5103                 my $drive = parse_drive($opt, $value);
5104                 if (drive_is_cloudinit($drive)) {
5105                     $cloudinit_opt = [$opt, $drive];
5106                     # apply all the other changes first, then generate the cloudinit disk
5107                     die "skip\n";
5108                 }
5109                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5110                                      $vmid, $opt, $value, $arch, $machine_type);
5111             } elsif ($opt =~ m/^memory$/) { #dimms
5112                 die "skip\n" if !$hotplug_features->{memory};
5113                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5114             } elsif ($opt eq 'cpuunits') {
5115                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5116                 $cgroup->change_cpu_shares($new_cpuunits);
5117             } elsif ($opt eq 'cpulimit') {
5118                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5119                 $cgroup->change_cpu_quota($cpulimit, 100000);
5120             } elsif ($opt eq 'agent') {
5121                 vmconfig_update_agent($conf, $opt, $value);
5122             } else {
5123                 die "skip\n";  # skip non-hot-pluggable options
5124             }
5125         };
5126         if (my $err = $@) {
5127             &$add_error($opt, $err) if $err ne "skip\n";
5128         } else {
5129             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5130             $conf->{$opt} = $value;
5131             delete $conf->{pending}->{$opt};
5132         }
5133     }
5134
5135     if (defined($cloudinit_opt)) {
5136         my ($opt, $drive) = @$cloudinit_opt;
5137         my $value = $conf->{pending}->{$opt};
5138         eval {
5139             my $temp = {%$conf, $opt => $value};
5140             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5141             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5142                                  $vmid, $opt, $value, $arch, $machine_type);
5143         };
5144         if (my $err = $@) {
5145             &$add_error($opt, $err) if $err ne "skip\n";
5146         } else {
5147             $conf->{$opt} = $value;
5148             delete $conf->{pending}->{$opt};
5149         }
5150     }
5151
5152     # unplug xhci controller if no usb device is left
5153     if ($usb_hotplug) {
5154         my $has_usb = 0;
5155         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5156             next if !defined($conf->{"usb$i"});
5157             $has_usb = 1;
5158             last;
5159         }
5160         if (!$has_usb) {
5161             vm_deviceunplug($vmid, $conf, 'xhci');
5162         }
5163     }
5164
5165     PVE::QemuConfig->write_config($vmid, $conf);
5166
5167     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5168         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5169     }
5170 }
5171
5172 sub try_deallocate_drive {
5173     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5174
5175     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5176         my $volid = $drive->{file};
5177         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5178             my $sid = PVE::Storage::parse_volume_id($volid);
5179             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5180
5181             # check if the disk is really unused
5182             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5183                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5184             PVE::Storage::vdisk_free($storecfg, $volid);
5185             return 1;
5186         } else {
5187             # If vm is not owner of this disk remove from config
5188             return 1;
5189         }
5190     }
5191
5192     return;
5193 }
5194
5195 sub vmconfig_delete_or_detach_drive {
5196     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5197
5198     my $drive = parse_drive($opt, $conf->{$opt});
5199
5200     my $rpcenv = PVE::RPCEnvironment::get();
5201     my $authuser = $rpcenv->get_user();
5202
5203     if ($force) {
5204         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5205         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5206     } else {
5207         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5208     }
5209 }
5210
5211
5212
5213 sub vmconfig_apply_pending {
5214     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5215
5216     return if !scalar(keys %{$conf->{pending}});
5217
5218     my $add_apply_error = sub {
5219         my ($opt, $msg) = @_;
5220         my $err_msg = "unable to apply pending change $opt : $msg";
5221         $errors->{$opt} = $err_msg;
5222         warn $err_msg;
5223     };
5224
5225     # cold plug
5226
5227     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5228     foreach my $opt (sort keys %$pending_delete_hash) {
5229         my $force = $pending_delete_hash->{$opt}->{force};
5230         eval {
5231             if ($opt =~ m/^unused/) {
5232                 die "internal error";
5233             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5234                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5235             }
5236         };
5237         if (my $err = $@) {
5238             $add_apply_error->($opt, $err);
5239         } else {
5240             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5241             delete $conf->{$opt};
5242         }
5243     }
5244
5245     PVE::QemuConfig->cleanup_pending($conf);
5246
5247     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5248
5249     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5250         next if $opt eq 'delete'; # just to be sure
5251         eval {
5252             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5253                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5254             }
5255         };
5256         if (my $err = $@) {
5257             $add_apply_error->($opt, $err);
5258         } else {
5259
5260             if (is_valid_drivename($opt)) {
5261                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5262                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5263             }
5264
5265             $conf->{$opt} = delete $conf->{pending}->{$opt};
5266         }
5267     }
5268
5269     # write all changes at once to avoid unnecessary i/o
5270     PVE::QemuConfig->write_config($vmid, $conf);
5271     if ($generate_cloudinit) {
5272         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5273             # After successful generation and if there were changes to be applied, update the
5274             # config to drop the {cloudinit} entry.
5275             PVE::QemuConfig->write_config($vmid, $conf);
5276         }
5277     }
5278 }
5279
5280 sub vmconfig_update_net {
5281     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5282
5283     my $newnet = parse_net($value);
5284
5285     if ($conf->{$opt}) {
5286         my $oldnet = parse_net($conf->{$opt});
5287
5288         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5289             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5290             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5291             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5292
5293             # for non online change, we try to hot-unplug
5294             die "skip\n" if !$hotplug;
5295             vm_deviceunplug($vmid, $conf, $opt);
5296         } else {
5297
5298             die "internal error" if $opt !~ m/net(\d+)/;
5299             my $iface = "tap${vmid}i$1";
5300
5301             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5302                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5303                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5304                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5305                 PVE::Network::tap_unplug($iface);
5306
5307                 if ($have_sdn) {
5308                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5309                 } else {
5310                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5311                 }
5312             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5313                 # Rate can be applied on its own but any change above needs to
5314                 # include the rate in tap_plug since OVS resets everything.
5315                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5316             }
5317
5318             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5319                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5320             }
5321
5322             return 1;
5323         }
5324     }
5325
5326     if ($hotplug) {
5327         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5328     } else {
5329         die "skip\n";
5330     }
5331 }
5332
5333 sub vmconfig_update_agent {
5334     my ($conf, $opt, $value) = @_;
5335
5336     die "skip\n" if !$conf->{$opt};
5337
5338     my $hotplug_options = { fstrim_cloned_disks => 1 };
5339
5340     my $old_agent = parse_guest_agent($conf);
5341     my $agent = parse_guest_agent({$opt => $value});
5342
5343     for my $option (keys %$agent) { # added/changed options
5344         next if defined($hotplug_options->{$option});
5345         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5346     }
5347
5348     for my $option (keys %$old_agent) { # removed options
5349         next if defined($hotplug_options->{$option});
5350         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5351     }
5352
5353     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5354 }
5355
5356 sub vmconfig_update_disk {
5357     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5358
5359     my $drive = parse_drive($opt, $value);
5360
5361     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5362         my $media = $drive->{media} || 'disk';
5363         my $oldmedia = $old_drive->{media} || 'disk';
5364         die "unable to change media type\n" if $media ne $oldmedia;
5365
5366         if (!drive_is_cdrom($old_drive)) {
5367
5368             if ($drive->{file} ne $old_drive->{file}) {
5369
5370                 die "skip\n" if !$hotplug;
5371
5372                 # unplug and register as unused
5373                 vm_deviceunplug($vmid, $conf, $opt);
5374                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5375
5376             } else {
5377                 # update existing disk
5378
5379                 # skip non hotpluggable value
5380                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5381                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5382                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5383                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5384                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5385                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5386                     die "skip\n";
5387                 }
5388
5389                 # apply throttle
5390                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5391                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5392                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5393                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5394                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5395                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5396                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5397                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5398                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5399                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5400                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5401                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5402                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5403                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5404                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5405                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5406                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5407                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5408
5409                     qemu_block_set_io_throttle(
5410                         $vmid,"drive-$opt",
5411                         ($drive->{mbps} || 0)*1024*1024,
5412                         ($drive->{mbps_rd} || 0)*1024*1024,
5413                         ($drive->{mbps_wr} || 0)*1024*1024,
5414                          $drive->{iops} || 0,
5415                          $drive->{iops_rd} || 0,
5416                          $drive->{iops_wr} || 0,
5417                         ($drive->{mbps_max} || 0)*1024*1024,
5418                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5419                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5420                          $drive->{iops_max} || 0,
5421                          $drive->{iops_rd_max} || 0,
5422                          $drive->{iops_wr_max} || 0,
5423                          $drive->{bps_max_length} || 1,
5424                          $drive->{bps_rd_max_length} || 1,
5425                          $drive->{bps_wr_max_length} || 1,
5426                          $drive->{iops_max_length} || 1,
5427                          $drive->{iops_rd_max_length} || 1,
5428                          $drive->{iops_wr_max_length} || 1,
5429                     );
5430
5431                 }
5432
5433                 return 1;
5434             }
5435
5436         } else { # cdrom
5437
5438             if ($drive->{file} eq 'none') {
5439                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5440                 if (drive_is_cloudinit($old_drive)) {
5441                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5442                 }
5443             } else {
5444                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5445
5446                 # force eject if locked
5447                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5448
5449                 if ($path) {
5450                     mon_cmd($vmid, "blockdev-change-medium",
5451                         id => "$opt", filename => "$path");
5452                 }
5453             }
5454
5455             return 1;
5456         }
5457     }
5458
5459     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5460     # hotplug new disks
5461     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5462     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5463 }
5464
5465 sub vmconfig_update_cloudinit_drive {
5466     my ($storecfg, $conf, $vmid) = @_;
5467
5468     my $cloudinit_ds = undef;
5469     my $cloudinit_drive = undef;
5470
5471     PVE::QemuConfig->foreach_volume($conf, sub {
5472         my ($ds, $drive) = @_;
5473         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5474             $cloudinit_ds = $ds;
5475             $cloudinit_drive = $drive;
5476         }
5477     });
5478
5479     return if !$cloudinit_drive;
5480
5481     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5482         PVE::QemuConfig->write_config($vmid, $conf);
5483     }
5484
5485     my $running = PVE::QemuServer::check_running($vmid);
5486
5487     if ($running) {
5488         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5489         if ($path) {
5490             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5491             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5492         }
5493     }
5494 }
5495
5496 # called in locked context by incoming migration
5497 sub vm_migrate_get_nbd_disks {
5498     my ($storecfg, $conf, $replicated_volumes) = @_;
5499
5500     my $local_volumes = {};
5501     PVE::QemuConfig->foreach_volume($conf, sub {
5502         my ($ds, $drive) = @_;
5503
5504         return if drive_is_cdrom($drive);
5505         return if $ds eq 'tpmstate0';
5506
5507         my $volid = $drive->{file};
5508
5509         return if !$volid;
5510
5511         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5512
5513         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5514         return if $scfg->{shared};
5515
5516         # replicated disks re-use existing state via bitmap
5517         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5518         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5519     });
5520     return $local_volumes;
5521 }
5522
5523 # called in locked context by incoming migration
5524 sub vm_migrate_alloc_nbd_disks {
5525     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5526
5527     my $nbd = {};
5528     foreach my $opt (sort keys %$source_volumes) {
5529         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5530
5531         if ($use_existing) {
5532             $nbd->{$opt}->{drivestr} = print_drive($drive);
5533             $nbd->{$opt}->{volid} = $volid;
5534             $nbd->{$opt}->{replicated} = 1;
5535             next;
5536         }
5537
5538         # storage mapping + volname = regular migration
5539         # storage mapping + format = remote migration
5540         # order of precedence, filtered by whether storage supports it:
5541         # 1. explicit requested format
5542         # 2. format of current volume
5543         # 3. default format of storage
5544         if (!$storagemap->{identity}) {
5545             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5546             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5547             if (!$format || !grep { $format eq $_ } @$validFormats) {
5548                 if ($volname) {
5549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5550                     my $fileFormat = qemu_img_format($scfg, $volname);
5551                     $format = $fileFormat
5552                         if grep { $fileFormat eq $_ } @$validFormats;
5553                 }
5554                 $format //= $defFormat;
5555             }
5556         } else {
5557             # can't happen for remote migration, so $volname is always defined
5558             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5559             $format = qemu_img_format($scfg, $volname);
5560         }
5561
5562         my $size = $drive->{size} / 1024;
5563         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5564         my $newdrive = $drive;
5565         $newdrive->{format} = $format;
5566         $newdrive->{file} = $newvolid;
5567         my $drivestr = print_drive($newdrive);
5568         $nbd->{$opt}->{drivestr} = $drivestr;
5569         $nbd->{$opt}->{volid} = $newvolid;
5570     }
5571
5572     return $nbd;
5573 }
5574
5575 # see vm_start_nolock for parameters, additionally:
5576 # migrate_opts:
5577 #   storagemap = parsed storage map for allocating NBD disks
5578 sub vm_start {
5579     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5580
5581     return PVE::QemuConfig->lock_config($vmid, sub {
5582         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5583
5584         die "you can't start a vm if it's a template\n"
5585             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5586
5587         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5588         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5589
5590         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5591
5592         if ($has_backup_lock && $running) {
5593             # a backup is currently running, attempt to start the guest in the
5594             # existing QEMU instance
5595             return vm_resume($vmid);
5596         }
5597
5598         PVE::QemuConfig->check_lock($conf)
5599             if !($params->{skiplock} || $has_suspended_lock);
5600
5601         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5602
5603         die "VM $vmid already running\n" if $running;
5604
5605         if (my $storagemap = $migrate_opts->{storagemap}) {
5606             my $replicated = $migrate_opts->{replicated_volumes};
5607             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5608             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5609
5610             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5611                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5612             }
5613         }
5614
5615         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5616     });
5617 }
5618
5619
5620 # params:
5621 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5622 #   skiplock => 0/1, skip checking for config lock
5623 #   skiptemplate => 0/1, skip checking whether VM is template
5624 #   forcemachine => to force Qemu machine (rollback/migration)
5625 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5626 #   timeout => in seconds
5627 #   paused => start VM in paused state (backup)
5628 #   resume => resume from hibernation
5629 #   pbs-backing => {
5630 #      sata0 => {
5631 #         repository
5632 #         snapshot
5633 #         keyfile
5634 #         archive
5635 #      },
5636 #      virtio2 => ...
5637 #   }
5638 # migrate_opts:
5639 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5640 #   migratedfrom => source node
5641 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5642 #   network => CIDR of migration network
5643 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5644 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5645 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5646 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5647 #       contained in config
5648 sub vm_start_nolock {
5649     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5650
5651     my $statefile = $params->{statefile};
5652     my $resume = $params->{resume};
5653
5654     my $migratedfrom = $migrate_opts->{migratedfrom};
5655     my $migration_type = $migrate_opts->{type};
5656
5657     my $res = {};
5658
5659     # clean up leftover reboot request files
5660     eval { clear_reboot_request($vmid); };
5661     warn $@ if $@;
5662
5663     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5664         vmconfig_apply_pending($vmid, $conf, $storecfg);
5665         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5666     }
5667
5668     # don't regenerate the ISO if the VM is started as part of a live migration
5669     # this way we can reuse the old ISO with the correct config
5670     if (!$migratedfrom) {
5671         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5672             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5673             # $conf->{cloudinit}, so we could just not do this?
5674             # But we do it above, so for now let's be consistent.
5675             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5676         }
5677     }
5678
5679     # override offline migrated volumes, conf is out of date still
5680     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5681         for my $key (sort keys $offline_volumes->%*) {
5682             my $parsed = parse_drive($key, $conf->{$key});
5683             $parsed->{file} = $offline_volumes->{$key};
5684             $conf->{$key} = print_drive($parsed);
5685         }
5686     }
5687
5688     my $defaults = load_defaults();
5689
5690     # set environment variable useful inside network script
5691     # for remote migration the config is available on the target node!
5692     if (!$migrate_opts->{remote_node}) {
5693         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5694     }
5695
5696     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5697
5698     my $forcemachine = $params->{forcemachine};
5699     my $forcecpu = $params->{forcecpu};
5700     if ($resume) {
5701         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5702         $forcemachine = $conf->{runningmachine};
5703         $forcecpu = $conf->{runningcpu};
5704         print "Resuming suspended VM\n";
5705     }
5706
5707     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5708         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5709
5710     my $migration_ip;
5711     my $get_migration_ip = sub {
5712         my ($nodename) = @_;
5713
5714         return $migration_ip if defined($migration_ip);
5715
5716         my $cidr = $migrate_opts->{network};
5717
5718         if (!defined($cidr)) {
5719             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5720             $cidr = $dc_conf->{migration}->{network};
5721         }
5722
5723         if (defined($cidr)) {
5724             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5725
5726             die "could not get IP: no address configured on local " .
5727                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5728
5729             die "could not get IP: multiple addresses configured on local " .
5730                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5731
5732             $migration_ip = @$ips[0];
5733         }
5734
5735         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5736             if !defined($migration_ip);
5737
5738         return $migration_ip;
5739     };
5740
5741     if ($statefile) {
5742         if ($statefile eq 'tcp') {
5743             my $migrate = $res->{migrate} = { proto => 'tcp' };
5744             $migrate->{addr} = "localhost";
5745             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5746             my $nodename = nodename();
5747
5748             if (!defined($migration_type)) {
5749                 if (defined($datacenterconf->{migration}->{type})) {
5750                     $migration_type = $datacenterconf->{migration}->{type};
5751                 } else {
5752                     $migration_type = 'secure';
5753                 }
5754             }
5755
5756             if ($migration_type eq 'insecure') {
5757                 $migrate->{addr} = $get_migration_ip->($nodename);
5758                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5759             }
5760
5761             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5762             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5763             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5764             push @$cmd, '-incoming', $migrate->{uri};
5765             push @$cmd, '-S';
5766
5767         } elsif ($statefile eq 'unix') {
5768             # should be default for secure migrations as a ssh TCP forward
5769             # tunnel is not deterministic reliable ready and fails regurarly
5770             # to set up in time, so use UNIX socket forwards
5771             my $migrate = $res->{migrate} = { proto => 'unix' };
5772             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5773             unlink $migrate->{addr};
5774
5775             $migrate->{uri} = "unix:$migrate->{addr}";
5776             push @$cmd, '-incoming', $migrate->{uri};
5777             push @$cmd, '-S';
5778
5779         } elsif (-e $statefile) {
5780             push @$cmd, '-loadstate', $statefile;
5781         } else {
5782             my $statepath = PVE::Storage::path($storecfg, $statefile);
5783             push @$vollist, $statefile;
5784             push @$cmd, '-loadstate', $statepath;
5785         }
5786     } elsif ($params->{paused}) {
5787         push @$cmd, '-S';
5788     }
5789
5790     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5791
5792     my $pci_devices = {}; # host pci devices
5793     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5794         my $dev = $conf->{"hostpci$i"} or next;
5795         $pci_devices->{$i} = parse_hostpci($dev);
5796     }
5797
5798     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5799     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5800
5801     # map to a flat list of pci ids
5802     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5803
5804     # reserve all PCI IDs before actually doing anything with them
5805     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5806
5807     eval {
5808         my $uuid;
5809         for my $id (sort keys %$pci_devices) {
5810             my $d = $pci_devices->{$id};
5811             for my $dev ($d->{pciid}->@*) {
5812                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5813
5814                 # nvidia grid needs the uuid of the mdev as qemu parameter
5815                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5816                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5817                 }
5818             }
5819         }
5820         push @$cmd, '-uuid', $uuid if defined($uuid);
5821     };
5822     if (my $err = $@) {
5823         eval { cleanup_pci_devices($vmid, $conf) };
5824         warn $@ if $@;
5825         die $err;
5826     }
5827
5828     PVE::Storage::activate_volumes($storecfg, $vollist);
5829
5830     eval {
5831         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5832     };
5833     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5834     # timeout should be more than enough here...
5835     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5836
5837     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5838
5839     my %run_params = (
5840         timeout => $statefile ? undef : $start_timeout,
5841         umask => 0077,
5842         noerr => 1,
5843     );
5844
5845     # when migrating, prefix QEMU output so other side can pick up any
5846     # errors that might occur and show the user
5847     if ($migratedfrom) {
5848         $run_params{quiet} = 1;
5849         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5850     }
5851
5852     my %systemd_properties = (
5853         Slice => 'qemu.slice',
5854         KillMode => 'process',
5855         SendSIGKILL => 0,
5856         TimeoutStopUSec => ULONG_MAX, # infinity
5857     );
5858
5859     if (PVE::CGroup::cgroup_mode() == 2) {
5860         $systemd_properties{CPUWeight} = $cpuunits;
5861     } else {
5862         $systemd_properties{CPUShares} = $cpuunits;
5863     }
5864
5865     if (my $cpulimit = $conf->{cpulimit}) {
5866         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5867     }
5868     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5869
5870     my $run_qemu = sub {
5871         PVE::Tools::run_fork sub {
5872             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5873
5874             my $tpmpid;
5875             if (my $tpm = $conf->{tpmstate0}) {
5876                 # start the TPM emulator so QEMU can connect on start
5877                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5878             }
5879
5880             my $exitcode = run_command($cmd, %run_params);
5881             if ($exitcode) {
5882                 if ($tpmpid) {
5883                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5884                     kill 'TERM', $tpmpid;
5885                 }
5886                 die "QEMU exited with code $exitcode\n";
5887             }
5888         };
5889     };
5890
5891     if ($conf->{hugepages}) {
5892
5893         my $code = sub {
5894             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5895             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5896
5897             PVE::QemuServer::Memory::hugepages_mount();
5898             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5899
5900             eval { $run_qemu->() };
5901             if (my $err = $@) {
5902                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5903                     if !$conf->{keephugepages};
5904                 die $err;
5905             }
5906
5907             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5908                 if !$conf->{keephugepages};
5909         };
5910         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5911
5912     } else {
5913         eval { $run_qemu->() };
5914     }
5915
5916     if (my $err = $@) {
5917         # deactivate volumes if start fails
5918         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5919         warn $@ if $@;
5920         eval { cleanup_pci_devices($vmid, $conf) };
5921         warn $@ if $@;
5922
5923         die "start failed: $err";
5924     }
5925
5926     # re-reserve all PCI IDs now that we can know the actual VM PID
5927     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5928     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5929     warn $@ if $@;
5930
5931     if (defined($res->{migrate})) {
5932         print "migration listens on $res->{migrate}->{uri}\n";
5933     } elsif ($statefile) {
5934         eval { mon_cmd($vmid, "cont"); };
5935         warn $@ if $@;
5936     }
5937
5938     #start nbd server for storage migration
5939     if (my $nbd = $migrate_opts->{nbd}) {
5940         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5941
5942         my $migrate_storage_uri;
5943         # nbd_protocol_version > 0 for unix socket support
5944         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5945             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5946             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5947             $migrate_storage_uri = "nbd:unix:$socket_path";
5948             $res->{migrate}->{unix_sockets} = [$socket_path];
5949         } else {
5950             my $nodename = nodename();
5951             my $localip = $get_migration_ip->($nodename);
5952             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5953             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5954
5955             mon_cmd($vmid, "nbd-server-start", addr => {
5956                 type => 'inet',
5957                 data => {
5958                     host => "${localip}",
5959                     port => "${storage_migrate_port}",
5960                 },
5961             });
5962             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5963             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5964         }
5965
5966         foreach my $opt (sort keys %$nbd) {
5967             my $drivestr = $nbd->{$opt}->{drivestr};
5968             my $volid = $nbd->{$opt}->{volid};
5969             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5970             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5971             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5972             print "re-using replicated volume: $opt - $volid\n"
5973                 if $nbd->{$opt}->{replicated};
5974
5975             $res->{drives}->{$opt} = $nbd->{$opt};
5976             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5977         }
5978     }
5979
5980     if ($migratedfrom) {
5981         eval {
5982             set_migration_caps($vmid);
5983         };
5984         warn $@ if $@;
5985
5986         if ($spice_port) {
5987             print "spice listens on port $spice_port\n";
5988             $res->{spice_port} = $spice_port;
5989             if ($migrate_opts->{spice_ticket}) {
5990                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5991                     $migrate_opts->{spice_ticket});
5992                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5993             }
5994         }
5995
5996     } else {
5997         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5998             if !$statefile && $conf->{balloon};
5999
6000         foreach my $opt (keys %$conf) {
6001             next if $opt !~  m/^net\d+$/;
6002             my $nicconf = parse_net($conf->{$opt});
6003             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6004         }
6005         add_nets_bridge_fdb($conf, $vmid);
6006     }
6007
6008     mon_cmd($vmid, 'qom-set',
6009                 path => "machine/peripheral/balloon0",
6010                 property => "guest-stats-polling-interval",
6011                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6012
6013     if ($resume) {
6014         print "Resumed VM, removing state\n";
6015         if (my $vmstate = $conf->{vmstate}) {
6016             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6017             PVE::Storage::vdisk_free($storecfg, $vmstate);
6018         }
6019         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6020         PVE::QemuConfig->write_config($vmid, $conf);
6021     }
6022
6023     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6024
6025     return $res;
6026 }
6027
6028 sub vm_commandline {
6029     my ($storecfg, $vmid, $snapname) = @_;
6030
6031     my $conf = PVE::QemuConfig->load_config($vmid);
6032
6033     my ($forcemachine, $forcecpu);
6034     if ($snapname) {
6035         my $snapshot = $conf->{snapshots}->{$snapname};
6036         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6037
6038         # check for machine or CPU overrides in snapshot
6039         $forcemachine = $snapshot->{runningmachine};
6040         $forcecpu = $snapshot->{runningcpu};
6041
6042         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6043
6044         $conf = $snapshot;
6045     }
6046
6047     my $defaults = load_defaults();
6048
6049     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6050
6051     return PVE::Tools::cmd2string($cmd);
6052 }
6053
6054 sub vm_reset {
6055     my ($vmid, $skiplock) = @_;
6056
6057     PVE::QemuConfig->lock_config($vmid, sub {
6058
6059         my $conf = PVE::QemuConfig->load_config($vmid);
6060
6061         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6062
6063         mon_cmd($vmid, "system_reset");
6064     });
6065 }
6066
6067 sub get_vm_volumes {
6068     my ($conf) = @_;
6069
6070     my $vollist = [];
6071     foreach_volid($conf, sub {
6072         my ($volid, $attr) = @_;
6073
6074         return if $volid =~ m|^/|;
6075
6076         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6077         return if !$sid;
6078
6079         push @$vollist, $volid;
6080     });
6081
6082     return $vollist;
6083 }
6084
6085 sub cleanup_pci_devices {
6086     my ($vmid, $conf) = @_;
6087
6088     foreach my $key (keys %$conf) {
6089         next if $key !~ m/^hostpci(\d+)$/;
6090         my $hostpciindex = $1;
6091         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6092         my $d = parse_hostpci($conf->{$key});
6093         if ($d->{mdev}) {
6094             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6095             # don't want to break ABI just for this two liner
6096             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6097             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6098         }
6099     }
6100     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6101 }
6102
6103 sub vm_stop_cleanup {
6104     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6105
6106     eval {
6107
6108         if (!$keepActive) {
6109             my $vollist = get_vm_volumes($conf);
6110             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6111
6112             if (my $tpmdrive = $conf->{tpmstate0}) {
6113                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6114                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6115                 if ($storeid) {
6116                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6117                 }
6118             }
6119         }
6120
6121         foreach my $ext (qw(mon qmp pid vnc qga)) {
6122             unlink "/var/run/qemu-server/${vmid}.$ext";
6123         }
6124
6125         if ($conf->{ivshmem}) {
6126             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6127             # just delete it for now, VMs which have this already open do not
6128             # are affected, but new VMs will get a separated one. If this
6129             # becomes an issue we either add some sort of ref-counting or just
6130             # add a "don't delete on stop" flag to the ivshmem format.
6131             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6132         }
6133
6134         cleanup_pci_devices($vmid, $conf);
6135
6136         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6137     };
6138     warn $@ if $@; # avoid errors - just warn
6139 }
6140
6141 # call only in locked context
6142 sub _do_vm_stop {
6143     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6144
6145     my $pid = check_running($vmid, $nocheck);
6146     return if !$pid;
6147
6148     my $conf;
6149     if (!$nocheck) {
6150         $conf = PVE::QemuConfig->load_config($vmid);
6151         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6152         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6153             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6154             $timeout = $opts->{down} if $opts->{down};
6155         }
6156         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6157     }
6158
6159     eval {
6160         if ($shutdown) {
6161             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6162                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6163             } else {
6164                 mon_cmd($vmid, "system_powerdown");
6165             }
6166         } else {
6167             mon_cmd($vmid, "quit");
6168         }
6169     };
6170     my $err = $@;
6171
6172     if (!$err) {
6173         $timeout = 60 if !defined($timeout);
6174
6175         my $count = 0;
6176         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6177             $count++;
6178             sleep 1;
6179         }
6180
6181         if ($count >= $timeout) {
6182             if ($force) {
6183                 warn "VM still running - terminating now with SIGTERM\n";
6184                 kill 15, $pid;
6185             } else {
6186                 die "VM quit/powerdown failed - got timeout\n";
6187             }
6188         } else {
6189             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6190             return;
6191         }
6192     } else {
6193        if (!check_running($vmid, $nocheck)) {
6194             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6195             return;
6196        }
6197        if ($force) {
6198             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6199             kill 15, $pid;
6200         } else {
6201             die "VM quit/powerdown failed\n";
6202         }
6203     }
6204
6205     # wait again
6206     $timeout = 10;
6207
6208     my $count = 0;
6209     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6210         $count++;
6211         sleep 1;
6212     }
6213
6214     if ($count >= $timeout) {
6215         warn "VM still running - terminating now with SIGKILL\n";
6216         kill 9, $pid;
6217         sleep 1;
6218     }
6219
6220     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6221 }
6222
6223 # Note: use $nocheck to skip tests if VM configuration file exists.
6224 # We need that when migration VMs to other nodes (files already moved)
6225 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6226 sub vm_stop {
6227     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6228
6229     $force = 1 if !defined($force) && !$shutdown;
6230
6231     if ($migratedfrom){
6232         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6233         kill 15, $pid if $pid;
6234         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6235         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6236         return;
6237     }
6238
6239     PVE::QemuConfig->lock_config($vmid, sub {
6240         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6241    });
6242 }
6243
6244 sub vm_reboot {
6245     my ($vmid, $timeout) = @_;
6246
6247     PVE::QemuConfig->lock_config($vmid, sub {
6248         eval {
6249
6250             # only reboot if running, as qmeventd starts it again on a stop event
6251             return if !check_running($vmid);
6252
6253             create_reboot_request($vmid);
6254
6255             my $storecfg = PVE::Storage::config();
6256             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6257
6258         };
6259         if (my $err = $@) {
6260             # avoid that the next normal shutdown will be confused for a reboot
6261             clear_reboot_request($vmid);
6262             die $err;
6263         }
6264    });
6265 }
6266
6267 # note: if using the statestorage parameter, the caller has to check privileges
6268 sub vm_suspend {
6269     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6270
6271     my $conf;
6272     my $path;
6273     my $storecfg;
6274     my $vmstate;
6275
6276     PVE::QemuConfig->lock_config($vmid, sub {
6277
6278         $conf = PVE::QemuConfig->load_config($vmid);
6279
6280         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6281         PVE::QemuConfig->check_lock($conf)
6282             if !($skiplock || $is_backing_up);
6283
6284         die "cannot suspend to disk during backup\n"
6285             if $is_backing_up && $includestate;
6286
6287         if ($includestate) {
6288             $conf->{lock} = 'suspending';
6289             my $date = strftime("%Y-%m-%d", localtime(time()));
6290             $storecfg = PVE::Storage::config();
6291             if (!$statestorage) {
6292                 $statestorage = find_vmstate_storage($conf, $storecfg);
6293                 # check permissions for the storage
6294                 my $rpcenv = PVE::RPCEnvironment::get();
6295                 if ($rpcenv->{type} ne 'cli') {
6296                     my $authuser = $rpcenv->get_user();
6297                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6298                 }
6299             }
6300
6301
6302             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6303                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6304             $path = PVE::Storage::path($storecfg, $vmstate);
6305             PVE::QemuConfig->write_config($vmid, $conf);
6306         } else {
6307             mon_cmd($vmid, "stop");
6308         }
6309     });
6310
6311     if ($includestate) {
6312         # save vm state
6313         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6314
6315         eval {
6316             set_migration_caps($vmid, 1);
6317             mon_cmd($vmid, "savevm-start", statefile => $path);
6318             for(;;) {
6319                 my $state = mon_cmd($vmid, "query-savevm");
6320                 if (!$state->{status}) {
6321                     die "savevm not active\n";
6322                 } elsif ($state->{status} eq 'active') {
6323                     sleep(1);
6324                     next;
6325                 } elsif ($state->{status} eq 'completed') {
6326                     print "State saved, quitting\n";
6327                     last;
6328                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6329                     die "query-savevm failed with error '$state->{error}'\n"
6330                 } else {
6331                     die "query-savevm returned status '$state->{status}'\n";
6332                 }
6333             }
6334         };
6335         my $err = $@;
6336
6337         PVE::QemuConfig->lock_config($vmid, sub {
6338             $conf = PVE::QemuConfig->load_config($vmid);
6339             if ($err) {
6340                 # cleanup, but leave suspending lock, to indicate something went wrong
6341                 eval {
6342                     mon_cmd($vmid, "savevm-end");
6343                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6344                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6345                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6346                     PVE::QemuConfig->write_config($vmid, $conf);
6347                 };
6348                 warn $@ if $@;
6349                 die $err;
6350             }
6351
6352             die "lock changed unexpectedly\n"
6353                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6354
6355             mon_cmd($vmid, "quit");
6356             $conf->{lock} = 'suspended';
6357             PVE::QemuConfig->write_config($vmid, $conf);
6358         });
6359     }
6360 }
6361
6362 sub vm_resume {
6363     my ($vmid, $skiplock, $nocheck) = @_;
6364
6365     PVE::QemuConfig->lock_config($vmid, sub {
6366         my $res = mon_cmd($vmid, 'query-status');
6367         my $resume_cmd = 'cont';
6368         my $reset = 0;
6369         my $conf = PVE::QemuConfig->load_config($vmid);
6370
6371         if ($res->{status}) {
6372             return if $res->{status} eq 'running'; # job done, go home
6373             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6374             $reset = 1 if $res->{status} eq 'shutdown';
6375         }
6376
6377         if (!$nocheck) {
6378
6379             PVE::QemuConfig->check_lock($conf)
6380                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6381         }
6382
6383         if ($reset) {
6384             # required if a VM shuts down during a backup and we get a resume
6385             # request before the backup finishes for example
6386             mon_cmd($vmid, "system_reset");
6387         }
6388
6389         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6390
6391         mon_cmd($vmid, $resume_cmd);
6392     });
6393 }
6394
6395 sub vm_sendkey {
6396     my ($vmid, $skiplock, $key) = @_;
6397
6398     PVE::QemuConfig->lock_config($vmid, sub {
6399
6400         my $conf = PVE::QemuConfig->load_config($vmid);
6401
6402         # there is no qmp command, so we use the human monitor command
6403         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6404         die $res if $res ne '';
6405     });
6406 }
6407
6408 # vzdump restore implementaion
6409
6410 sub tar_archive_read_firstfile {
6411     my $archive = shift;
6412
6413     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6414
6415     # try to detect archive type first
6416     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6417         die "unable to open file '$archive'\n";
6418     my $firstfile = <$fh>;
6419     kill 15, $pid;
6420     close $fh;
6421
6422     die "ERROR: archive contaions no data\n" if !$firstfile;
6423     chomp $firstfile;
6424
6425     return $firstfile;
6426 }
6427
6428 sub tar_restore_cleanup {
6429     my ($storecfg, $statfile) = @_;
6430
6431     print STDERR "starting cleanup\n";
6432
6433     if (my $fd = IO::File->new($statfile, "r")) {
6434         while (defined(my $line = <$fd>)) {
6435             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6436                 my $volid = $2;
6437                 eval {
6438                     if ($volid =~ m|^/|) {
6439                         unlink $volid || die 'unlink failed\n';
6440                     } else {
6441                         PVE::Storage::vdisk_free($storecfg, $volid);
6442                     }
6443                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6444                 };
6445                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6446             } else {
6447                 print STDERR "unable to parse line in statfile - $line";
6448             }
6449         }
6450         $fd->close();
6451     }
6452 }
6453
6454 sub restore_file_archive {
6455     my ($archive, $vmid, $user, $opts) = @_;
6456
6457     return restore_vma_archive($archive, $vmid, $user, $opts)
6458         if $archive eq '-';
6459
6460     my $info = PVE::Storage::archive_info($archive);
6461     my $format = $opts->{format} // $info->{format};
6462     my $comp = $info->{compression};
6463
6464     # try to detect archive format
6465     if ($format eq 'tar') {
6466         return restore_tar_archive($archive, $vmid, $user, $opts);
6467     } else {
6468         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6469     }
6470 }
6471
6472 # hepler to remove disks that will not be used after restore
6473 my $restore_cleanup_oldconf = sub {
6474     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6475
6476     my $kept_disks = {};
6477
6478     PVE::QemuConfig->foreach_volume($oldconf, sub {
6479         my ($ds, $drive) = @_;
6480
6481         return if drive_is_cdrom($drive, 1);
6482
6483         my $volid = $drive->{file};
6484         return if !$volid || $volid =~ m|^/|;
6485
6486         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6487         return if !$path || !$owner || ($owner != $vmid);
6488
6489         # Note: only delete disk we want to restore
6490         # other volumes will become unused
6491         if ($virtdev_hash->{$ds}) {
6492             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6493             if (my $err = $@) {
6494                 warn $err;
6495             }
6496         } else {
6497             $kept_disks->{$volid} = 1;
6498         }
6499     });
6500
6501     # after the restore we have no snapshots anymore
6502     for my $snapname (keys $oldconf->{snapshots}->%*) {
6503         my $snap = $oldconf->{snapshots}->{$snapname};
6504         if ($snap->{vmstate}) {
6505             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6506             if (my $err = $@) {
6507                 warn $err;
6508             }
6509         }
6510
6511         for my $volid (keys $kept_disks->%*) {
6512             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6513             warn $@ if $@;
6514         }
6515     }
6516 };
6517
6518 # Helper to parse vzdump backup device hints
6519 #
6520 # $rpcenv: Environment, used to ckeck storage permissions
6521 # $user: User ID, to check storage permissions
6522 # $storecfg: Storage configuration
6523 # $fh: the file handle for reading the configuration
6524 # $devinfo: should contain device sizes for all backu-up'ed devices
6525 # $options: backup options (pool, default storage)
6526 #
6527 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6528 my $parse_backup_hints = sub {
6529     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6530
6531     my $check_storage = sub { # assert if an image can be allocate
6532         my ($storeid, $scfg) = @_;
6533         die "Content type 'images' is not available on storage '$storeid'\n"
6534             if !$scfg->{content}->{images};
6535         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6536             if $user ne 'root@pam';
6537     };
6538
6539     my $virtdev_hash = {};
6540     while (defined(my $line = <$fh>)) {
6541         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6542             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6543             die "archive does not contain data for drive '$virtdev'\n"
6544                 if !$devinfo->{$devname};
6545
6546             if (defined($options->{storage})) {
6547                 $storeid = $options->{storage} || 'local';
6548             } elsif (!$storeid) {
6549                 $storeid = 'local';
6550             }
6551             $format = 'raw' if !$format;
6552             $devinfo->{$devname}->{devname} = $devname;
6553             $devinfo->{$devname}->{virtdev} = $virtdev;
6554             $devinfo->{$devname}->{format} = $format;
6555             $devinfo->{$devname}->{storeid} = $storeid;
6556
6557             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6558             $check_storage->($storeid, $scfg); # permission and content type check
6559
6560             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6561         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6562             my $virtdev = $1;
6563             my $drive = parse_drive($virtdev, $2);
6564
6565             if (drive_is_cloudinit($drive)) {
6566                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6567                 $storeid = $options->{storage} if defined ($options->{storage});
6568                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6569                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6570
6571                 $check_storage->($storeid, $scfg); # permission and content type check
6572
6573                 $virtdev_hash->{$virtdev} = {
6574                     format => $format,
6575                     storeid => $storeid,
6576                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6577                     is_cloudinit => 1,
6578                 };
6579             }
6580         }
6581     }
6582
6583     return $virtdev_hash;
6584 };
6585
6586 # Helper to allocate and activate all volumes required for a restore
6587 #
6588 # $storecfg: Storage configuration
6589 # $virtdev_hash: as returned by parse_backup_hints()
6590 #
6591 # Returns: { $virtdev => $volid }
6592 my $restore_allocate_devices = sub {
6593     my ($storecfg, $virtdev_hash, $vmid) = @_;
6594
6595     my $map = {};
6596     foreach my $virtdev (sort keys %$virtdev_hash) {
6597         my $d = $virtdev_hash->{$virtdev};
6598         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6599         my $storeid = $d->{storeid};
6600         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6601
6602         # test if requested format is supported
6603         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6604         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6605         $d->{format} = $defFormat if !$supported;
6606
6607         my $name;
6608         if ($d->{is_cloudinit}) {
6609             $name = "vm-$vmid-cloudinit";
6610             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6611             if ($scfg->{path}) {
6612                 $name .= ".$d->{format}";
6613             }
6614         }
6615
6616         my $volid = PVE::Storage::vdisk_alloc(
6617             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6618
6619         print STDERR "new volume ID is '$volid'\n";
6620         $d->{volid} = $volid;
6621
6622         PVE::Storage::activate_volumes($storecfg, [$volid]);
6623
6624         $map->{$virtdev} = $volid;
6625     }
6626
6627     return $map;
6628 };
6629
6630 sub restore_update_config_line {
6631     my ($cookie, $map, $line, $unique) = @_;
6632
6633     return '' if $line =~ m/^\#qmdump\#/;
6634     return '' if $line =~ m/^\#vzdump\#/;
6635     return '' if $line =~ m/^lock:/;
6636     return '' if $line =~ m/^unused\d+:/;
6637     return '' if $line =~ m/^parent:/;
6638
6639     my $res = '';
6640
6641     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6642     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6643         # try to convert old 1.X settings
6644         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6645         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6646             my ($model, $macaddr) = split(/\=/, $devconfig);
6647             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6648             my $net = {
6649                 model => $model,
6650                 bridge => "vmbr$ind",
6651                 macaddr => $macaddr,
6652             };
6653             my $netstr = print_net($net);
6654
6655             $res .= "net$cookie->{netcount}: $netstr\n";
6656             $cookie->{netcount}++;
6657         }
6658     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6659         my ($id, $netstr) = ($1, $2);
6660         my $net = parse_net($netstr);
6661         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6662         $netstr = print_net($net);
6663         $res .= "$id: $netstr\n";
6664     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6665         my $virtdev = $1;
6666         my $value = $3;
6667         my $di = parse_drive($virtdev, $value);
6668         if (defined($di->{backup}) && !$di->{backup}) {
6669             $res .= "#$line";
6670         } elsif ($map->{$virtdev}) {
6671             delete $di->{format}; # format can change on restore
6672             $di->{file} = $map->{$virtdev};
6673             $value = print_drive($di);
6674             $res .= "$virtdev: $value\n";
6675         } else {
6676             $res .= $line;
6677         }
6678     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6679         my $vmgenid = $1;
6680         if ($vmgenid ne '0') {
6681             # always generate a new vmgenid if there was a valid one setup
6682             $vmgenid = generate_uuid();
6683         }
6684         $res .= "vmgenid: $vmgenid\n";
6685     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6686         my ($uuid, $uuid_str);
6687         UUID::generate($uuid);
6688         UUID::unparse($uuid, $uuid_str);
6689         my $smbios1 = parse_smbios1($2);
6690         $smbios1->{uuid} = $uuid_str;
6691         $res .= $1.print_smbios1($smbios1)."\n";
6692     } else {
6693         $res .= $line;
6694     }
6695
6696     return $res;
6697 }
6698
6699 my $restore_deactivate_volumes = sub {
6700     my ($storecfg, $virtdev_hash) = @_;
6701
6702     my $vollist = [];
6703     for my $dev (values $virtdev_hash->%*) {
6704         push $vollist->@*, $dev->{volid} if $dev->{volid};
6705     }
6706
6707     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6708     print STDERR $@ if $@;
6709 };
6710
6711 my $restore_destroy_volumes = sub {
6712     my ($storecfg, $virtdev_hash) = @_;
6713
6714     for my $dev (values $virtdev_hash->%*) {
6715         my $volid = $dev->{volid} or next;
6716         eval {
6717             PVE::Storage::vdisk_free($storecfg, $volid);
6718             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6719         };
6720         print STDERR "unable to cleanup '$volid' - $@" if $@;
6721     }
6722 };
6723
6724 my $restore_merge_config = sub {
6725     my ($filename, $backup_conf_raw, $override_conf) = @_;
6726
6727     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6728     for my $key (keys $override_conf->%*) {
6729         $backup_conf->{$key} = $override_conf->{$key};
6730     }
6731
6732     return $backup_conf;
6733 };
6734
6735 sub scan_volids {
6736     my ($cfg, $vmid) = @_;
6737
6738     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6739
6740     my $volid_hash = {};
6741     foreach my $storeid (keys %$info) {
6742         foreach my $item (@{$info->{$storeid}}) {
6743             next if !($item->{volid} && $item->{size});
6744             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6745             $volid_hash->{$item->{volid}} = $item;
6746         }
6747     }
6748
6749     return $volid_hash;
6750 }
6751
6752 sub update_disk_config {
6753     my ($vmid, $conf, $volid_hash) = @_;
6754
6755     my $changes;
6756     my $prefix = "VM $vmid";
6757
6758     # used and unused disks
6759     my $referenced = {};
6760
6761     # Note: it is allowed to define multiple storages with same path (alias), so
6762     # we need to check both 'volid' and real 'path' (two different volid can point
6763     # to the same path).
6764
6765     my $referencedpath = {};
6766
6767     # update size info
6768     PVE::QemuConfig->foreach_volume($conf, sub {
6769         my ($opt, $drive) = @_;
6770
6771         my $volid = $drive->{file};
6772         return if !$volid;
6773         my $volume = $volid_hash->{$volid};
6774
6775         # mark volid as "in-use" for next step
6776         $referenced->{$volid} = 1;
6777         if ($volume && (my $path = $volume->{path})) {
6778             $referencedpath->{$path} = 1;
6779         }
6780
6781         return if drive_is_cdrom($drive);
6782         return if !$volume;
6783
6784         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6785         if (defined($updated)) {
6786             $changes = 1;
6787             $conf->{$opt} = print_drive($updated);
6788             print "$prefix ($opt): $msg\n";
6789         }
6790     });
6791
6792     # remove 'unusedX' entry if volume is used
6793     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6794         my ($opt, $drive) = @_;
6795
6796         my $volid = $drive->{file};
6797         return if !$volid;
6798
6799         my $path;
6800         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6801         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6802             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6803             $changes = 1;
6804             delete $conf->{$opt};
6805         }
6806
6807         $referenced->{$volid} = 1;
6808         $referencedpath->{$path} = 1 if $path;
6809     });
6810
6811     foreach my $volid (sort keys %$volid_hash) {
6812         next if $volid =~ m/vm-$vmid-state-/;
6813         next if $referenced->{$volid};
6814         my $path = $volid_hash->{$volid}->{path};
6815         next if !$path; # just to be sure
6816         next if $referencedpath->{$path};
6817         $changes = 1;
6818         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6819         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6820         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6821     }
6822
6823     return $changes;
6824 }
6825
6826 sub rescan {
6827     my ($vmid, $nolock, $dryrun) = @_;
6828
6829     my $cfg = PVE::Storage::config();
6830
6831     print "rescan volumes...\n";
6832     my $volid_hash = scan_volids($cfg, $vmid);
6833
6834     my $updatefn =  sub {
6835         my ($vmid) = @_;
6836
6837         my $conf = PVE::QemuConfig->load_config($vmid);
6838
6839         PVE::QemuConfig->check_lock($conf);
6840
6841         my $vm_volids = {};
6842         foreach my $volid (keys %$volid_hash) {
6843             my $info = $volid_hash->{$volid};
6844             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6845         }
6846
6847         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6848
6849         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6850     };
6851
6852     if (defined($vmid)) {
6853         if ($nolock) {
6854             &$updatefn($vmid);
6855         } else {
6856             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6857         }
6858     } else {
6859         my $vmlist = config_list();
6860         foreach my $vmid (keys %$vmlist) {
6861             if ($nolock) {
6862                 &$updatefn($vmid);
6863             } else {
6864                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6865             }
6866         }
6867     }
6868 }
6869
6870 sub restore_proxmox_backup_archive {
6871     my ($archive, $vmid, $user, $options) = @_;
6872
6873     my $storecfg = PVE::Storage::config();
6874
6875     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6876     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6877
6878     my $fingerprint = $scfg->{fingerprint};
6879     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6880
6881     my $repo = PVE::PBSClient::get_repository($scfg);
6882     my $namespace = $scfg->{namespace};
6883
6884     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6885     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6886     local $ENV{PBS_PASSWORD} = $password;
6887     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6888
6889     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6890         PVE::Storage::parse_volname($storecfg, $archive);
6891
6892     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6893
6894     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6895
6896     my $tmpdir = "/var/tmp/vzdumptmp$$";
6897     rmtree $tmpdir;
6898     mkpath $tmpdir;
6899
6900     my $conffile = PVE::QemuConfig->config_file($vmid);
6901      # disable interrupts (always do cleanups)
6902     local $SIG{INT} =
6903         local $SIG{TERM} =
6904         local $SIG{QUIT} =
6905         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6906
6907     # Note: $oldconf is undef if VM does not exists
6908     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6909     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6910     my $new_conf_raw = '';
6911
6912     my $rpcenv = PVE::RPCEnvironment::get();
6913     my $devinfo = {}; # info about drives included in backup
6914     my $virtdev_hash = {}; # info about allocated drives
6915
6916     eval {
6917         # enable interrupts
6918         local $SIG{INT} =
6919             local $SIG{TERM} =
6920             local $SIG{QUIT} =
6921             local $SIG{HUP} =
6922             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6923
6924         my $cfgfn = "$tmpdir/qemu-server.conf";
6925         my $firewall_config_fn = "$tmpdir/fw.conf";
6926         my $index_fn = "$tmpdir/index.json";
6927
6928         my $cmd = "restore";
6929
6930         my $param = [$pbs_backup_name, "index.json", $index_fn];
6931         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6932         my $index = PVE::Tools::file_get_contents($index_fn);
6933         $index = decode_json($index);
6934
6935         foreach my $info (@{$index->{files}}) {
6936             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6937                 my $devname = $1;
6938                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6939                     $devinfo->{$devname}->{size} = $1;
6940                 } else {
6941                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6942                 }
6943             }
6944         }
6945
6946         my $is_qemu_server_backup = scalar(
6947             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6948         );
6949         if (!$is_qemu_server_backup) {
6950             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6951         }
6952         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6953
6954         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6955         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6956
6957         if ($has_firewall_config) {
6958             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6959             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6960
6961             my $pve_firewall_dir = '/etc/pve/firewall';
6962             mkdir $pve_firewall_dir; # make sure the dir exists
6963             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6964         }
6965
6966         my $fh = IO::File->new($cfgfn, "r") ||
6967             die "unable to read qemu-server.conf - $!\n";
6968
6969         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6970
6971         # fixme: rate limit?
6972
6973         # create empty/temp config
6974         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6975
6976         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6977
6978         # allocate volumes
6979         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6980
6981         foreach my $virtdev (sort keys %$virtdev_hash) {
6982             my $d = $virtdev_hash->{$virtdev};
6983             next if $d->{is_cloudinit}; # no need to restore cloudinit
6984
6985             # this fails if storage is unavailable
6986             my $volid = $d->{volid};
6987             my $path = PVE::Storage::path($storecfg, $volid);
6988
6989             # for live-restore we only want to preload the efidisk and TPM state
6990             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6991
6992             my @ns_arg;
6993             if (defined(my $ns = $scfg->{namespace})) {
6994                 @ns_arg = ('--ns', $ns);
6995             }
6996
6997             my $pbs_restore_cmd = [
6998                 '/usr/bin/pbs-restore',
6999                 '--repository', $repo,
7000                 @ns_arg,
7001                 $pbs_backup_name,
7002                 "$d->{devname}.img.fidx",
7003                 $path,
7004                 '--verbose',
7005                 ];
7006
7007             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7008             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7009
7010             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7011                 push @$pbs_restore_cmd, '--skip-zero';
7012             }
7013
7014             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7015             print "restore proxmox backup image: $dbg_cmdstring\n";
7016             run_command($pbs_restore_cmd);
7017         }
7018
7019         $fh->seek(0, 0) || die "seek failed - $!\n";
7020
7021         my $cookie = { netcount => 0 };
7022         while (defined(my $line = <$fh>)) {
7023             $new_conf_raw .= restore_update_config_line(
7024                 $cookie,
7025                 $map,
7026                 $line,
7027                 $options->{unique},
7028             );
7029         }
7030
7031         $fh->close();
7032     };
7033     my $err = $@;
7034
7035     if ($err || !$options->{live}) {
7036         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7037     }
7038
7039     rmtree $tmpdir;
7040
7041     if ($err) {
7042         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7043         die $err;
7044     }
7045
7046     if ($options->{live}) {
7047         # keep lock during live-restore
7048         $new_conf_raw .= "\nlock: create";
7049     }
7050
7051     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7052     PVE::QemuConfig->write_config($vmid, $new_conf);
7053
7054     eval { rescan($vmid, 1); };
7055     warn $@ if $@;
7056
7057     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7058
7059     if ($options->{live}) {
7060         # enable interrupts
7061         local $SIG{INT} =
7062             local $SIG{TERM} =
7063             local $SIG{QUIT} =
7064             local $SIG{HUP} =
7065             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7066
7067         my $conf = PVE::QemuConfig->load_config($vmid);
7068         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7069
7070         # these special drives are already restored before start
7071         delete $devinfo->{'drive-efidisk0'};
7072         delete $devinfo->{'drive-tpmstate0-backup'};
7073
7074         my $pbs_opts = {
7075             repo => $repo,
7076             keyfile => $keyfile,
7077             snapshot => $pbs_backup_name,
7078             namespace => $namespace,
7079         };
7080         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7081
7082         PVE::QemuConfig->remove_lock($vmid, "create");
7083     }
7084 }
7085
7086 sub pbs_live_restore {
7087     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7088
7089     print "starting VM for live-restore\n";
7090     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7091
7092     my $pbs_backing = {};
7093     for my $ds (keys %$restored_disks) {
7094         $ds =~ m/^drive-(.*)$/;
7095         my $confname = $1;
7096         $pbs_backing->{$confname} = {
7097             repository => $opts->{repo},
7098             snapshot => $opts->{snapshot},
7099             archive => "$ds.img.fidx",
7100         };
7101         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7102         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7103
7104         my $drive = parse_drive($confname, $conf->{$confname});
7105         print "restoring '$ds' to '$drive->{file}'\n";
7106     }
7107
7108     my $drives_streamed = 0;
7109     eval {
7110         # make sure HA doesn't interrupt our restore by stopping the VM
7111         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7112             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7113         }
7114
7115         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7116         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7117         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7118
7119         my $qmeventd_fd = register_qmeventd_handle($vmid);
7120
7121         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7122         # this will effectively collapse the backing image chain consisting of
7123         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7124         # removes itself once all backing images vanish with 'auto-remove=on')
7125         my $jobs = {};
7126         for my $ds (sort keys %$restored_disks) {
7127             my $job_id = "restore-$ds";
7128             mon_cmd($vmid, 'block-stream',
7129                 'job-id' => $job_id,
7130                 device => "$ds",
7131             );
7132             $jobs->{$job_id} = {};
7133         }
7134
7135         mon_cmd($vmid, 'cont');
7136         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7137
7138         print "restore-drive jobs finished successfully, removing all tracking block devices"
7139             ." to disconnect from Proxmox Backup Server\n";
7140
7141         for my $ds (sort keys %$restored_disks) {
7142             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7143         }
7144
7145         close($qmeventd_fd);
7146     };
7147
7148     my $err = $@;
7149
7150     if ($err) {
7151         warn "An error occurred during live-restore: $err\n";
7152         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7153         die "live-restore failed\n";
7154     }
7155 }
7156
7157 sub restore_vma_archive {
7158     my ($archive, $vmid, $user, $opts, $comp) = @_;
7159
7160     my $readfrom = $archive;
7161
7162     my $cfg = PVE::Storage::config();
7163     my $commands = [];
7164     my $bwlimit = $opts->{bwlimit};
7165
7166     my $dbg_cmdstring = '';
7167     my $add_pipe = sub {
7168         my ($cmd) = @_;
7169         push @$commands, $cmd;
7170         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7171         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7172         $readfrom = '-';
7173     };
7174
7175     my $input = undef;
7176     if ($archive eq '-') {
7177         $input = '<&STDIN';
7178     } else {
7179         # If we use a backup from a PVE defined storage we also consider that
7180         # storage's rate limit:
7181         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7182         if (defined($volid)) {
7183             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7184             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7185             if ($readlimit) {
7186                 print STDERR "applying read rate limit: $readlimit\n";
7187                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7188                 $add_pipe->($cstream);
7189             }
7190         }
7191     }
7192
7193     if ($comp) {
7194         my $info = PVE::Storage::decompressor_info('vma', $comp);
7195         my $cmd = $info->{decompressor};
7196         push @$cmd, $readfrom;
7197         $add_pipe->($cmd);
7198     }
7199
7200     my $tmpdir = "/var/tmp/vzdumptmp$$";
7201     rmtree $tmpdir;
7202
7203     # disable interrupts (always do cleanups)
7204     local $SIG{INT} =
7205         local $SIG{TERM} =
7206         local $SIG{QUIT} =
7207         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7208
7209     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7210     POSIX::mkfifo($mapfifo, 0600);
7211     my $fifofh;
7212     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7213
7214     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7215
7216     my $oldtimeout;
7217     my $timeout = 5;
7218
7219     my $devinfo = {}; # info about drives included in backup
7220     my $virtdev_hash = {}; # info about allocated drives
7221
7222     my $rpcenv = PVE::RPCEnvironment::get();
7223
7224     my $conffile = PVE::QemuConfig->config_file($vmid);
7225
7226     # Note: $oldconf is undef if VM does not exist
7227     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7228     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7229     my $new_conf_raw = '';
7230
7231     my %storage_limits;
7232
7233     my $print_devmap = sub {
7234         my $cfgfn = "$tmpdir/qemu-server.conf";
7235
7236         # we can read the config - that is already extracted
7237         my $fh = IO::File->new($cfgfn, "r") ||
7238             die "unable to read qemu-server.conf - $!\n";
7239
7240         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7241         if (-f $fwcfgfn) {
7242             my $pve_firewall_dir = '/etc/pve/firewall';
7243             mkdir $pve_firewall_dir; # make sure the dir exists
7244             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7245         }
7246
7247         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7248
7249         foreach my $info (values %{$virtdev_hash}) {
7250             my $storeid = $info->{storeid};
7251             next if defined($storage_limits{$storeid});
7252
7253             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7254             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7255             $storage_limits{$storeid} = $limit * 1024;
7256         }
7257
7258         foreach my $devname (keys %$devinfo) {
7259             die "found no device mapping information for device '$devname'\n"
7260                 if !$devinfo->{$devname}->{virtdev};
7261         }
7262
7263         # create empty/temp config
7264         if ($oldconf) {
7265             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7266             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7267         }
7268
7269         # allocate volumes
7270         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7271
7272         # print restore information to $fifofh
7273         foreach my $virtdev (sort keys %$virtdev_hash) {
7274             my $d = $virtdev_hash->{$virtdev};
7275             next if $d->{is_cloudinit}; # no need to restore cloudinit
7276
7277             my $storeid = $d->{storeid};
7278             my $volid = $d->{volid};
7279
7280             my $map_opts = '';
7281             if (my $limit = $storage_limits{$storeid}) {
7282                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7283             }
7284
7285             my $write_zeros = 1;
7286             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7287                 $write_zeros = 0;
7288             }
7289
7290             my $path = PVE::Storage::path($cfg, $volid);
7291
7292             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7293
7294             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7295         }
7296
7297         $fh->seek(0, 0) || die "seek failed - $!\n";
7298
7299         my $cookie = { netcount => 0 };
7300         while (defined(my $line = <$fh>)) {
7301             $new_conf_raw .= restore_update_config_line(
7302                 $cookie,
7303                 $map,
7304                 $line,
7305                 $opts->{unique},
7306             );
7307         }
7308
7309         $fh->close();
7310     };
7311
7312     eval {
7313         # enable interrupts
7314         local $SIG{INT} =
7315             local $SIG{TERM} =
7316             local $SIG{QUIT} =
7317             local $SIG{HUP} =
7318             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7319         local $SIG{ALRM} = sub { die "got timeout\n"; };
7320
7321         $oldtimeout = alarm($timeout);
7322
7323         my $parser = sub {
7324             my $line = shift;
7325
7326             print "$line\n";
7327
7328             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7329                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7330                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7331             } elsif ($line =~ m/^CTIME: /) {
7332                 # we correctly received the vma config, so we can disable
7333                 # the timeout now for disk allocation (set to 10 minutes, so
7334                 # that we always timeout if something goes wrong)
7335                 alarm(600);
7336                 &$print_devmap();
7337                 print $fifofh "done\n";
7338                 my $tmp = $oldtimeout || 0;
7339                 $oldtimeout = undef;
7340                 alarm($tmp);
7341                 close($fifofh);
7342                 $fifofh = undef;
7343             }
7344         };
7345
7346         print "restore vma archive: $dbg_cmdstring\n";
7347         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7348     };
7349     my $err = $@;
7350
7351     alarm($oldtimeout) if $oldtimeout;
7352
7353     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7354
7355     close($fifofh) if $fifofh;
7356     unlink $mapfifo;
7357     rmtree $tmpdir;
7358
7359     if ($err) {
7360         $restore_destroy_volumes->($cfg, $virtdev_hash);
7361         die $err;
7362     }
7363
7364     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7365     PVE::QemuConfig->write_config($vmid, $new_conf);
7366
7367     eval { rescan($vmid, 1); };
7368     warn $@ if $@;
7369
7370     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7371 }
7372
7373 sub restore_tar_archive {
7374     my ($archive, $vmid, $user, $opts) = @_;
7375
7376     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7377         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7378         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7379     }
7380
7381     if ($archive ne '-') {
7382         my $firstfile = tar_archive_read_firstfile($archive);
7383         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7384             if $firstfile ne 'qemu-server.conf';
7385     }
7386
7387     my $storecfg = PVE::Storage::config();
7388
7389     # avoid zombie disks when restoring over an existing VM -> cleanup first
7390     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7391     # skiplock=1 because qmrestore has set the 'create' lock itself already
7392     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7393     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7394
7395     my $tocmd = "/usr/lib/qemu-server/qmextract";
7396
7397     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7398     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7399     $tocmd .= ' --prealloc' if $opts->{prealloc};
7400     $tocmd .= ' --info' if $opts->{info};
7401
7402     # tar option "xf" does not autodetect compression when read from STDIN,
7403     # so we pipe to zcat
7404     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7405         PVE::Tools::shellquote("--to-command=$tocmd");
7406
7407     my $tmpdir = "/var/tmp/vzdumptmp$$";
7408     mkpath $tmpdir;
7409
7410     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7411     local $ENV{VZDUMP_VMID} = $vmid;
7412     local $ENV{VZDUMP_USER} = $user;
7413
7414     my $conffile = PVE::QemuConfig->config_file($vmid);
7415     my $new_conf_raw = '';
7416
7417     # disable interrupts (always do cleanups)
7418     local $SIG{INT} =
7419         local $SIG{TERM} =
7420         local $SIG{QUIT} =
7421         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7422
7423     eval {
7424         # enable interrupts
7425         local $SIG{INT} =
7426             local $SIG{TERM} =
7427             local $SIG{QUIT} =
7428             local $SIG{HUP} =
7429             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7430
7431         if ($archive eq '-') {
7432             print "extracting archive from STDIN\n";
7433             run_command($cmd, input => "<&STDIN");
7434         } else {
7435             print "extracting archive '$archive'\n";
7436             run_command($cmd);
7437         }
7438
7439         return if $opts->{info};
7440
7441         # read new mapping
7442         my $map = {};
7443         my $statfile = "$tmpdir/qmrestore.stat";
7444         if (my $fd = IO::File->new($statfile, "r")) {
7445             while (defined (my $line = <$fd>)) {
7446                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7447                     $map->{$1} = $2 if $1;
7448                 } else {
7449                     print STDERR "unable to parse line in statfile - $line\n";
7450                 }
7451             }
7452             $fd->close();
7453         }
7454
7455         my $confsrc = "$tmpdir/qemu-server.conf";
7456
7457         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7458
7459         my $cookie = { netcount => 0 };
7460         while (defined (my $line = <$srcfd>)) {
7461             $new_conf_raw .= restore_update_config_line(
7462                 $cookie,
7463                 $map,
7464                 $line,
7465                 $opts->{unique},
7466             );
7467         }
7468
7469         $srcfd->close();
7470     };
7471     if (my $err = $@) {
7472         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7473         die $err;
7474     }
7475
7476     rmtree $tmpdir;
7477
7478     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7479
7480     PVE::Cluster::cfs_update(); # make sure we read new file
7481
7482     eval { rescan($vmid, 1); };
7483     warn $@ if $@;
7484 };
7485
7486 sub foreach_storage_used_by_vm {
7487     my ($conf, $func) = @_;
7488
7489     my $sidhash = {};
7490
7491     PVE::QemuConfig->foreach_volume($conf, sub {
7492         my ($ds, $drive) = @_;
7493         return if drive_is_cdrom($drive);
7494
7495         my $volid = $drive->{file};
7496
7497         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7498         $sidhash->{$sid} = $sid if $sid;
7499     });
7500
7501     foreach my $sid (sort keys %$sidhash) {
7502         &$func($sid);
7503     }
7504 }
7505
7506 my $qemu_snap_storage = {
7507     rbd => 1,
7508 };
7509 sub do_snapshots_with_qemu {
7510     my ($storecfg, $volid, $deviceid) = @_;
7511
7512     return if $deviceid =~ m/tpmstate0/;
7513
7514     my $storage_name = PVE::Storage::parse_volume_id($volid);
7515     my $scfg = $storecfg->{ids}->{$storage_name};
7516     die "could not find storage '$storage_name'\n" if !defined($scfg);
7517
7518     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7519         return 1;
7520     }
7521
7522     if ($volid =~ m/\.(qcow2|qed)$/){
7523         return 1;
7524     }
7525
7526     return;
7527 }
7528
7529 sub qga_check_running {
7530     my ($vmid, $nowarn) = @_;
7531
7532     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7533     if ($@) {
7534         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7535         return 0;
7536     }
7537     return 1;
7538 }
7539
7540 sub template_create {
7541     my ($vmid, $conf, $disk) = @_;
7542
7543     my $storecfg = PVE::Storage::config();
7544
7545     PVE::QemuConfig->foreach_volume($conf, sub {
7546         my ($ds, $drive) = @_;
7547
7548         return if drive_is_cdrom($drive);
7549         return if $disk && $ds ne $disk;
7550
7551         my $volid = $drive->{file};
7552         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7553
7554         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7555         $drive->{file} = $voliddst;
7556         $conf->{$ds} = print_drive($drive);
7557         PVE::QemuConfig->write_config($vmid, $conf);
7558     });
7559 }
7560
7561 sub convert_iscsi_path {
7562     my ($path) = @_;
7563
7564     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7565         my $portal = $1;
7566         my $target = $2;
7567         my $lun = $3;
7568
7569         my $initiator_name = get_initiator_name();
7570
7571         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7572                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7573     }
7574
7575     die "cannot convert iscsi path '$path', unkown format\n";
7576 }
7577
7578 sub qemu_img_convert {
7579     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7580
7581     my $storecfg = PVE::Storage::config();
7582     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7583     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7584
7585     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7586
7587     my $cachemode;
7588     my $src_path;
7589     my $src_is_iscsi = 0;
7590     my $src_format;
7591
7592     if ($src_storeid) {
7593         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7594         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7595         $src_format = qemu_img_format($src_scfg, $src_volname);
7596         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7597         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7598         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7599     } elsif (-f $src_volid || -b $src_volid) {
7600         $src_path = $src_volid;
7601         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7602             $src_format = $1;
7603         }
7604     }
7605
7606     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7607
7608     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7609     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7610     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7611     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7612
7613     my $cmd = [];
7614     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7615     push @$cmd, '-l', "snapshot.name=$snapname"
7616         if $snapname && $src_format && $src_format eq "qcow2";
7617     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7618     push @$cmd, '-T', $cachemode if defined($cachemode);
7619
7620     if ($src_is_iscsi) {
7621         push @$cmd, '--image-opts';
7622         $src_path = convert_iscsi_path($src_path);
7623     } elsif ($src_format) {
7624         push @$cmd, '-f', $src_format;
7625     }
7626
7627     if ($dst_is_iscsi) {
7628         push @$cmd, '--target-image-opts';
7629         $dst_path = convert_iscsi_path($dst_path);
7630     } else {
7631         push @$cmd, '-O', $dst_format;
7632     }
7633
7634     push @$cmd, $src_path;
7635
7636     if (!$dst_is_iscsi && $is_zero_initialized) {
7637         push @$cmd, "zeroinit:$dst_path";
7638     } else {
7639         push @$cmd, $dst_path;
7640     }
7641
7642     my $parser = sub {
7643         my $line = shift;
7644         if($line =~ m/\((\S+)\/100\%\)/){
7645             my $percent = $1;
7646             my $transferred = int($size * $percent / 100);
7647             my $total_h = render_bytes($size, 1);
7648             my $transferred_h = render_bytes($transferred, 1);
7649
7650             print "transferred $transferred_h of $total_h ($percent%)\n";
7651         }
7652
7653     };
7654
7655     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7656     my $err = $@;
7657     die "copy failed: $err" if $err;
7658 }
7659
7660 sub qemu_img_format {
7661     my ($scfg, $volname) = @_;
7662
7663     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7664         return $1;
7665     } else {
7666         return "raw";
7667     }
7668 }
7669
7670 sub qemu_drive_mirror {
7671     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7672
7673     $jobs = {} if !$jobs;
7674
7675     my $qemu_target;
7676     my $format;
7677     $jobs->{"drive-$drive"} = {};
7678
7679     if ($dst_volid =~ /^nbd:/) {
7680         $qemu_target = $dst_volid;
7681         $format = "nbd";
7682     } else {
7683         my $storecfg = PVE::Storage::config();
7684         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7685
7686         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7687
7688         $format = qemu_img_format($dst_scfg, $dst_volname);
7689
7690         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7691
7692         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7693     }
7694
7695     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7696     $opts->{format} = $format if $format;
7697
7698     if (defined($src_bitmap)) {
7699         $opts->{sync} = 'incremental';
7700         $opts->{bitmap} = $src_bitmap;
7701         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7702     }
7703
7704     if (defined($bwlimit)) {
7705         $opts->{speed} = $bwlimit * 1024;
7706         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7707     } else {
7708         print "drive mirror is starting for drive-$drive\n";
7709     }
7710
7711     # if a job already runs for this device we get an error, catch it for cleanup
7712     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7713     if (my $err = $@) {
7714         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7715         warn "$@\n" if $@;
7716         die "mirroring error: $err\n";
7717     }
7718
7719     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7720 }
7721
7722 # $completion can be either
7723 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7724 # 'cancel': wait until all jobs are ready, block-job-cancel them
7725 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7726 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7727 sub qemu_drive_mirror_monitor {
7728     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7729
7730     $completion //= 'complete';
7731     $op //= "mirror";
7732
7733     eval {
7734         my $err_complete = 0;
7735
7736         my $starttime = time ();
7737         while (1) {
7738             die "block job ('$op') timed out\n" if $err_complete > 300;
7739
7740             my $stats = mon_cmd($vmid, "query-block-jobs");
7741             my $ctime = time();
7742
7743             my $running_jobs = {};
7744             for my $stat (@$stats) {
7745                 next if $stat->{type} ne $op;
7746                 $running_jobs->{$stat->{device}} = $stat;
7747             }
7748
7749             my $readycounter = 0;
7750
7751             for my $job_id (sort keys %$jobs) {
7752                 my $job = $running_jobs->{$job_id};
7753
7754                 my $vanished = !defined($job);
7755                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7756                 if($complete || ($vanished && $completion eq 'auto')) {
7757                     print "$job_id: $op-job finished\n";
7758                     delete $jobs->{$job_id};
7759                     next;
7760                 }
7761
7762                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7763
7764                 my $busy = $job->{busy};
7765                 my $ready = $job->{ready};
7766                 if (my $total = $job->{len}) {
7767                     my $transferred = $job->{offset} || 0;
7768                     my $remaining = $total - $transferred;
7769                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7770
7771                     my $duration = $ctime - $starttime;
7772                     my $total_h = render_bytes($total, 1);
7773                     my $transferred_h = render_bytes($transferred, 1);
7774
7775                     my $status = sprintf(
7776                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7777                         render_duration($duration),
7778                     );
7779
7780                     if ($ready) {
7781                         if ($busy) {
7782                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7783                         } else {
7784                             $status .= ", ready";
7785                         }
7786                     }
7787                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7788                     $jobs->{$job_id}->{ready} = $ready;
7789                 }
7790
7791                 $readycounter++ if $job->{ready};
7792             }
7793
7794             last if scalar(keys %$jobs) == 0;
7795
7796             if ($readycounter == scalar(keys %$jobs)) {
7797                 print "all '$op' jobs are ready\n";
7798
7799                 # do the complete later (or has already been done)
7800                 last if $completion eq 'skip' || $completion eq 'auto';
7801
7802                 if ($vmiddst && $vmiddst != $vmid) {
7803                     my $agent_running = $qga && qga_check_running($vmid);
7804                     if ($agent_running) {
7805                         print "freeze filesystem\n";
7806                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7807                         warn $@ if $@;
7808                     } else {
7809                         print "suspend vm\n";
7810                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7811                         warn $@ if $@;
7812                     }
7813
7814                     # if we clone a disk for a new target vm, we don't switch the disk
7815                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7816
7817                     if ($agent_running) {
7818                         print "unfreeze filesystem\n";
7819                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7820                         warn $@ if $@;
7821                     } else {
7822                         print "resume vm\n";
7823                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7824                         warn $@ if $@;
7825                     }
7826
7827                     last;
7828                 } else {
7829
7830                     for my $job_id (sort keys %$jobs) {
7831                         # try to switch the disk if source and destination are on the same guest
7832                         print "$job_id: Completing block job_id...\n";
7833
7834                         my $op;
7835                         if ($completion eq 'complete') {
7836                             $op = 'block-job-complete';
7837                         } elsif ($completion eq 'cancel') {
7838                             $op = 'block-job-cancel';
7839                         } else {
7840                             die "invalid completion value: $completion\n";
7841                         }
7842                         eval { mon_cmd($vmid, $op, device => $job_id) };
7843                         if ($@ =~ m/cannot be completed/) {
7844                             print "$job_id: block job cannot be completed, trying again.\n";
7845                             $err_complete++;
7846                         }else {
7847                             print "$job_id: Completed successfully.\n";
7848                             $jobs->{$job_id}->{complete} = 1;
7849                         }
7850                     }
7851                 }
7852             }
7853             sleep 1;
7854         }
7855     };
7856     my $err = $@;
7857
7858     if ($err) {
7859         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7860         die "block job ($op) error: $err";
7861     }
7862 }
7863
7864 sub qemu_blockjobs_cancel {
7865     my ($vmid, $jobs) = @_;
7866
7867     foreach my $job (keys %$jobs) {
7868         print "$job: Cancelling block job\n";
7869         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7870         $jobs->{$job}->{cancel} = 1;
7871     }
7872
7873     while (1) {
7874         my $stats = mon_cmd($vmid, "query-block-jobs");
7875
7876         my $running_jobs = {};
7877         foreach my $stat (@$stats) {
7878             $running_jobs->{$stat->{device}} = $stat;
7879         }
7880
7881         foreach my $job (keys %$jobs) {
7882
7883             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7884                 print "$job: Done.\n";
7885                 delete $jobs->{$job};
7886             }
7887         }
7888
7889         last if scalar(keys %$jobs) == 0;
7890
7891         sleep 1;
7892     }
7893 }
7894
7895 sub clone_disk {
7896     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7897
7898     my ($vmid, $running) = $source->@{qw(vmid running)};
7899     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7900
7901     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7902     my ($storage, $format) = $dest->@{qw(storage format)};
7903
7904     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7905
7906     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7907         die "cloning from/to EFI disk requires EFI disk\n"
7908             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7909         die "cloning from/to TPM state requires TPM state\n"
7910             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7911
7912         # This would lead to two device nodes in QEMU pointing to the same backing image!
7913         die "cannot change drive name when cloning disk from/to the same VM\n"
7914             if $use_drive_mirror && $vmid == $newvmid;
7915     }
7916
7917     die "cannot move TPM state while VM is running\n"
7918         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7919
7920     my $newvolid;
7921
7922     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7923     print "$src_drivename " if $src_drivename;
7924     print "($drive->{file})\n";
7925
7926     if (!$full) {
7927         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7928         push @$newvollist, $newvolid;
7929     } else {
7930
7931         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7932         $storeid = $storage if $storage;
7933
7934         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7935
7936         my $name = undef;
7937         my $size = undef;
7938         if (drive_is_cloudinit($drive)) {
7939             $name = "vm-$newvmid-cloudinit";
7940             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7941             if ($scfg->{path}) {
7942                 $name .= ".$dst_format";
7943             }
7944             $snapname = undef;
7945             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7946         } elsif ($dst_drivename eq 'efidisk0') {
7947             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7948         } elsif ($dst_drivename eq 'tpmstate0') {
7949             $dst_format = 'raw';
7950             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7951         } else {
7952             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7953         }
7954         $newvolid = PVE::Storage::vdisk_alloc(
7955             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7956         );
7957         push @$newvollist, $newvolid;
7958
7959         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7960
7961         if (drive_is_cloudinit($drive)) {
7962             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7963             # if this is the case, we have to complete any block-jobs still there from
7964             # previous drive-mirrors
7965             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7966                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7967             }
7968             goto no_data_clone;
7969         }
7970
7971         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7972         if ($use_drive_mirror) {
7973             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7974                 $completion, $qga, $bwlimit);
7975         } else {
7976             # TODO: handle bwlimits
7977             if ($dst_drivename eq 'efidisk0') {
7978                 # the relevant data on the efidisk may be smaller than the source
7979                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7980                 # that is given by the OVMF_VARS.fd
7981                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7982                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7983
7984                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7985
7986                 # better for Ceph if block size is not too small, see bug #3324
7987                 my $bs = 1024*1024;
7988
7989                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7990
7991                 if ($src_format eq 'qcow2' && $snapname) {
7992                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7993                         if !min_version(kvm_user_version(), 6, 2);
7994                     push $cmd->@*, '-l', $snapname;
7995                 }
7996                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7997                 run_command($cmd);
7998             } else {
7999                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8000             }
8001         }
8002     }
8003
8004 no_data_clone:
8005     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8006
8007     my $disk = dclone($drive);
8008     delete $disk->{format};
8009     $disk->{file} = $newvolid;
8010     $disk->{size} = $size if defined($size);
8011
8012     return $disk;
8013 }
8014
8015 sub get_running_qemu_version {
8016     my ($vmid) = @_;
8017     my $res = mon_cmd($vmid, "query-version");
8018     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8019 }
8020
8021 sub qemu_use_old_bios_files {
8022     my ($machine_type) = @_;
8023
8024     return if !$machine_type;
8025
8026     my $use_old_bios_files = undef;
8027
8028     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8029         $machine_type = $1;
8030         $use_old_bios_files = 1;
8031     } else {
8032         my $version = extract_version($machine_type, kvm_user_version());
8033         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8034         # load new efi bios files on migration. So this hack is required to allow
8035         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8036         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8037         $use_old_bios_files = !min_version($version, 2, 4);
8038     }
8039
8040     return ($use_old_bios_files, $machine_type);
8041 }
8042
8043 sub get_efivars_size {
8044     my ($conf, $efidisk) = @_;
8045
8046     my $arch = get_vm_arch($conf);
8047     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8048     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8049     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8050     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8051     return -s $ovmf_vars;
8052 }
8053
8054 sub update_efidisk_size {
8055     my ($conf) = @_;
8056
8057     return if !defined($conf->{efidisk0});
8058
8059     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8060     $disk->{size} = get_efivars_size($conf);
8061     $conf->{efidisk0} = print_drive($disk);
8062
8063     return;
8064 }
8065
8066 sub update_tpmstate_size {
8067     my ($conf) = @_;
8068
8069     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8070     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8071     $conf->{tpmstate0} = print_drive($disk);
8072 }
8073
8074 sub create_efidisk($$$$$$$) {
8075     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8076
8077     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8078     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8079
8080     my $vars_size_b = -s $ovmf_vars;
8081     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8082     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8083     PVE::Storage::activate_volumes($storecfg, [$volid]);
8084
8085     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8086     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8087
8088     return ($volid, $size/1024);
8089 }
8090
8091 sub vm_iothreads_list {
8092     my ($vmid) = @_;
8093
8094     my $res = mon_cmd($vmid, 'query-iothreads');
8095
8096     my $iothreads = {};
8097     foreach my $iothread (@$res) {
8098         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8099     }
8100
8101     return $iothreads;
8102 }
8103
8104 sub scsihw_infos {
8105     my ($conf, $drive) = @_;
8106
8107     my $maxdev = 0;
8108
8109     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8110         $maxdev = 7;
8111     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8112         $maxdev = 1;
8113     } else {
8114         $maxdev = 256;
8115     }
8116
8117     my $controller = int($drive->{index} / $maxdev);
8118     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8119         ? "virtioscsi"
8120         : "scsihw";
8121
8122     return ($maxdev, $controller, $controller_prefix);
8123 }
8124
8125 sub resolve_dst_disk_format {
8126         my ($storecfg, $storeid, $src_volname, $format) = @_;
8127         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8128
8129         if (!$format) {
8130             # if no target format is specified, use the source disk format as hint
8131             if ($src_volname) {
8132                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8133                 $format = qemu_img_format($scfg, $src_volname);
8134             } else {
8135                 return $defFormat;
8136             }
8137         }
8138
8139         # test if requested format is supported - else use default
8140         my $supported = grep { $_ eq $format } @$validFormats;
8141         $format = $defFormat if !$supported;
8142         return $format;
8143 }
8144
8145 # NOTE: if this logic changes, please update docs & possibly gui logic
8146 sub find_vmstate_storage {
8147     my ($conf, $storecfg) = @_;
8148
8149     # first, return storage from conf if set
8150     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8151
8152     my ($target, $shared, $local);
8153
8154     foreach_storage_used_by_vm($conf, sub {
8155         my ($sid) = @_;
8156         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8157         my $dst = $scfg->{shared} ? \$shared : \$local;
8158         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8159     });
8160
8161     # second, use shared storage where VM has at least one disk
8162     # third, use local storage where VM has at least one disk
8163     # fall back to local storage
8164     $target = $shared // $local // 'local';
8165
8166     return $target;
8167 }
8168
8169 sub generate_uuid {
8170     my ($uuid, $uuid_str);
8171     UUID::generate($uuid);
8172     UUID::unparse($uuid, $uuid_str);
8173     return $uuid_str;
8174 }
8175
8176 sub generate_smbios1_uuid {
8177     return "uuid=".generate_uuid();
8178 }
8179
8180 sub nbd_stop {
8181     my ($vmid) = @_;
8182
8183     mon_cmd($vmid, 'nbd-server-stop');
8184 }
8185
8186 sub create_reboot_request {
8187     my ($vmid) = @_;
8188     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8189         or die "failed to create reboot trigger file: $!\n";
8190     close($fh);
8191 }
8192
8193 sub clear_reboot_request {
8194     my ($vmid) = @_;
8195     my $path = "/run/qemu-server/$vmid.reboot";
8196     my $res = 0;
8197
8198     $res = unlink($path);
8199     die "could not remove reboot request for $vmid: $!"
8200         if !$res && $! != POSIX::ENOENT;
8201
8202     return $res;
8203 }
8204
8205 sub bootorder_from_legacy {
8206     my ($conf, $bootcfg) = @_;
8207
8208     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8209     my $bootindex_hash = {};
8210     my $i = 1;
8211     foreach my $o (split(//, $boot)) {
8212         $bootindex_hash->{$o} = $i*100;
8213         $i++;
8214     }
8215
8216     my $bootorder = {};
8217
8218     PVE::QemuConfig->foreach_volume($conf, sub {
8219         my ($ds, $drive) = @_;
8220
8221         if (drive_is_cdrom ($drive, 1)) {
8222             if ($bootindex_hash->{d}) {
8223                 $bootorder->{$ds} = $bootindex_hash->{d};
8224                 $bootindex_hash->{d} += 1;
8225             }
8226         } elsif ($bootindex_hash->{c}) {
8227             $bootorder->{$ds} = $bootindex_hash->{c}
8228                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8229             $bootindex_hash->{c} += 1;
8230         }
8231     });
8232
8233     if ($bootindex_hash->{n}) {
8234         for (my $i = 0; $i < $MAX_NETS; $i++) {
8235             my $netname = "net$i";
8236             next if !$conf->{$netname};
8237             $bootorder->{$netname} = $bootindex_hash->{n};
8238             $bootindex_hash->{n} += 1;
8239         }
8240     }
8241
8242     return $bootorder;
8243 }
8244
8245 # Generate default device list for 'boot: order=' property. Matches legacy
8246 # default boot order, but with explicit device names. This is important, since
8247 # the fallback for when neither 'order' nor the old format is specified relies
8248 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8249 sub get_default_bootdevices {
8250     my ($conf) = @_;
8251
8252     my @ret = ();
8253
8254     # harddisk
8255     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8256     push @ret, $first if $first;
8257
8258     # cdrom
8259     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8260     push @ret, $first if $first;
8261
8262     # network
8263     for (my $i = 0; $i < $MAX_NETS; $i++) {
8264         my $netname = "net$i";
8265         next if !$conf->{$netname};
8266         push @ret, $netname;
8267         last;
8268     }
8269
8270     return \@ret;
8271 }
8272
8273 sub device_bootorder {
8274     my ($conf) = @_;
8275
8276     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8277
8278     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8279
8280     my $bootorder = {};
8281     if (!defined($boot) || $boot->{legacy}) {
8282         $bootorder = bootorder_from_legacy($conf, $boot);
8283     } elsif ($boot->{order}) {
8284         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8285         for my $dev (PVE::Tools::split_list($boot->{order})) {
8286             $bootorder->{$dev} = $i++;
8287         }
8288     }
8289
8290     return $bootorder;
8291 }
8292
8293 sub register_qmeventd_handle {
8294     my ($vmid) = @_;
8295
8296     my $fh;
8297     my $peer = "/var/run/qmeventd.sock";
8298     my $count = 0;
8299
8300     for (;;) {
8301         $count++;
8302         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8303         last if $fh;
8304         if ($! != EINTR && $! != EAGAIN) {
8305             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8306         }
8307         if ($count > 4) {
8308             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8309               . "after $count retries\n";
8310         }
8311         usleep(25000);
8312     }
8313
8314     # send handshake to mark VM as backing up
8315     print $fh to_json({vzdump => {vmid => "$vmid"}});
8316
8317     # return handle to be closed later when inhibit is no longer required
8318     return $fh;
8319 }
8320
8321 # bash completion helper
8322
8323 sub complete_backup_archives {
8324     my ($cmdname, $pname, $cvalue) = @_;
8325
8326     my $cfg = PVE::Storage::config();
8327
8328     my $storeid;
8329
8330     if ($cvalue =~ m/^([^:]+):/) {
8331         $storeid = $1;
8332     }
8333
8334     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8335
8336     my $res = [];
8337     foreach my $id (keys %$data) {
8338         foreach my $item (@{$data->{$id}}) {
8339             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8340             push @$res, $item->{volid} if defined($item->{volid});
8341         }
8342     }
8343
8344     return $res;
8345 }
8346
8347 my $complete_vmid_full = sub {
8348     my ($running) = @_;
8349
8350     my $idlist = vmstatus();
8351
8352     my $res = [];
8353
8354     foreach my $id (keys %$idlist) {
8355         my $d = $idlist->{$id};
8356         if (defined($running)) {
8357             next if $d->{template};
8358             next if $running && $d->{status} ne 'running';
8359             next if !$running && $d->{status} eq 'running';
8360         }
8361         push @$res, $id;
8362
8363     }
8364     return $res;
8365 };
8366
8367 sub complete_vmid {
8368     return &$complete_vmid_full();
8369 }
8370
8371 sub complete_vmid_stopped {
8372     return &$complete_vmid_full(0);
8373 }
8374
8375 sub complete_vmid_running {
8376     return &$complete_vmid_full(1);
8377 }
8378
8379 sub complete_storage {
8380
8381     my $cfg = PVE::Storage::config();
8382     my $ids = $cfg->{ids};
8383
8384     my $res = [];
8385     foreach my $sid (keys %$ids) {
8386         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8387         next if !$ids->{$sid}->{content}->{images};
8388         push @$res, $sid;
8389     }
8390
8391     return $res;
8392 }
8393
8394 sub complete_migration_storage {
8395     my ($cmd, $param, $current_value, $all_args) = @_;
8396
8397     my $targetnode = @$all_args[1];
8398
8399     my $cfg = PVE::Storage::config();
8400     my $ids = $cfg->{ids};
8401
8402     my $res = [];
8403     foreach my $sid (keys %$ids) {
8404         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8405         next if !$ids->{$sid}->{content}->{images};
8406         push @$res, $sid;
8407     }
8408
8409     return $res;
8410 }
8411
8412 sub vm_is_paused {
8413     my ($vmid) = @_;
8414     my $qmpstatus = eval {
8415         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8416         mon_cmd($vmid, "query-status");
8417     };
8418     warn "$@\n" if $@;
8419     return $qmpstatus && $qmpstatus->{status} eq "paused";
8420 }
8421
8422 sub check_volume_storage_type {
8423     my ($storecfg, $vol) = @_;
8424
8425     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8426     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8427     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8428
8429     die "storage '$storeid' does not support content-type '$vtype'\n"
8430         if !$scfg->{content}->{$vtype};
8431
8432     return 1;
8433 }
8434
8435 sub add_nets_bridge_fdb {
8436     my ($conf, $vmid) = @_;
8437
8438     for my $opt (keys %$conf) {
8439         next if $opt !~ m/^net(\d+)$/;
8440         my $iface = "tap${vmid}i$1";
8441         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8442         my $net = parse_net($conf->{$opt}, 1) or next;
8443
8444         my $mac = $net->{macaddr};
8445         if (!$mac) {
8446             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8447                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8448             next;
8449         }
8450
8451         my $bridge = $net->{bridge};
8452         if ($have_sdn) {
8453             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8454         } else {
8455             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8456         }
8457     }
8458 }
8459
8460 sub del_nets_bridge_fdb {
8461     my ($conf, $vmid) = @_;
8462
8463     for my $opt (keys %$conf) {
8464         next if $opt !~ m/^net(\d+)$/;
8465         my $iface = "tap${vmid}i$1";
8466
8467         my $net = parse_net($conf->{$opt}) or next;
8468         my $mac = $net->{macaddr} or next;
8469
8470         my $bridge = $net->{bridge};
8471         if ($have_sdn) {
8472             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8473         } else {
8474             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8475         }
8476     }
8477 }
8478
8479 1;