PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2345
2346     if ($conf->{template}) {
2347         # check if any base image is still used by a linked clone
2348         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2349                 my ($ds, $drive) = @_;
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353                 return if !$volid || $volid =~ m|^/|;
2354
2355                 die "base volume '$volid' is still in use by linked cloned\n"
2356                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2357
2358         });
2359     }
2360
2361     my $volids = {};
2362     my $remove_owned_drive = sub {
2363         my ($ds, $drive) = @_;
2364         return if drive_is_cdrom($drive, 1);
2365
2366         my $volid = $drive->{file};
2367         return if !$volid || $volid =~ m|^/|;
2368         return if $volids->{$volid};
2369
2370         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2371         return if !$path || !$owner || ($owner != $vmid);
2372
2373         $volids->{$volid} = 1;
2374         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2375         warn "Could not remove disk '$volid', check manually: $@" if $@;
2376     };
2377
2378     # only remove disks owned by this VM (referenced in the config)
2379     my $include_opts = {
2380         include_unused => 1,
2381         extra_keys => ['vmstate'],
2382     };
2383     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2384
2385     for my $snap (values %{$conf->{snapshots}}) {
2386         next if !defined($snap->{vmstate});
2387         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2388         next if !defined($drive);
2389         $remove_owned_drive->('vmstate', $drive);
2390     }
2391
2392     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2393
2394     if ($purge_unreferenced) { # also remove unreferenced disk
2395         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2396         PVE::Storage::foreach_volid($vmdisks, sub {
2397             my ($volid, $sid, $volname, $d) = @_;
2398             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2399             warn $@ if $@;
2400         });
2401     }
2402
2403     if (defined $replacement_conf) {
2404         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2405     } else {
2406         PVE::QemuConfig->destroy_config($vmid);
2407     }
2408 }
2409
2410 sub parse_vm_config {
2411     my ($filename, $raw, $strict) = @_;
2412
2413     return if !defined($raw);
2414
2415     my $res = {
2416         digest => Digest::SHA::sha1_hex($raw),
2417         snapshots => {},
2418         pending => {},
2419         cloudinit => {},
2420     };
2421
2422     my $handle_error = sub {
2423         my ($msg) = @_;
2424
2425         if ($strict) {
2426             die $msg;
2427         } else {
2428             warn $msg;
2429         }
2430     };
2431
2432     $filename =~ m|/qemu-server/(\d+)\.conf$|
2433         || die "got strange filename '$filename'";
2434
2435     my $vmid = $1;
2436
2437     my $conf = $res;
2438     my $descr;
2439     my $finish_description = sub {
2440         if (defined($descr)) {
2441             $descr =~ s/\s+$//;
2442             $conf->{description} = $descr;
2443         }
2444         $descr = undef;
2445     };
2446     my $section = '';
2447
2448     my @lines = split(/\n/, $raw);
2449     foreach my $line (@lines) {
2450         next if $line =~ m/^\s*$/;
2451
2452         if ($line =~ m/^\[PENDING\]\s*$/i) {
2453             $section = 'pending';
2454             $finish_description->();
2455             $conf = $res->{$section} = {};
2456             next;
2457         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458             $section = 'cloudinit';
2459             $finish_description->();
2460             $conf = $res->{$section} = {};
2461             next;
2462
2463         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2464             $section = $1;
2465             $finish_description->();
2466             $conf = $res->{snapshots}->{$section} = {};
2467             next;
2468         }
2469
2470         if ($line =~ m/^\#(.*)$/) {
2471             $descr = '' if !defined($descr);
2472             $descr .= PVE::Tools::decode_text($1) . "\n";
2473             next;
2474         }
2475
2476         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477             $descr = '' if !defined($descr);
2478             $descr .= PVE::Tools::decode_text($2);
2479         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480             $conf->{snapstate} = $1;
2481         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2482             my $key = $1;
2483             my $value = $2;
2484             $conf->{$key} = $value;
2485         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2486             my $value = $1;
2487             if ($section eq 'pending') {
2488                 $conf->{delete} = $value; # we parse this later
2489             } else {
2490                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2491             }
2492         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             if ($section eq 'cloudinit') {
2496                 # ignore validation only used for informative purpose
2497                 $conf->{$key} = $value;
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             if ($@) {
2502                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2503             } else {
2504                 $key = 'ide2' if $key eq 'cdrom';
2505                 my $fmt = $confdesc->{$key}->{format};
2506                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507                     my $v = parse_drive($key, $value);
2508                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2509                         $v->{file} = $volid;
2510                         $value = print_drive($v);
2511                     } else {
2512                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513                         next;
2514                     }
2515                 }
2516
2517                 $conf->{$key} = $value;
2518             }
2519         } else {
2520             $handle_error->("vm $vmid - unable to parse config: $line\n");
2521         }
2522     }
2523
2524     $finish_description->();
2525     delete $res->{snapstate}; # just to be sure
2526
2527     return $res;
2528 }
2529
2530 sub write_vm_config {
2531     my ($filename, $conf) = @_;
2532
2533     delete $conf->{snapstate}; # just to be sure
2534
2535     if ($conf->{cdrom}) {
2536         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2537         $conf->{ide2} = $conf->{cdrom};
2538         delete $conf->{cdrom};
2539     }
2540
2541     # we do not use 'smp' any longer
2542     if ($conf->{sockets}) {
2543         delete $conf->{smp};
2544     } elsif ($conf->{smp}) {
2545         $conf->{sockets} = $conf->{smp};
2546         delete $conf->{cores};
2547         delete $conf->{smp};
2548     }
2549
2550     my $used_volids = {};
2551
2552     my $cleanup_config = sub {
2553         my ($cref, $pending, $snapname) = @_;
2554
2555         foreach my $key (keys %$cref) {
2556             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558             my $value = $cref->{$key};
2559             if ($key eq 'delete') {
2560                 die "propertry 'delete' is only allowed in [PENDING]\n"
2561                     if !$pending;
2562                 # fixme: check syntax?
2563                 next;
2564             }
2565             eval { $value = check_type($key, $value); };
2566             die "unable to parse value of '$key' - $@" if $@;
2567
2568             $cref->{$key} = $value;
2569
2570             if (!$snapname && is_valid_drivename($key)) {
2571                 my $drive = parse_drive($key, $value);
2572                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2573             }
2574         }
2575     };
2576
2577     &$cleanup_config($conf);
2578
2579     &$cleanup_config($conf->{pending}, 1);
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2584     }
2585
2586     # remove 'unusedX' settings if we re-add a volume
2587     foreach my $key (keys %$conf) {
2588         my $value = $conf->{$key};
2589         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590             delete $conf->{$key};
2591         }
2592     }
2593
2594     my $generate_raw_config = sub {
2595         my ($conf, $pending) = @_;
2596
2597         my $raw = '';
2598
2599         # add description as comment to top of file
2600         if (defined(my $descr = $conf->{description})) {
2601             if ($descr) {
2602                 foreach my $cl (split(/\n/, $descr)) {
2603                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2604                 }
2605             } else {
2606                 $raw .= "#\n" if $pending;
2607             }
2608         }
2609
2610         foreach my $key (sort keys %$conf) {
2611             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612             $raw .= "$key: $conf->{$key}\n";
2613         }
2614         return $raw;
2615     };
2616
2617     my $raw = &$generate_raw_config($conf);
2618
2619     if (scalar(keys %{$conf->{pending}})){
2620         $raw .= "\n[PENDING]\n";
2621         $raw .= &$generate_raw_config($conf->{pending}, 1);
2622     }
2623
2624     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2625         $raw .= "\n[special:cloudinit]\n";
2626         $raw .= &$generate_raw_config($conf->{cloudinit});
2627     }
2628
2629     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2630         $raw .= "\n[$snapname]\n";
2631         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2632     }
2633
2634     return $raw;
2635 }
2636
2637 sub load_defaults {
2638
2639     my $res = {};
2640
2641     # we use static defaults from our JSON schema configuration
2642     foreach my $key (keys %$confdesc) {
2643         if (defined(my $default = $confdesc->{$key}->{default})) {
2644             $res->{$key} = $default;
2645         }
2646     }
2647
2648     return $res;
2649 }
2650
2651 sub config_list {
2652     my $vmlist = PVE::Cluster::get_vmlist();
2653     my $res = {};
2654     return $res if !$vmlist || !$vmlist->{ids};
2655     my $ids = $vmlist->{ids};
2656     my $nodename = nodename();
2657
2658     foreach my $vmid (keys %$ids) {
2659         my $d = $ids->{$vmid};
2660         next if !$d->{node} || $d->{node} ne $nodename;
2661         next if !$d->{type} || $d->{type} ne 'qemu';
2662         $res->{$vmid}->{exists} = 1;
2663     }
2664     return $res;
2665 }
2666
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources {
2669     my ($conf, $noerr) = @_;
2670
2671     my @loc_res = ();
2672
2673     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2674     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2675
2676     push @loc_res, "ivshmem" if $conf->{ivshmem};
2677
2678     foreach my $k (keys %$conf) {
2679         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680         # sockets are safe: they will recreated be on the target side post-migrate
2681         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2683     }
2684
2685     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2686
2687     return \@loc_res;
2688 }
2689
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability {
2692     my ($storecfg, $conf, $node) = @_;
2693
2694     PVE::QemuConfig->foreach_volume($conf, sub {
2695         my ($ds, $drive) = @_;
2696
2697         my $volid = $drive->{file};
2698         return if !$volid;
2699
2700         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2701         return if !$sid;
2702
2703         # check if storage is available on both nodes
2704         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2705         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2706
2707         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2708
2709         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710             if !$scfg->{content}->{$vtype};
2711    });
2712 }
2713
2714 # list nodes where all VM images are available (used by has_feature API)
2715 sub shared_nodes {
2716     my ($conf, $storecfg) = @_;
2717
2718     my $nodelist = PVE::Cluster::get_nodelist();
2719     my $nodehash = { map { $_ => 1 } @$nodelist };
2720     my $nodename = nodename();
2721
2722     PVE::QemuConfig->foreach_volume($conf, sub {
2723         my ($ds, $drive) = @_;
2724
2725         my $volid = $drive->{file};
2726         return if !$volid;
2727
2728         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2729         if ($storeid) {
2730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2731             if ($scfg->{disable}) {
2732                 $nodehash = {};
2733             } elsif (my $avail = $scfg->{nodes}) {
2734                 foreach my $node (keys %$nodehash) {
2735                     delete $nodehash->{$node} if !$avail->{$node};
2736                 }
2737             } elsif (!$scfg->{shared}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     delete $nodehash->{$node} if $node ne $nodename
2740                 }
2741             }
2742         }
2743     });
2744
2745     return $nodehash
2746 }
2747
2748 sub check_local_storage_availability {
2749     my ($conf, $storecfg) = @_;
2750
2751     my $nodelist = PVE::Cluster::get_nodelist();
2752     my $nodehash = { map { $_ => {} } @$nodelist };
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763
2764             if ($scfg->{disable}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2767                 }
2768             } elsif (my $avail = $scfg->{nodes}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     if (!$avail->{$node}) {
2771                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2772                     }
2773                 }
2774             }
2775         }
2776     });
2777
2778     foreach my $node (values %$nodehash) {
2779         if (my $unavail = $node->{unavailable_storages}) {
2780             $node->{unavailable_storages} = [ sort keys %$unavail ];
2781         }
2782     }
2783
2784     return $nodehash
2785 }
2786
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2788 sub check_running {
2789     my ($vmid, $nocheck, $node) = @_;
2790
2791     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2792     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2793 }
2794
2795 sub vzlist {
2796
2797     my $vzlist = config_list();
2798
2799     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2800
2801     while (defined(my $de = $fd->read)) {
2802         next if $de !~ m/^(\d+)\.pid$/;
2803         my $vmid = $1;
2804         next if !defined($vzlist->{$vmid});
2805         if (my $pid = check_running($vmid)) {
2806             $vzlist->{$vmid}->{pid} = $pid;
2807         }
2808     }
2809
2810     return $vzlist;
2811 }
2812
2813 our $vmstatus_return_properties = {
2814     vmid => get_standard_option('pve-vmid'),
2815     status => {
2816         description => "Qemu process status.",
2817         type => 'string',
2818         enum => ['stopped', 'running'],
2819     },
2820     maxmem => {
2821         description => "Maximum memory in bytes.",
2822         type => 'integer',
2823         optional => 1,
2824         renderer => 'bytes',
2825     },
2826     maxdisk => {
2827         description => "Root disk size in bytes.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'bytes',
2831     },
2832     name => {
2833         description => "VM name.",
2834         type => 'string',
2835         optional => 1,
2836     },
2837     qmpstatus => {
2838         description => "Qemu QMP agent status.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     pid => {
2843         description => "PID of running qemu process.",
2844         type => 'integer',
2845         optional => 1,
2846     },
2847     uptime => {
2848         description => "Uptime.",
2849         type => 'integer',
2850         optional => 1,
2851         renderer => 'duration',
2852     },
2853     cpus => {
2854         description => "Maximum usable CPUs.",
2855         type => 'number',
2856         optional => 1,
2857     },
2858     lock => {
2859         description => "The current config lock, if any.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     tags => {
2864         description  => "The current configured tags, if any",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     'running-machine' => {
2869         description => "The currently running machine type (if running).",
2870         type => 'string',
2871         optional => 1,
2872     },
2873     'running-qemu' => {
2874         description => "The currently running QEMU version (if running).",
2875         type => 'string',
2876         optional => 1,
2877     },
2878 };
2879
2880 my $last_proc_pid_stat;
2881
2882 # get VM status information
2883 # This must be fast and should not block ($full == false)
2884 # We only query KVM using QMP if $full == true (this can be slow)
2885 sub vmstatus {
2886     my ($opt_vmid, $full) = @_;
2887
2888     my $res = {};
2889
2890     my $storecfg = PVE::Storage::config();
2891
2892     my $list = vzlist();
2893     my $defaults = load_defaults();
2894
2895     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2896
2897     my $cpucount = $cpuinfo->{cpus} || 1;
2898
2899     foreach my $vmid (keys %$list) {
2900         next if $opt_vmid && ($vmid ne $opt_vmid);
2901
2902         my $conf = PVE::QemuConfig->load_config($vmid);
2903
2904         my $d = { vmid => int($vmid) };
2905         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2906
2907         # fixme: better status?
2908         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2909
2910         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2911         if (defined($size)) {
2912             $d->{disk} = 0; # no info available
2913             $d->{maxdisk} = $size;
2914         } else {
2915             $d->{disk} = 0;
2916             $d->{maxdisk} = 0;
2917         }
2918
2919         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2920             * ($conf->{cores} || $defaults->{cores});
2921         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2922         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2923
2924         $d->{name} = $conf->{name} || "VM $vmid";
2925         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2926             : $defaults->{memory}*(1024*1024);
2927
2928         if ($conf->{balloon}) {
2929             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2930             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2931                 : $defaults->{shares};
2932         }
2933
2934         $d->{uptime} = 0;
2935         $d->{cpu} = 0;
2936         $d->{mem} = 0;
2937
2938         $d->{netout} = 0;
2939         $d->{netin} = 0;
2940
2941         $d->{diskread} = 0;
2942         $d->{diskwrite} = 0;
2943
2944         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2945
2946         $d->{serial} = 1 if conf_has_serial($conf);
2947         $d->{lock} = $conf->{lock} if $conf->{lock};
2948         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2949
2950         $res->{$vmid} = $d;
2951     }
2952
2953     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2954     foreach my $dev (keys %$netdev) {
2955         next if $dev !~ m/^tap([1-9]\d*)i/;
2956         my $vmid = $1;
2957         my $d = $res->{$vmid};
2958         next if !$d;
2959
2960         $d->{netout} += $netdev->{$dev}->{receive};
2961         $d->{netin} += $netdev->{$dev}->{transmit};
2962
2963         if ($full) {
2964             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2965             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2966         }
2967
2968     }
2969
2970     my $ctime = gettimeofday;
2971
2972     foreach my $vmid (keys %$list) {
2973
2974         my $d = $res->{$vmid};
2975         my $pid = $d->{pid};
2976         next if !$pid;
2977
2978         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2979         next if !$pstat; # not running
2980
2981         my $used = $pstat->{utime} + $pstat->{stime};
2982
2983         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2984
2985         if ($pstat->{vsize}) {
2986             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2987         }
2988
2989         my $old = $last_proc_pid_stat->{$pid};
2990         if (!$old) {
2991             $last_proc_pid_stat->{$pid} = {
2992                 time => $ctime,
2993                 used => $used,
2994                 cpu => 0,
2995             };
2996             next;
2997         }
2998
2999         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3000
3001         if ($dtime > 1000) {
3002             my $dutime = $used -  $old->{used};
3003
3004             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3005             $last_proc_pid_stat->{$pid} = {
3006                 time => $ctime,
3007                 used => $used,
3008                 cpu => $d->{cpu},
3009             };
3010         } else {
3011             $d->{cpu} = $old->{cpu};
3012         }
3013     }
3014
3015     return $res if !$full;
3016
3017     my $qmpclient = PVE::QMPClient->new();
3018
3019     my $ballooncb = sub {
3020         my ($vmid, $resp) = @_;
3021
3022         my $info = $resp->{'return'};
3023         return if !$info->{max_mem};
3024
3025         my $d = $res->{$vmid};
3026
3027         # use memory assigned to VM
3028         $d->{maxmem} = $info->{max_mem};
3029         $d->{balloon} = $info->{actual};
3030
3031         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3032             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3033             $d->{freemem} = $info->{free_mem};
3034         }
3035
3036         $d->{ballooninfo} = $info;
3037     };
3038
3039     my $blockstatscb = sub {
3040         my ($vmid, $resp) = @_;
3041         my $data = $resp->{'return'} || [];
3042         my $totalrdbytes = 0;
3043         my $totalwrbytes = 0;
3044
3045         for my $blockstat (@$data) {
3046             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3047             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3048
3049             $blockstat->{device} =~ s/drive-//;
3050             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3051         }
3052         $res->{$vmid}->{diskread} = $totalrdbytes;
3053         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3054     };
3055
3056     my $machinecb = sub {
3057         my ($vmid, $resp) = @_;
3058         my $data = $resp->{'return'} || [];
3059
3060         $res->{$vmid}->{'running-machine'} =
3061             PVE::QemuServer::Machine::current_from_query_machines($data);
3062     };
3063
3064     my $versioncb = sub {
3065         my ($vmid, $resp) = @_;
3066         my $data = $resp->{'return'} // {};
3067         my $version = 'unknown';
3068
3069         if (my $v = $data->{qemu}) {
3070             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3071         }
3072
3073         $res->{$vmid}->{'running-qemu'} = $version;
3074     };
3075
3076     my $statuscb = sub {
3077         my ($vmid, $resp) = @_;
3078
3079         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3080         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3081         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3082         # this fails if ballon driver is not loaded, so this must be
3083         # the last commnand (following command are aborted if this fails).
3084         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3085
3086         my $status = 'unknown';
3087         if (!defined($status = $resp->{'return'}->{status})) {
3088             warn "unable to get VM status\n";
3089             return;
3090         }
3091
3092         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3093     };
3094
3095     foreach my $vmid (keys %$list) {
3096         next if $opt_vmid && ($vmid ne $opt_vmid);
3097         next if !$res->{$vmid}->{pid}; # not running
3098         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3099     }
3100
3101     $qmpclient->queue_execute(undef, 2);
3102
3103     foreach my $vmid (keys %$list) {
3104         next if $opt_vmid && ($vmid ne $opt_vmid);
3105         next if !$res->{$vmid}->{pid}; #not running
3106
3107         # we can't use the $qmpclient since it might have already aborted on
3108         # 'query-balloon', but this might also fail for older versions...
3109         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3110         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3111     }
3112
3113     foreach my $vmid (keys %$list) {
3114         next if $opt_vmid && ($vmid ne $opt_vmid);
3115         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3116     }
3117
3118     return $res;
3119 }
3120
3121 sub conf_has_serial {
3122     my ($conf) = @_;
3123
3124     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3125         if ($conf->{"serial$i"}) {
3126             return 1;
3127         }
3128     }
3129
3130     return 0;
3131 }
3132
3133 sub conf_has_audio {
3134     my ($conf, $id) = @_;
3135
3136     $id //= 0;
3137     my $audio = $conf->{"audio$id"};
3138     return if !defined($audio);
3139
3140     my $audioproperties = parse_property_string($audio_fmt, $audio);
3141     my $audiodriver = $audioproperties->{driver} // 'spice';
3142
3143     return {
3144         dev => $audioproperties->{device},
3145         dev_id => "audiodev$id",
3146         backend => $audiodriver,
3147         backend_id => "$audiodriver-backend${id}",
3148     };
3149 }
3150
3151 sub audio_devs {
3152     my ($audio, $audiopciaddr, $machine_version) = @_;
3153
3154     my $devs = [];
3155
3156     my $id = $audio->{dev_id};
3157     my $audiodev = "";
3158     if (min_version($machine_version, 4, 2)) {
3159         $audiodev = ",audiodev=$audio->{backend_id}";
3160     }
3161
3162     if ($audio->{dev} eq 'AC97') {
3163         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3164     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3165         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3166         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3167         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3168     } else {
3169         die "unkown audio device '$audio->{dev}', implement me!";
3170     }
3171
3172     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3173
3174     return $devs;
3175 }
3176
3177 sub get_tpm_paths {
3178     my ($vmid) = @_;
3179     return {
3180         socket => "/var/run/qemu-server/$vmid.swtpm",
3181         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3182     };
3183 }
3184
3185 sub add_tpm_device {
3186     my ($vmid, $devices, $conf) = @_;
3187
3188     return if !$conf->{tpmstate0};
3189
3190     my $paths = get_tpm_paths($vmid);
3191
3192     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3193     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3194     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3195 }
3196
3197 sub start_swtpm {
3198     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3199
3200     return if !$tpmdrive;
3201
3202     my $state;
3203     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3204     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3205     if ($storeid) {
3206         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3207     } else {
3208         $state = $tpm->{file};
3209     }
3210
3211     my $paths = get_tpm_paths($vmid);
3212
3213     # during migration, we will get state from remote
3214     #
3215     if (!$migration) {
3216         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3217         my $setup_cmd = [
3218             "swtpm_setup",
3219             "--tpmstate",
3220             "file://$state",
3221             "--createek",
3222             "--create-ek-cert",
3223             "--create-platform-cert",
3224             "--lock-nvram",
3225             "--config",
3226             "/etc/swtpm_setup.conf", # do not use XDG configs
3227             "--runas",
3228             "0", # force creation as root, error if not possible
3229             "--not-overwrite", # ignore existing state, do not modify
3230         ];
3231
3232         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3233         # TPM 2.0 supports ECC crypto, use if possible
3234         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3235
3236         run_command($setup_cmd, outfunc => sub {
3237             print "swtpm_setup: $1\n";
3238         });
3239     }
3240
3241     my $emulator_cmd = [
3242         "swtpm",
3243         "socket",
3244         "--tpmstate",
3245         "backend-uri=file://$state,mode=0600",
3246         "--ctrl",
3247         "type=unixio,path=$paths->{socket},mode=0600",
3248         "--pid",
3249         "file=$paths->{pid}",
3250         "--terminate", # terminate on QEMU disconnect
3251         "--daemon",
3252     ];
3253     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3254     run_command($emulator_cmd, outfunc => sub { print $1; });
3255
3256     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3257     while (! -e $paths->{pid}) {
3258         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3259         usleep(50_000);
3260     }
3261
3262     # return untainted PID of swtpm daemon so it can be killed on error
3263     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3264     return $1;
3265 }
3266
3267 sub vga_conf_has_spice {
3268     my ($vga) = @_;
3269
3270     my $vgaconf = parse_vga($vga);
3271     my $vgatype = $vgaconf->{type};
3272     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3273
3274     return $1 || 1;
3275 }
3276
3277 sub is_native($) {
3278     my ($arch) = @_;
3279     return get_host_arch() eq $arch;
3280 }
3281
3282 sub get_vm_arch {
3283     my ($conf) = @_;
3284     return $conf->{arch} // get_host_arch();
3285 }
3286
3287 my $default_machines = {
3288     x86_64 => 'pc',
3289     aarch64 => 'virt',
3290 };
3291
3292 sub get_installed_machine_version {
3293     my ($kvmversion) = @_;
3294     $kvmversion = kvm_user_version() if !defined($kvmversion);
3295     $kvmversion =~ m/^(\d+\.\d+)/;
3296     return $1;
3297 }
3298
3299 sub windows_get_pinned_machine_version {
3300     my ($machine, $base_version, $kvmversion) = @_;
3301
3302     my $pin_version = $base_version;
3303     if (!defined($base_version) ||
3304         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3305     ) {
3306         $pin_version = get_installed_machine_version($kvmversion);
3307     }
3308     if (!$machine || $machine eq 'pc') {
3309         $machine = "pc-i440fx-$pin_version";
3310     } elsif ($machine eq 'q35') {
3311         $machine = "pc-q35-$pin_version";
3312     } elsif ($machine eq 'virt') {
3313         $machine = "virt-$pin_version";
3314     } else {
3315         warn "unknown machine type '$machine', not touching that!\n";
3316     }
3317
3318     return $machine;
3319 }
3320
3321 sub get_vm_machine {
3322     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3323
3324     my $machine = $forcemachine || $conf->{machine};
3325
3326     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3327         $kvmversion //= kvm_user_version();
3328         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3329         # layout which confuses windows quite a bit and may result in various regressions..
3330         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3331         if (windows_version($conf->{ostype})) {
3332             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3333         }
3334         $arch //= 'x86_64';
3335         $machine ||= $default_machines->{$arch};
3336         if ($add_pve_version) {
3337             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3338             $machine .= "+pve$pvever";
3339         }
3340     }
3341
3342     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3343         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3344         $machine = $1 if $is_pxe;
3345
3346         # for version-pinned machines that do not include a pve-version (e.g.
3347         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3348         $machine .= '+pve0';
3349
3350         $machine .= '.pxe' if $is_pxe;
3351     }
3352
3353     return $machine;
3354 }
3355
3356 sub get_ovmf_files($$$) {
3357     my ($arch, $efidisk, $smm) = @_;
3358
3359     my $types = $OVMF->{$arch}
3360         or die "no OVMF images known for architecture '$arch'\n";
3361
3362     my $type = 'default';
3363     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3364         $type = $smm ? "4m" : "4m-no-smm";
3365         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3366     }
3367
3368     return $types->{$type}->@*;
3369 }
3370
3371 my $Arch2Qemu = {
3372     aarch64 => '/usr/bin/qemu-system-aarch64',
3373     x86_64 => '/usr/bin/qemu-system-x86_64',
3374 };
3375 sub get_command_for_arch($) {
3376     my ($arch) = @_;
3377     return '/usr/bin/kvm' if is_native($arch);
3378
3379     my $cmd = $Arch2Qemu->{$arch}
3380         or die "don't know how to emulate architecture '$arch'\n";
3381     return $cmd;
3382 }
3383
3384 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3385 # to use in a QEMU command line (-cpu element), first array_intersect the result
3386 # of query_supported_ with query_understood_. This is necessary because:
3387 #
3388 # a) query_understood_ returns flags the host cannot use and
3389 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3390 #    flags, but CPU settings - with most of them being flags. Those settings
3391 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3392 #
3393 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3394 # expensive. If you need the value returned from this, you can get it much
3395 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3396 # $accel being 'kvm' or 'tcg'.
3397 #
3398 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3399 # changes, automatically populating pmxcfs.
3400 #
3401 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3402 # since kvm and tcg machines support different flags
3403 #
3404 sub query_supported_cpu_flags {
3405     my ($arch) = @_;
3406
3407     $arch //= get_host_arch();
3408     my $default_machine = $default_machines->{$arch};
3409
3410     my $flags = {};
3411
3412     # FIXME: Once this is merged, the code below should work for ARM as well:
3413     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3414     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3415         $arch eq "aarch64";
3416
3417     my $kvm_supported = defined(kvm_version());
3418     my $qemu_cmd = get_command_for_arch($arch);
3419     my $fakevmid = -1;
3420     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3421
3422     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3423     my $query_supported_run_qemu = sub {
3424         my ($kvm) = @_;
3425
3426         my $flags = {};
3427         my $cmd = [
3428             $qemu_cmd,
3429             '-machine', $default_machine,
3430             '-display', 'none',
3431             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3432             '-mon', 'chardev=qmp,mode=control',
3433             '-pidfile', $pidfile,
3434             '-S', '-daemonize'
3435         ];
3436
3437         if (!$kvm) {
3438             push @$cmd, '-accel', 'tcg';
3439         }
3440
3441         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3442         die "QEMU flag querying VM exited with code " . $rc if $rc;
3443
3444         eval {
3445             my $cmd_result = mon_cmd(
3446                 $fakevmid,
3447                 'query-cpu-model-expansion',
3448                 type => 'full',
3449                 model => { name => 'host' }
3450             );
3451
3452             my $props = $cmd_result->{model}->{props};
3453             foreach my $prop (keys %$props) {
3454                 next if $props->{$prop} ne '1';
3455                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3456                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3457                 # We only keep those with underscores, to match /proc/cpuinfo
3458                 $prop =~ s/\.|-/_/g;
3459                 $flags->{$prop} = 1;
3460             }
3461         };
3462         my $err = $@;
3463
3464         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3465         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3466
3467         die $err if $err;
3468
3469         return [ sort keys %$flags ];
3470     };
3471
3472     # We need to query QEMU twice, since KVM and TCG have different supported flags
3473     PVE::QemuConfig->lock_config($fakevmid, sub {
3474         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3475         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3476
3477         if ($kvm_supported) {
3478             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3479             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3480         }
3481     });
3482
3483     return $flags;
3484 }
3485
3486 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3487 my $understood_cpu_flag_dir = "/usr/share/kvm";
3488 sub query_understood_cpu_flags {
3489     my $arch = get_host_arch();
3490     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3491
3492     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3493         if ! -e $filepath;
3494
3495     my $raw = file_get_contents($filepath);
3496     $raw =~ s/^\s+|\s+$//g;
3497     my @flags = split(/\s+/, $raw);
3498
3499     return \@flags;
3500 }
3501
3502 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3503 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3504 my sub should_disable_smm {
3505     my ($conf, $vga) = @_;
3506
3507     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3508         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3509 }
3510
3511 sub config_to_command {
3512     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3513         $pbs_backing) = @_;
3514
3515     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3516     my $devices = [];
3517     my $bridges = {};
3518     my $ostype = $conf->{ostype};
3519     my $winversion = windows_version($ostype);
3520     my $kvm = $conf->{kvm};
3521     my $nodename = nodename();
3522
3523     my $arch = get_vm_arch($conf);
3524     my $kvm_binary = get_command_for_arch($arch);
3525     my $kvmver = kvm_user_version($kvm_binary);
3526
3527     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3528         $kvmver //= "undefined";
3529         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3530     }
3531
3532     my $add_pve_version = min_version($kvmver, 4, 1);
3533
3534     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3535     my $machine_version = extract_version($machine_type, $kvmver);
3536     $kvm //= 1 if is_native($arch);
3537
3538     $machine_version =~ m/(\d+)\.(\d+)/;
3539     my ($machine_major, $machine_minor) = ($1, $2);
3540
3541     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3542         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3543     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3544         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3545             ." please upgrade node '$nodename'\n"
3546     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3547         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3548         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3549             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3550             ." node '$nodename'\n";
3551     }
3552
3553     # if a specific +pve version is required for a feature, use $version_guard
3554     # instead of min_version to allow machines to be run with the minimum
3555     # required version
3556     my $required_pve_version = 0;
3557     my $version_guard = sub {
3558         my ($major, $minor, $pve) = @_;
3559         return 0 if !min_version($machine_version, $major, $minor, $pve);
3560         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3561         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3562         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3563         return 1;
3564     };
3565
3566     if ($kvm && !defined kvm_version()) {
3567         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3568             ." or enable in BIOS.\n";
3569     }
3570
3571     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3572     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3573     my $use_old_bios_files = undef;
3574     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3575
3576     my $cmd = [];
3577     if ($conf->{affinity}) {
3578         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3579     }
3580
3581     push @$cmd, $kvm_binary;
3582
3583     push @$cmd, '-id', $vmid;
3584
3585     my $vmname = $conf->{name} || "vm$vmid";
3586
3587     push @$cmd, '-name', "$vmname,debug-threads=on";
3588
3589     push @$cmd, '-no-shutdown';
3590
3591     my $use_virtio = 0;
3592
3593     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3594     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3595     push @$cmd, '-mon', "chardev=qmp,mode=control";
3596
3597     if (min_version($machine_version, 2, 12)) {
3598         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3599         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3600     }
3601
3602     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3603
3604     push @$cmd, '-daemonize';
3605
3606     if ($conf->{smbios1}) {
3607         my $smbios_conf = parse_smbios1($conf->{smbios1});
3608         if ($smbios_conf->{base64}) {
3609             # Do not pass base64 flag to qemu
3610             delete $smbios_conf->{base64};
3611             my $smbios_string = "";
3612             foreach my $key (keys %$smbios_conf) {
3613                 my $value;
3614                 if ($key eq "uuid") {
3615                     $value = $smbios_conf->{uuid}
3616                 } else {
3617                     $value = decode_base64($smbios_conf->{$key});
3618                 }
3619                 # qemu accepts any binary data, only commas need escaping by double comma
3620                 $value =~ s/,/,,/g;
3621                 $smbios_string .= "," . $key . "=" . $value if $value;
3622             }
3623             push @$cmd, '-smbios', "type=1" . $smbios_string;
3624         } else {
3625             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3626         }
3627     }
3628
3629     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3630         my $d;
3631         if (my $efidisk = $conf->{efidisk0}) {
3632             $d = parse_drive('efidisk0', $efidisk);
3633         }
3634
3635         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3636         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3637
3638         my ($path, $format);
3639         my $read_only_str = '';
3640         if ($d) {
3641             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3642             $format = $d->{format};
3643             if ($storeid) {
3644                 $path = PVE::Storage::path($storecfg, $d->{file});
3645                 if (!defined($format)) {
3646                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3647                     $format = qemu_img_format($scfg, $volname);
3648                 }
3649             } else {
3650                 $path = $d->{file};
3651                 die "efidisk format must be specified\n"
3652                     if !defined($format);
3653             }
3654
3655             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3656         } else {
3657             log_warn("no efidisk configured! Using temporary efivars disk.");
3658             $path = "/tmp/$vmid-ovmf.fd";
3659             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3660             $format = 'raw';
3661         }
3662
3663         my $size_str = "";
3664
3665         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3666             $size_str = ",size=" . (-s $ovmf_vars);
3667         }
3668
3669         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3670         my $cache = "";
3671         if ($path =~ m/^rbd:/) {
3672                 $cache = ',cache=writeback';
3673                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3674         }
3675
3676         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3677         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3678     }
3679
3680     if ($q35) { # tell QEMU to load q35 config early
3681         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3682         if (min_version($machine_version, 4, 0)) {
3683             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3684         } else {
3685             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3686         }
3687     }
3688
3689     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3690         push @$cmd, $fixups->@*;
3691     }
3692
3693     if ($conf->{vmgenid}) {
3694         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3695     }
3696
3697     # add usb controllers
3698     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3699         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3700     push @$devices, @usbcontrollers if @usbcontrollers;
3701     my $vga = parse_vga($conf->{vga});
3702
3703     my $qxlnum = vga_conf_has_spice($conf->{vga});
3704     $vga->{type} = 'qxl' if $qxlnum;
3705
3706     if (!$vga->{type}) {
3707         if ($arch eq 'aarch64') {
3708             $vga->{type} = 'virtio';
3709         } elsif (min_version($machine_version, 2, 9)) {
3710             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3711         } else {
3712             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3713         }
3714     }
3715
3716     # enable absolute mouse coordinates (needed by vnc)
3717     my $tablet = $conf->{tablet};
3718     if (!defined($tablet)) {
3719         $tablet = $defaults->{tablet};
3720         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3721         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3722     }
3723
3724     if ($tablet) {
3725         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3726         my $kbd = print_keyboarddevice_full($conf, $arch);
3727         push @$devices, '-device', $kbd if defined($kbd);
3728     }
3729
3730     my $bootorder = device_bootorder($conf);
3731
3732     # host pci device passthrough
3733     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3734         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3735
3736     # usb devices
3737     my $usb_dev_features = {};
3738     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3739
3740     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3741         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3742     push @$devices, @usbdevices if @usbdevices;
3743
3744     # serial devices
3745     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3746         my $path = $conf->{"serial$i"} or next;
3747         if ($path eq 'socket') {
3748             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3749             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3750             # On aarch64, serial0 is the UART device. Qemu only allows
3751             # connecting UART devices via the '-serial' command line, as
3752             # the device has a fixed slot on the hardware...
3753             if ($arch eq 'aarch64' && $i == 0) {
3754                 push @$devices, '-serial', "chardev:serial$i";
3755             } else {
3756                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3757             }
3758         } else {
3759             die "no such serial device\n" if ! -c $path;
3760             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3761             push @$devices, '-device', "isa-serial,chardev=serial$i";
3762         }
3763     }
3764
3765     # parallel devices
3766     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3767         if (my $path = $conf->{"parallel$i"}) {
3768             die "no such parallel device\n" if ! -c $path;
3769             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3770             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3771             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3772         }
3773     }
3774
3775     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3776         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3777         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3778         push @$devices, @$audio_devs;
3779     }
3780
3781     add_tpm_device($vmid, $devices, $conf);
3782
3783     my $sockets = 1;
3784     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3785     $sockets = $conf->{sockets} if  $conf->{sockets};
3786
3787     my $cores = $conf->{cores} || 1;
3788
3789     my $maxcpus = $sockets * $cores;
3790
3791     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3792
3793     my $allowed_vcpus = $cpuinfo->{cpus};
3794
3795     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3796
3797     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3798         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3799         for (my $i = 2; $i <= $vcpus; $i++)  {
3800             my $cpustr = print_cpu_device($conf,$i);
3801             push @$cmd, '-device', $cpustr;
3802         }
3803
3804     } else {
3805
3806         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3807     }
3808     push @$cmd, '-nodefaults';
3809
3810     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3811
3812     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3813
3814     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3815
3816     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3817         push @$devices, '-device', print_vga_device(
3818             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3819
3820         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3821
3822         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3823         push @$cmd,  '-vnc', "unix:$socket,password=on";
3824     } else {
3825         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3826         push @$cmd, '-nographic';
3827     }
3828
3829     # time drift fix
3830     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3831     my $useLocaltime = $conf->{localtime};
3832
3833     if ($winversion >= 5) { # windows
3834         $useLocaltime = 1 if !defined($conf->{localtime});
3835
3836         # use time drift fix when acpi is enabled
3837         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3838             $tdf = 1 if !defined($conf->{tdf});
3839         }
3840     }
3841
3842     if ($winversion >= 6) {
3843         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3844         push @$cmd, '-no-hpet';
3845     }
3846
3847     push @$rtcFlags, 'driftfix=slew' if $tdf;
3848
3849     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3850         push @$rtcFlags, "base=$conf->{startdate}";
3851     } elsif ($useLocaltime) {
3852         push @$rtcFlags, 'base=localtime';
3853     }
3854
3855     if ($forcecpu) {
3856         push @$cmd, '-cpu', $forcecpu;
3857     } else {
3858         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3859     }
3860
3861     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3862
3863     push @$cmd, '-S' if $conf->{freeze};
3864
3865     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3866
3867     my $guest_agent = parse_guest_agent($conf);
3868
3869     if ($guest_agent->{enabled}) {
3870         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3871         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3872
3873         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3874             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3875             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3876             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3877         } elsif ($guest_agent->{type} eq 'isa') {
3878             push @$devices, '-device', "isa-serial,chardev=qga0";
3879         }
3880     }
3881
3882     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3883     if ($rng && $version_guard->(4, 1, 2)) {
3884         check_rng_source($rng->{source});
3885
3886         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3887         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3888         my $limiter_str = "";
3889         if ($max_bytes) {
3890             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3891         }
3892
3893         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3894         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3895         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3896     }
3897
3898     my $spice_port;
3899
3900     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3901         if ($qxlnum > 1) {
3902             if ($winversion){
3903                 for (my $i = 1; $i < $qxlnum; $i++){
3904                     push @$devices, '-device', print_vga_device(
3905                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3906                 }
3907             } else {
3908                 # assume other OS works like Linux
3909                 my ($ram, $vram) = ("134217728", "67108864");
3910                 if ($vga->{memory}) {
3911                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3912                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3913                 }
3914                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3915                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3916             }
3917         }
3918
3919         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3920
3921         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3922         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3923         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3924
3925         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3926         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3927         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3928
3929         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3930         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3931
3932         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3933         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3934         if ($spice_enhancement->{foldersharing}) {
3935             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3936             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3937         }
3938
3939         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3940         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3941             if $spice_enhancement->{videostreaming};
3942
3943         push @$devices, '-spice', "$spice_opts";
3944     }
3945
3946     # enable balloon by default, unless explicitly disabled
3947     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3948         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3949         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3950         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3951         push @$devices, '-device', $ballooncmd;
3952     }
3953
3954     if ($conf->{watchdog}) {
3955         my $wdopts = parse_watchdog($conf->{watchdog});
3956         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3957         my $watchdog = $wdopts->{model} || 'i6300esb';
3958         push @$devices, '-device', "$watchdog$pciaddr";
3959         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3960     }
3961
3962     my $vollist = [];
3963     my $scsicontroller = {};
3964     my $ahcicontroller = {};
3965     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3966
3967     # Add iscsi initiator name if available
3968     if (my $initiator = get_initiator_name()) {
3969         push @$devices, '-iscsi', "initiator-name=$initiator";
3970     }
3971
3972     PVE::QemuConfig->foreach_volume($conf, sub {
3973         my ($ds, $drive) = @_;
3974
3975         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3976             check_volume_storage_type($storecfg, $drive->{file});
3977             push @$vollist, $drive->{file};
3978         }
3979
3980         # ignore efidisk here, already added in bios/fw handling code above
3981         return if $drive->{interface} eq 'efidisk';
3982         # similar for TPM
3983         return if $drive->{interface} eq 'tpmstate';
3984
3985         $use_virtio = 1 if $ds =~ m/^virtio/;
3986
3987         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3988
3989         if ($drive->{interface} eq 'virtio'){
3990            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3991         }
3992
3993         if ($drive->{interface} eq 'scsi') {
3994
3995             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3996
3997             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3998                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3999
4000             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4001             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4002
4003             my $iothread = '';
4004             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4005                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4006                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4007             } elsif ($drive->{iothread}) {
4008                 log_warn(
4009                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4010                 );
4011             }
4012
4013             my $queues = '';
4014             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4015                 $queues = ",num_queues=$drive->{queues}";
4016             }
4017
4018             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4019                 if !$scsicontroller->{$controller};
4020             $scsicontroller->{$controller}=1;
4021         }
4022
4023         if ($drive->{interface} eq 'sata') {
4024             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4025             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4026             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4027                 if !$ahcicontroller->{$controller};
4028             $ahcicontroller->{$controller}=1;
4029         }
4030
4031         my $pbs_conf = $pbs_backing->{$ds};
4032         my $pbs_name = undef;
4033         if ($pbs_conf) {
4034             $pbs_name = "drive-$ds-pbs";
4035             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4036         }
4037
4038         my $drive_cmd = print_drive_commandline_full(
4039             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4040
4041         # extra protection for templates, but SATA and IDE don't support it..
4042         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4043
4044         push @$devices, '-drive',$drive_cmd;
4045         push @$devices, '-device', print_drivedevice_full(
4046             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4047     });
4048
4049     for (my $i = 0; $i < $MAX_NETS; $i++) {
4050         my $netname = "net$i";
4051
4052         next if !$conf->{$netname};
4053         my $d = parse_net($conf->{$netname});
4054         next if !$d;
4055         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4056
4057         $use_virtio = 1 if $d->{model} eq 'virtio';
4058
4059         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4060
4061         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4062         push @$devices, '-netdev', $netdevfull;
4063
4064         my $netdevicefull = print_netdevice_full(
4065             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4066
4067         push @$devices, '-device', $netdevicefull;
4068     }
4069
4070     if ($conf->{ivshmem}) {
4071         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4072
4073         my $bus;
4074         if ($q35) {
4075             $bus = print_pcie_addr("ivshmem");
4076         } else {
4077             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4078         }
4079
4080         my $ivshmem_name = $ivshmem->{name} // $vmid;
4081         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4082
4083         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4084         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4085             .",size=$ivshmem->{size}M";
4086     }
4087
4088     # pci.4 is nested in pci.1
4089     $bridges->{1} = 1 if $bridges->{4};
4090
4091     if (!$q35) { # add pci bridges
4092         if (min_version($machine_version, 2, 3)) {
4093            $bridges->{1} = 1;
4094            $bridges->{2} = 1;
4095         }
4096         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4097     }
4098
4099     for my $k (sort {$b cmp $a} keys %$bridges) {
4100         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4101
4102         my $k_name = $k;
4103         if ($k == 2 && $legacy_igd) {
4104             $k_name = "$k-igd";
4105         }
4106         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4107         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4108
4109         if ($q35) { # add after -readconfig pve-q35.cfg
4110             splice @$devices, 2, 0, '-device', $devstr;
4111         } else {
4112             unshift @$devices, '-device', $devstr if $k > 0;
4113         }
4114     }
4115
4116     if (!$kvm) {
4117         push @$machineFlags, 'accel=tcg';
4118     }
4119
4120     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4121
4122     my $machine_type_min = $machine_type;
4123     if ($add_pve_version) {
4124         $machine_type_min =~ s/\+pve\d+$//;
4125         $machine_type_min .= "+pve$required_pve_version";
4126     }
4127     push @$machineFlags, "type=${machine_type_min}";
4128
4129     push @$cmd, @$devices;
4130     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4131     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4132     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4133
4134     if (my $vmstate = $conf->{vmstate}) {
4135         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4136         push @$vollist, $vmstate;
4137         push @$cmd, '-loadstate', $statepath;
4138         print "activating and using '$vmstate' as vmstate\n";
4139     }
4140
4141     if (PVE::QemuConfig->is_template($conf)) {
4142         # needed to workaround base volumes being read-only
4143         push @$cmd, '-snapshot';
4144     }
4145
4146     # add custom args
4147     if ($conf->{args}) {
4148         my $aa = PVE::Tools::split_args($conf->{args});
4149         push @$cmd, @$aa;
4150     }
4151
4152     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4153 }
4154
4155 sub check_rng_source {
4156     my ($source) = @_;
4157
4158     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4159     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4160         if ! -e $source;
4161
4162     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4163     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4164         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4165         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4166         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4167             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4168             ." to the host.\n";
4169     }
4170 }
4171
4172 sub spice_port {
4173     my ($vmid) = @_;
4174
4175     my $res = mon_cmd($vmid, 'query-spice');
4176
4177     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4178 }
4179
4180 sub vm_devices_list {
4181     my ($vmid) = @_;
4182
4183     my $res = mon_cmd($vmid, 'query-pci');
4184     my $devices_to_check = [];
4185     my $devices = {};
4186     foreach my $pcibus (@$res) {
4187         push @$devices_to_check, @{$pcibus->{devices}},
4188     }
4189
4190     while (@$devices_to_check) {
4191         my $to_check = [];
4192         for my $d (@$devices_to_check) {
4193             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4194             next if !$d->{'pci_bridge'};
4195
4196             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4197             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4198         }
4199         $devices_to_check = $to_check;
4200     }
4201
4202     my $resblock = mon_cmd($vmid, 'query-block');
4203     foreach my $block (@$resblock) {
4204         if($block->{device} =~ m/^drive-(\S+)/){
4205                 $devices->{$1} = 1;
4206         }
4207     }
4208
4209     my $resmice = mon_cmd($vmid, 'query-mice');
4210     foreach my $mice (@$resmice) {
4211         if ($mice->{name} eq 'QEMU HID Tablet') {
4212             $devices->{tablet} = 1;
4213             last;
4214         }
4215     }
4216
4217     # for usb devices there is no query-usb
4218     # but we can iterate over the entries in
4219     # qom-list path=/machine/peripheral
4220     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4221     foreach my $per (@$resperipheral) {
4222         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4223             $devices->{$per->{name}} = 1;
4224         }
4225     }
4226
4227     return $devices;
4228 }
4229
4230 sub vm_deviceplug {
4231     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4232
4233     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4234
4235     my $devices_list = vm_devices_list($vmid);
4236     return 1 if defined($devices_list->{$deviceid});
4237
4238     # add PCI bridge if we need it for the device
4239     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4240
4241     if ($deviceid eq 'tablet') {
4242         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4243     } elsif ($deviceid eq 'keyboard') {
4244         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4245     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4246         my $id = $1;
4247         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4248         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4249     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4250         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4251     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4252         qemu_iothread_add($vmid, $deviceid, $device);
4253
4254         qemu_driveadd($storecfg, $vmid, $device);
4255         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4256
4257         qemu_deviceadd($vmid, $devicefull);
4258         eval { qemu_deviceaddverify($vmid, $deviceid); };
4259         if (my $err = $@) {
4260             eval { qemu_drivedel($vmid, $deviceid); };
4261             warn $@ if $@;
4262             die $err;
4263         }
4264     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4265         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4266         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4267         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4268
4269         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4270
4271         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4272             qemu_iothread_add($vmid, $deviceid, $device);
4273             $devicefull .= ",iothread=iothread-$deviceid";
4274         }
4275
4276         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4277             $devicefull .= ",num_queues=$device->{queues}";
4278         }
4279
4280         qemu_deviceadd($vmid, $devicefull);
4281         qemu_deviceaddverify($vmid, $deviceid);
4282     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4283         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4284         qemu_driveadd($storecfg, $vmid, $device);
4285
4286         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4287         eval { qemu_deviceadd($vmid, $devicefull); };
4288         if (my $err = $@) {
4289             eval { qemu_drivedel($vmid, $deviceid); };
4290             warn $@ if $@;
4291             die $err;
4292         }
4293     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4294         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4295
4296         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4297         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4298         my $use_old_bios_files = undef;
4299         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4300
4301         my $netdevicefull = print_netdevice_full(
4302             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4303         qemu_deviceadd($vmid, $netdevicefull);
4304         eval {
4305             qemu_deviceaddverify($vmid, $deviceid);
4306             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4307         };
4308         if (my $err = $@) {
4309             eval { qemu_netdevdel($vmid, $deviceid); };
4310             warn $@ if $@;
4311             die $err;
4312         }
4313     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4314         my $bridgeid = $2;
4315         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4316         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4317
4318         qemu_deviceadd($vmid, $devicefull);
4319         qemu_deviceaddverify($vmid, $deviceid);
4320     } else {
4321         die "can't hotplug device '$deviceid'\n";
4322     }
4323
4324     return 1;
4325 }
4326
4327 # fixme: this should raise exceptions on error!
4328 sub vm_deviceunplug {
4329     my ($vmid, $conf, $deviceid) = @_;
4330
4331     my $devices_list = vm_devices_list($vmid);
4332     return 1 if !defined($devices_list->{$deviceid});
4333
4334     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4335     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4336
4337     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4338         qemu_devicedel($vmid, $deviceid);
4339     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4340         qemu_devicedel($vmid, $deviceid);
4341         qemu_devicedelverify($vmid, $deviceid);
4342     } elsif ($deviceid =~ m/^usb\d+$/) {
4343         qemu_devicedel($vmid, $deviceid);
4344         qemu_devicedelverify($vmid, $deviceid);
4345     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4346         my $device = parse_drive($deviceid, $conf->{$deviceid});
4347
4348         qemu_devicedel($vmid, $deviceid);
4349         qemu_devicedelverify($vmid, $deviceid);
4350         qemu_drivedel($vmid, $deviceid);
4351         qemu_iothread_del($vmid, $deviceid, $device);
4352     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4353         qemu_devicedel($vmid, $deviceid);
4354         qemu_devicedelverify($vmid, $deviceid);
4355     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4356         my $device = parse_drive($deviceid, $conf->{$deviceid});
4357
4358         qemu_devicedel($vmid, $deviceid);
4359         qemu_devicedelverify($vmid, $deviceid);
4360         qemu_drivedel($vmid, $deviceid);
4361         qemu_deletescsihw($conf, $vmid, $deviceid);
4362
4363         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4364             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4365     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4366         qemu_devicedel($vmid, $deviceid);
4367         qemu_devicedelverify($vmid, $deviceid);
4368         qemu_netdevdel($vmid, $deviceid);
4369     } else {
4370         die "can't unplug device '$deviceid'\n";
4371     }
4372
4373     return 1;
4374 }
4375
4376 sub qemu_spice_usbredir_chardev_add {
4377     my ($vmid, $id) = @_;
4378
4379     mon_cmd($vmid, "chardev-add" , (
4380         id => $id,
4381         backend => {
4382             type => 'spicevmc',
4383             data => {
4384                 type => "usbredir",
4385             },
4386         },
4387     ));
4388 }
4389
4390 sub qemu_deviceadd {
4391     my ($vmid, $devicefull) = @_;
4392
4393     $devicefull = "driver=".$devicefull;
4394     my %options =  split(/[=,]/, $devicefull);
4395
4396     mon_cmd($vmid, "device_add" , %options);
4397 }
4398
4399 sub qemu_devicedel {
4400     my ($vmid, $deviceid) = @_;
4401
4402     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4403 }
4404
4405 sub qemu_iothread_add {
4406     my ($vmid, $deviceid, $device) = @_;
4407
4408     if ($device->{iothread}) {
4409         my $iothreads = vm_iothreads_list($vmid);
4410         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4411     }
4412 }
4413
4414 sub qemu_iothread_del {
4415     my ($vmid, $deviceid, $device) = @_;
4416
4417     if ($device->{iothread}) {
4418         my $iothreads = vm_iothreads_list($vmid);
4419         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4420     }
4421 }
4422
4423 sub qemu_objectadd {
4424     my ($vmid, $objectid, $qomtype) = @_;
4425
4426     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4427
4428     return 1;
4429 }
4430
4431 sub qemu_objectdel {
4432     my ($vmid, $objectid) = @_;
4433
4434     mon_cmd($vmid, "object-del", id => $objectid);
4435
4436     return 1;
4437 }
4438
4439 sub qemu_driveadd {
4440     my ($storecfg, $vmid, $device) = @_;
4441
4442     my $kvmver = get_running_qemu_version($vmid);
4443     my $io_uring = min_version($kvmver, 6, 0);
4444     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4445     $drive =~ s/\\/\\\\/g;
4446     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4447
4448     # If the command succeeds qemu prints: "OK"
4449     return 1 if $ret =~ m/OK/s;
4450
4451     die "adding drive failed: $ret\n";
4452 }
4453
4454 sub qemu_drivedel {
4455     my ($vmid, $deviceid) = @_;
4456
4457     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4458     $ret =~ s/^\s+//;
4459
4460     return 1 if $ret eq "";
4461
4462     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4463     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4464
4465     die "deleting drive $deviceid failed : $ret\n";
4466 }
4467
4468 sub qemu_deviceaddverify {
4469     my ($vmid, $deviceid) = @_;
4470
4471     for (my $i = 0; $i <= 5; $i++) {
4472          my $devices_list = vm_devices_list($vmid);
4473          return 1 if defined($devices_list->{$deviceid});
4474          sleep 1;
4475     }
4476
4477     die "error on hotplug device '$deviceid'\n";
4478 }
4479
4480
4481 sub qemu_devicedelverify {
4482     my ($vmid, $deviceid) = @_;
4483
4484     # need to verify that the device is correctly removed as device_del
4485     # is async and empty return is not reliable
4486
4487     for (my $i = 0; $i <= 5; $i++) {
4488          my $devices_list = vm_devices_list($vmid);
4489          return 1 if !defined($devices_list->{$deviceid});
4490          sleep 1;
4491     }
4492
4493     die "error on hot-unplugging device '$deviceid'\n";
4494 }
4495
4496 sub qemu_findorcreatescsihw {
4497     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4498
4499     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4500
4501     my $scsihwid="$controller_prefix$controller";
4502     my $devices_list = vm_devices_list($vmid);
4503
4504     if (!defined($devices_list->{$scsihwid})) {
4505         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4506     }
4507
4508     return 1;
4509 }
4510
4511 sub qemu_deletescsihw {
4512     my ($conf, $vmid, $opt) = @_;
4513
4514     my $device = parse_drive($opt, $conf->{$opt});
4515
4516     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4517         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4518         return 1;
4519     }
4520
4521     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4522
4523     my $devices_list = vm_devices_list($vmid);
4524     foreach my $opt (keys %{$devices_list}) {
4525         if (is_valid_drivename($opt)) {
4526             my $drive = parse_drive($opt, $conf->{$opt});
4527             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4528                 return 1;
4529             }
4530         }
4531     }
4532
4533     my $scsihwid="scsihw$controller";
4534
4535     vm_deviceunplug($vmid, $conf, $scsihwid);
4536
4537     return 1;
4538 }
4539
4540 sub qemu_add_pci_bridge {
4541     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4542
4543     my $bridges = {};
4544
4545     my $bridgeid;
4546
4547     print_pci_addr($device, $bridges, $arch, $machine_type);
4548
4549     while (my ($k, $v) = each %$bridges) {
4550         $bridgeid = $k;
4551     }
4552     return 1 if !defined($bridgeid) || $bridgeid < 1;
4553
4554     my $bridge = "pci.$bridgeid";
4555     my $devices_list = vm_devices_list($vmid);
4556
4557     if (!defined($devices_list->{$bridge})) {
4558         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4559     }
4560
4561     return 1;
4562 }
4563
4564 sub qemu_set_link_status {
4565     my ($vmid, $device, $up) = @_;
4566
4567     mon_cmd($vmid, "set_link", name => $device,
4568                up => $up ? JSON::true : JSON::false);
4569 }
4570
4571 sub qemu_netdevadd {
4572     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4573
4574     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4575     my %options =  split(/[=,]/, $netdev);
4576
4577     if (defined(my $vhost = $options{vhost})) {
4578         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4579     }
4580
4581     if (defined(my $queues = $options{queues})) {
4582         $options{queues} = $queues + 0;
4583     }
4584
4585     mon_cmd($vmid, "netdev_add",  %options);
4586     return 1;
4587 }
4588
4589 sub qemu_netdevdel {
4590     my ($vmid, $deviceid) = @_;
4591
4592     mon_cmd($vmid, "netdev_del", id => $deviceid);
4593 }
4594
4595 sub qemu_usb_hotplug {
4596     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4597
4598     return if !$device;
4599
4600     # remove the old one first
4601     vm_deviceunplug($vmid, $conf, $deviceid);
4602
4603     # check if xhci controller is necessary and available
4604     my $devicelist = vm_devices_list($vmid);
4605
4606     if (!$devicelist->{xhci}) {
4607         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4608         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4609     }
4610
4611     # print_usbdevice_full expects the parsed device
4612     my $d = parse_usb_device($device->{host});
4613     $d->{usb3} = $device->{usb3};
4614
4615     # add the new one
4616     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4617 }
4618
4619 sub qemu_cpu_hotplug {
4620     my ($vmid, $conf, $vcpus) = @_;
4621
4622     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4623
4624     my $sockets = 1;
4625     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4626     $sockets = $conf->{sockets} if  $conf->{sockets};
4627     my $cores = $conf->{cores} || 1;
4628     my $maxcpus = $sockets * $cores;
4629
4630     $vcpus = $maxcpus if !$vcpus;
4631
4632     die "you can't add more vcpus than maxcpus\n"
4633         if $vcpus > $maxcpus;
4634
4635     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4636
4637     if ($vcpus < $currentvcpus) {
4638
4639         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4640
4641             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4642                 qemu_devicedel($vmid, "cpu$i");
4643                 my $retry = 0;
4644                 my $currentrunningvcpus = undef;
4645                 while (1) {
4646                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4647                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4648                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4649                     $retry++;
4650                     sleep 1;
4651                 }
4652                 #update conf after each succesfull cpu unplug
4653                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4654                 PVE::QemuConfig->write_config($vmid, $conf);
4655             }
4656         } else {
4657             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4658         }
4659
4660         return;
4661     }
4662
4663     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4664     die "vcpus in running vm does not match its configuration\n"
4665         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4666
4667     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4668
4669         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4670             my $cpustr = print_cpu_device($conf, $i);
4671             qemu_deviceadd($vmid, $cpustr);
4672
4673             my $retry = 0;
4674             my $currentrunningvcpus = undef;
4675             while (1) {
4676                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677                 last if scalar(@{$currentrunningvcpus}) == $i;
4678                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4679                 sleep 1;
4680                 $retry++;
4681             }
4682             #update conf after each succesfull cpu hotplug
4683             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684             PVE::QemuConfig->write_config($vmid, $conf);
4685         }
4686     } else {
4687
4688         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4689             mon_cmd($vmid, "cpu-add", id => int($i));
4690         }
4691     }
4692 }
4693
4694 sub qemu_block_set_io_throttle {
4695     my ($vmid, $deviceid,
4696         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4697         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4698         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4699         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4700
4701     return if !check_running($vmid) ;
4702
4703     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4704         bps => int($bps),
4705         bps_rd => int($bps_rd),
4706         bps_wr => int($bps_wr),
4707         iops => int($iops),
4708         iops_rd => int($iops_rd),
4709         iops_wr => int($iops_wr),
4710         bps_max => int($bps_max),
4711         bps_rd_max => int($bps_rd_max),
4712         bps_wr_max => int($bps_wr_max),
4713         iops_max => int($iops_max),
4714         iops_rd_max => int($iops_rd_max),
4715         iops_wr_max => int($iops_wr_max),
4716         bps_max_length => int($bps_max_length),
4717         bps_rd_max_length => int($bps_rd_max_length),
4718         bps_wr_max_length => int($bps_wr_max_length),
4719         iops_max_length => int($iops_max_length),
4720         iops_rd_max_length => int($iops_rd_max_length),
4721         iops_wr_max_length => int($iops_wr_max_length),
4722     );
4723
4724 }
4725
4726 sub qemu_block_resize {
4727     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4728
4729     my $running = check_running($vmid);
4730
4731     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4732
4733     return if !$running;
4734
4735     my $padding = (1024 - $size % 1024) % 1024;
4736     $size = $size + $padding;
4737
4738     mon_cmd(
4739         $vmid,
4740         "block_resize",
4741         device => $deviceid,
4742         size => int($size),
4743         timeout => 60,
4744     );
4745 }
4746
4747 sub qemu_volume_snapshot {
4748     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4749
4750     my $running = check_running($vmid);
4751
4752     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4753         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4754     } else {
4755         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4756     }
4757 }
4758
4759 sub qemu_volume_snapshot_delete {
4760     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4761
4762     my $running = check_running($vmid);
4763
4764     if($running) {
4765
4766         $running = undef;
4767         my $conf = PVE::QemuConfig->load_config($vmid);
4768         PVE::QemuConfig->foreach_volume($conf, sub {
4769             my ($ds, $drive) = @_;
4770             $running = 1 if $drive->{file} eq $volid;
4771         });
4772     }
4773
4774     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4775         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4776     } else {
4777         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4778     }
4779 }
4780
4781 sub set_migration_caps {
4782     my ($vmid, $savevm) = @_;
4783
4784     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4785
4786     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4787     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4788
4789     my $cap_ref = [];
4790
4791     my $enabled_cap = {
4792         "auto-converge" => 1,
4793         "xbzrle" => 1,
4794         "x-rdma-pin-all" => 0,
4795         "zero-blocks" => 0,
4796         "compress" => 0,
4797         "dirty-bitmaps" => $dirty_bitmaps,
4798     };
4799
4800     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4801
4802     for my $supported_capability (@$supported_capabilities) {
4803         push @$cap_ref, {
4804             capability => $supported_capability->{capability},
4805             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4806         };
4807     }
4808
4809     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4810 }
4811
4812 sub foreach_volid {
4813     my ($conf, $func, @param) = @_;
4814
4815     my $volhash = {};
4816
4817     my $test_volid = sub {
4818         my ($key, $drive, $snapname) = @_;
4819
4820         my $volid = $drive->{file};
4821         return if !$volid;
4822
4823         $volhash->{$volid}->{cdrom} //= 1;
4824         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4825
4826         my $replicate = $drive->{replicate} // 1;
4827         $volhash->{$volid}->{replicate} //= 0;
4828         $volhash->{$volid}->{replicate} = 1 if $replicate;
4829
4830         $volhash->{$volid}->{shared} //= 0;
4831         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4832
4833         $volhash->{$volid}->{referenced_in_config} //= 0;
4834         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4835
4836         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4837             if defined($snapname);
4838
4839         my $size = $drive->{size};
4840         $volhash->{$volid}->{size} //= $size if $size;
4841
4842         $volhash->{$volid}->{is_vmstate} //= 0;
4843         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4844
4845         $volhash->{$volid}->{is_tpmstate} //= 0;
4846         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4847
4848         $volhash->{$volid}->{is_unused} //= 0;
4849         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4850
4851         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4852     };
4853
4854     my $include_opts = {
4855         extra_keys => ['vmstate'],
4856         include_unused => 1,
4857     };
4858
4859     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4860     foreach my $snapname (keys %{$conf->{snapshots}}) {
4861         my $snap = $conf->{snapshots}->{$snapname};
4862         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4863     }
4864
4865     foreach my $volid (keys %$volhash) {
4866         &$func($volid, $volhash->{$volid}, @param);
4867     }
4868 }
4869
4870 my $fast_plug_option = {
4871     'lock' => 1,
4872     'name' => 1,
4873     'onboot' => 1,
4874     'shares' => 1,
4875     'startup' => 1,
4876     'description' => 1,
4877     'protection' => 1,
4878     'vmstatestorage' => 1,
4879     'hookscript' => 1,
4880     'tags' => 1,
4881 };
4882
4883 for my $opt (keys %$confdesc_cloudinit) {
4884     $fast_plug_option->{$opt} = 1;
4885 };
4886
4887 # hotplug changes in [PENDING]
4888 # $selection hash can be used to only apply specified options, for
4889 # example: { cores => 1 } (only apply changed 'cores')
4890 # $errors ref is used to return error messages
4891 sub vmconfig_hotplug_pending {
4892     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4893
4894     my $defaults = load_defaults();
4895     my $arch = get_vm_arch($conf);
4896     my $machine_type = get_vm_machine($conf, undef, $arch);
4897
4898     # commit values which do not have any impact on running VM first
4899     # Note: those option cannot raise errors, we we do not care about
4900     # $selection and always apply them.
4901
4902     my $add_error = sub {
4903         my ($opt, $msg) = @_;
4904         $errors->{$opt} = "hotplug problem - $msg";
4905     };
4906
4907     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4908
4909     my $cloudinit_record_changed = sub {
4910         my ($conf, $opt, $old, $new) = @_;
4911         return if !$cloudinit_pending_properties->{$opt};
4912
4913         my $ci = ($conf->{cloudinit} //= {});
4914
4915         my $recorded = $ci->{$opt};
4916         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4917
4918         if (defined($new)) {
4919             if (defined($old)) {
4920                 # an existing value is being modified
4921                 if (defined($recorded)) {
4922                     # the value was already not in sync
4923                     if ($new eq $recorded) {
4924                         # a value is being reverted to the cloud-init state:
4925                         delete $ci->{$opt};
4926                         delete $added{$opt};
4927                     } else {
4928                         # the value was changed multiple times, do nothing
4929                     }
4930                 } elsif ($added{$opt}) {
4931                     # the value had been marked as added and is being changed, do nothing
4932                 } else {
4933                     # the value is new, record it:
4934                     $ci->{$opt} = $old;
4935                 }
4936             } else {
4937                 # a new value is being added
4938                 if (defined($recorded)) {
4939                     # it was already not in sync
4940                     if ($new eq $recorded) {
4941                         # a value is being reverted to the cloud-init state:
4942                         delete $ci->{$opt};
4943                         delete $added{$opt};
4944                     } else {
4945                         # the value had temporarily been removed, do nothing
4946                     }
4947                 } elsif ($added{$opt}) {
4948                     # the value had been marked as added already, do nothing
4949                 } else {
4950                     # the value is new, add it
4951                     $added{$opt} = 1;
4952                 }
4953             }
4954         } elsif (!defined($old)) {
4955             # a non-existent value is being removed? ignore...
4956         } else {
4957             # a value is being deleted
4958             if (defined($recorded)) {
4959                 # a value was already recorded, just keep it
4960             } elsif ($added{$opt}) {
4961                 # the value was marked as added, remove it
4962                 delete $added{$opt};
4963             } else {
4964                 # a previously unrecorded value is being removed, record the old value:
4965                 $ci->{$opt} = $old;
4966             }
4967         }
4968
4969         my $added = join(',', sort keys %added);
4970         $ci->{added} = $added if length($added);
4971     };
4972
4973     my $changes = 0;
4974     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4975         if ($fast_plug_option->{$opt}) {
4976             my $new = delete $conf->{pending}->{$opt};
4977             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4978             $conf->{$opt} = $new;
4979             $changes = 1;
4980         }
4981     }
4982
4983     if ($changes) {
4984         PVE::QemuConfig->write_config($vmid, $conf);
4985     }
4986
4987     my $ostype = $conf->{ostype};
4988     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4989     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4990     my $usb_hotplug = $hotplug_features->{usb}
4991         && min_version($version, 7, 1)
4992         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4993
4994     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4995     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4996
4997     foreach my $opt (sort keys %$pending_delete_hash) {
4998         next if $selection && !$selection->{$opt};
4999         my $force = $pending_delete_hash->{$opt}->{force};
5000         eval {
5001             if ($opt eq 'hotplug') {
5002                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5003             } elsif ($opt eq 'tablet') {
5004                 die "skip\n" if !$hotplug_features->{usb};
5005                 if ($defaults->{tablet}) {
5006                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5008                         if $arch eq 'aarch64';
5009                 } else {
5010                     vm_deviceunplug($vmid, $conf, 'tablet');
5011                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5012                 }
5013             } elsif ($opt =~ m/^usb(\d+)$/) {
5014                 my $index = $1;
5015                 die "skip\n" if !$usb_hotplug;
5016                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5017                 vm_deviceunplug($vmid, $conf, $opt);
5018             } elsif ($opt eq 'vcpus') {
5019                 die "skip\n" if !$hotplug_features->{cpu};
5020                 qemu_cpu_hotplug($vmid, $conf, undef);
5021             } elsif ($opt eq 'balloon') {
5022                 # enable balloon device is not hotpluggable
5023                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5024                 # here we reset the ballooning value to memory
5025                 my $balloon = $conf->{memory} || $defaults->{memory};
5026                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5027             } elsif ($fast_plug_option->{$opt}) {
5028                 # do nothing
5029             } elsif ($opt =~ m/^net(\d+)$/) {
5030                 die "skip\n" if !$hotplug_features->{network};
5031                 vm_deviceunplug($vmid, $conf, $opt);
5032             } elsif (is_valid_drivename($opt)) {
5033                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5034                 vm_deviceunplug($vmid, $conf, $opt);
5035                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5036             } elsif ($opt =~ m/^memory$/) {
5037                 die "skip\n" if !$hotplug_features->{memory};
5038                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5039             } elsif ($opt eq 'cpuunits') {
5040                 $cgroup->change_cpu_shares(undef);
5041             } elsif ($opt eq 'cpulimit') {
5042                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5043             } else {
5044                 die "skip\n";
5045             }
5046         };
5047         if (my $err = $@) {
5048             &$add_error($opt, $err) if $err ne "skip\n";
5049         } else {
5050             my $old = delete $conf->{$opt};
5051             $cloudinit_record_changed->($conf, $opt, $old, undef);
5052             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5053         }
5054     }
5055
5056     my $cloudinit_opt;
5057     foreach my $opt (keys %{$conf->{pending}}) {
5058         next if $selection && !$selection->{$opt};
5059         my $value = $conf->{pending}->{$opt};
5060         eval {
5061             if ($opt eq 'hotplug') {
5062                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5063             } elsif ($opt eq 'tablet') {
5064                 die "skip\n" if !$hotplug_features->{usb};
5065                 if ($value == 1) {
5066                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5067                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5068                         if $arch eq 'aarch64';
5069                 } elsif ($value == 0) {
5070                     vm_deviceunplug($vmid, $conf, 'tablet');
5071                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5072                 }
5073             } elsif ($opt =~ m/^usb(\d+)$/) {
5074                 my $index = $1;
5075                 die "skip\n" if !$usb_hotplug;
5076                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5077                 my $id = $opt;
5078                 if ($d->{host} eq 'spice')  {
5079                     $id = "usbredirdev$index";
5080                 }
5081                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5082             } elsif ($opt eq 'vcpus') {
5083                 die "skip\n" if !$hotplug_features->{cpu};
5084                 qemu_cpu_hotplug($vmid, $conf, $value);
5085             } elsif ($opt eq 'balloon') {
5086                 # enable/disable balloning device is not hotpluggable
5087                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5088                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5089                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5090
5091                 # allow manual ballooning if shares is set to zero
5092                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5093                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5094                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5095                 }
5096             } elsif ($opt =~ m/^net(\d+)$/) {
5097                 # some changes can be done without hotplug
5098                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5099                                     $vmid, $opt, $value, $arch, $machine_type);
5100             } elsif (is_valid_drivename($opt)) {
5101                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5102                 # some changes can be done without hotplug
5103                 my $drive = parse_drive($opt, $value);
5104                 if (drive_is_cloudinit($drive)) {
5105                     $cloudinit_opt = [$opt, $drive];
5106                     # apply all the other changes first, then generate the cloudinit disk
5107                     die "skip\n";
5108                 }
5109                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5110                                      $vmid, $opt, $value, $arch, $machine_type);
5111             } elsif ($opt =~ m/^memory$/) { #dimms
5112                 die "skip\n" if !$hotplug_features->{memory};
5113                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5114             } elsif ($opt eq 'cpuunits') {
5115                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5116                 $cgroup->change_cpu_shares($new_cpuunits);
5117             } elsif ($opt eq 'cpulimit') {
5118                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5119                 $cgroup->change_cpu_quota($cpulimit, 100000);
5120             } elsif ($opt eq 'agent') {
5121                 vmconfig_update_agent($conf, $opt, $value);
5122             } else {
5123                 die "skip\n";  # skip non-hot-pluggable options
5124             }
5125         };
5126         if (my $err = $@) {
5127             &$add_error($opt, $err) if $err ne "skip\n";
5128         } else {
5129             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5130             $conf->{$opt} = $value;
5131             delete $conf->{pending}->{$opt};
5132         }
5133     }
5134
5135     if (defined($cloudinit_opt)) {
5136         my ($opt, $drive) = @$cloudinit_opt;
5137         my $value = $conf->{pending}->{$opt};
5138         eval {
5139             my $temp = {%$conf, $opt => $value};
5140             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5141             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5142                                  $vmid, $opt, $value, $arch, $machine_type);
5143         };
5144         if (my $err = $@) {
5145             &$add_error($opt, $err) if $err ne "skip\n";
5146         } else {
5147             $conf->{$opt} = $value;
5148             delete $conf->{pending}->{$opt};
5149         }
5150     }
5151
5152     # unplug xhci controller if no usb device is left
5153     if ($usb_hotplug) {
5154         my $has_usb = 0;
5155         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5156             next if !defined($conf->{"usb$i"});
5157             $has_usb = 1;
5158             last;
5159         }
5160         if (!$has_usb) {
5161             vm_deviceunplug($vmid, $conf, 'xhci');
5162         }
5163     }
5164
5165     PVE::QemuConfig->write_config($vmid, $conf);
5166
5167     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5168         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5169     }
5170 }
5171
5172 sub try_deallocate_drive {
5173     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5174
5175     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5176         my $volid = $drive->{file};
5177         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5178             my $sid = PVE::Storage::parse_volume_id($volid);
5179             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5180
5181             # check if the disk is really unused
5182             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5183                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5184             PVE::Storage::vdisk_free($storecfg, $volid);
5185             return 1;
5186         } else {
5187             # If vm is not owner of this disk remove from config
5188             return 1;
5189         }
5190     }
5191
5192     return;
5193 }
5194
5195 sub vmconfig_delete_or_detach_drive {
5196     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5197
5198     my $drive = parse_drive($opt, $conf->{$opt});
5199
5200     my $rpcenv = PVE::RPCEnvironment::get();
5201     my $authuser = $rpcenv->get_user();
5202
5203     if ($force) {
5204         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5205         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5206     } else {
5207         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5208     }
5209 }
5210
5211
5212
5213 sub vmconfig_apply_pending {
5214     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5215
5216     return if !scalar(keys %{$conf->{pending}});
5217
5218     my $add_apply_error = sub {
5219         my ($opt, $msg) = @_;
5220         my $err_msg = "unable to apply pending change $opt : $msg";
5221         $errors->{$opt} = $err_msg;
5222         warn $err_msg;
5223     };
5224
5225     # cold plug
5226
5227     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5228     foreach my $opt (sort keys %$pending_delete_hash) {
5229         my $force = $pending_delete_hash->{$opt}->{force};
5230         eval {
5231             if ($opt =~ m/^unused/) {
5232                 die "internal error";
5233             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5234                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5235             }
5236         };
5237         if (my $err = $@) {
5238             $add_apply_error->($opt, $err);
5239         } else {
5240             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5241             delete $conf->{$opt};
5242         }
5243     }
5244
5245     PVE::QemuConfig->cleanup_pending($conf);
5246
5247     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5248
5249     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5250         next if $opt eq 'delete'; # just to be sure
5251         eval {
5252             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5253                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5254             }
5255         };
5256         if (my $err = $@) {
5257             $add_apply_error->($opt, $err);
5258         } else {
5259
5260             if (is_valid_drivename($opt)) {
5261                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5262                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5263             }
5264
5265             $conf->{$opt} = delete $conf->{pending}->{$opt};
5266         }
5267     }
5268
5269     # write all changes at once to avoid unnecessary i/o
5270     PVE::QemuConfig->write_config($vmid, $conf);
5271     if ($generate_cloudinit) {
5272         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5273             # After successful generation and if there were changes to be applied, update the
5274             # config to drop the {cloudinit} entry.
5275             PVE::QemuConfig->write_config($vmid, $conf);
5276         }
5277     }
5278 }
5279
5280 sub vmconfig_update_net {
5281     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5282
5283     my $newnet = parse_net($value);
5284
5285     if ($conf->{$opt}) {
5286         my $oldnet = parse_net($conf->{$opt});
5287
5288         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5289             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5290             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5291             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5292
5293             # for non online change, we try to hot-unplug
5294             die "skip\n" if !$hotplug;
5295             vm_deviceunplug($vmid, $conf, $opt);
5296         } else {
5297
5298             die "internal error" if $opt !~ m/net(\d+)/;
5299             my $iface = "tap${vmid}i$1";
5300
5301             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5302                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5303                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5304                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5305                 PVE::Network::tap_unplug($iface);
5306
5307                 if ($have_sdn) {
5308                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5309                 } else {
5310                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5311                 }
5312             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5313                 # Rate can be applied on its own but any change above needs to
5314                 # include the rate in tap_plug since OVS resets everything.
5315                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5316             }
5317
5318             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5319                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5320             }
5321
5322             return 1;
5323         }
5324     }
5325
5326     if ($hotplug) {
5327         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5328     } else {
5329         die "skip\n";
5330     }
5331 }
5332
5333 sub vmconfig_update_agent {
5334     my ($conf, $opt, $value) = @_;
5335
5336     die "skip\n" if !$conf->{$opt};
5337
5338     my $hotplug_options = { fstrim_cloned_disks => 1 };
5339
5340     my $old_agent = parse_guest_agent($conf);
5341     my $agent = parse_guest_agent({$opt => $value});
5342
5343     for my $option (keys %$agent) { # added/changed options
5344         next if defined($hotplug_options->{$option});
5345         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5346     }
5347
5348     for my $option (keys %$old_agent) { # removed options
5349         next if defined($hotplug_options->{$option});
5350         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5351     }
5352
5353     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5354 }
5355
5356 sub vmconfig_update_disk {
5357     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5358
5359     my $drive = parse_drive($opt, $value);
5360
5361     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5362         my $media = $drive->{media} || 'disk';
5363         my $oldmedia = $old_drive->{media} || 'disk';
5364         die "unable to change media type\n" if $media ne $oldmedia;
5365
5366         if (!drive_is_cdrom($old_drive)) {
5367
5368             if ($drive->{file} ne $old_drive->{file}) {
5369
5370                 die "skip\n" if !$hotplug;
5371
5372                 # unplug and register as unused
5373                 vm_deviceunplug($vmid, $conf, $opt);
5374                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5375
5376             } else {
5377                 # update existing disk
5378
5379                 # skip non hotpluggable value
5380                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5381                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5382                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5383                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5384                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5385                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5386                     die "skip\n";
5387                 }
5388
5389                 # apply throttle
5390                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5391                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5392                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5393                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5394                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5395                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5396                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5397                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5398                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5399                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5400                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5401                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5402                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5403                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5404                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5405                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5406                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5407                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5408
5409                     qemu_block_set_io_throttle(
5410                         $vmid,"drive-$opt",
5411                         ($drive->{mbps} || 0)*1024*1024,
5412                         ($drive->{mbps_rd} || 0)*1024*1024,
5413                         ($drive->{mbps_wr} || 0)*1024*1024,
5414                          $drive->{iops} || 0,
5415                          $drive->{iops_rd} || 0,
5416                          $drive->{iops_wr} || 0,
5417                         ($drive->{mbps_max} || 0)*1024*1024,
5418                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5419                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5420                          $drive->{iops_max} || 0,
5421                          $drive->{iops_rd_max} || 0,
5422                          $drive->{iops_wr_max} || 0,
5423                          $drive->{bps_max_length} || 1,
5424                          $drive->{bps_rd_max_length} || 1,
5425                          $drive->{bps_wr_max_length} || 1,
5426                          $drive->{iops_max_length} || 1,
5427                          $drive->{iops_rd_max_length} || 1,
5428                          $drive->{iops_wr_max_length} || 1,
5429                     );
5430
5431                 }
5432
5433                 return 1;
5434             }
5435
5436         } else { # cdrom
5437
5438             if ($drive->{file} eq 'none') {
5439                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5440                 if (drive_is_cloudinit($old_drive)) {
5441                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5442                 }
5443             } else {
5444                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5445
5446                 # force eject if locked
5447                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5448
5449                 if ($path) {
5450                     mon_cmd($vmid, "blockdev-change-medium",
5451                         id => "$opt", filename => "$path");
5452                 }
5453             }
5454
5455             return 1;
5456         }
5457     }
5458
5459     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5460     # hotplug new disks
5461     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5462     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5463 }
5464
5465 sub vmconfig_update_cloudinit_drive {
5466     my ($storecfg, $conf, $vmid) = @_;
5467
5468     my $cloudinit_ds = undef;
5469     my $cloudinit_drive = undef;
5470
5471     PVE::QemuConfig->foreach_volume($conf, sub {
5472         my ($ds, $drive) = @_;
5473         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5474             $cloudinit_ds = $ds;
5475             $cloudinit_drive = $drive;
5476         }
5477     });
5478
5479     return if !$cloudinit_drive;
5480
5481     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5482         PVE::QemuConfig->write_config($vmid, $conf);
5483     }
5484
5485     my $running = PVE::QemuServer::check_running($vmid);
5486
5487     if ($running) {
5488         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5489         if ($path) {
5490             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5491             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5492         }
5493     }
5494 }
5495
5496 # called in locked context by incoming migration
5497 sub vm_migrate_get_nbd_disks {
5498     my ($storecfg, $conf, $replicated_volumes) = @_;
5499
5500     my $local_volumes = {};
5501     PVE::QemuConfig->foreach_volume($conf, sub {
5502         my ($ds, $drive) = @_;
5503
5504         return if drive_is_cdrom($drive);
5505         return if $ds eq 'tpmstate0';
5506
5507         my $volid = $drive->{file};
5508
5509         return if !$volid;
5510
5511         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5512
5513         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5514         return if $scfg->{shared};
5515
5516         # replicated disks re-use existing state via bitmap
5517         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5518         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5519     });
5520     return $local_volumes;
5521 }
5522
5523 # called in locked context by incoming migration
5524 sub vm_migrate_alloc_nbd_disks {
5525     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5526
5527     my $nbd = {};
5528     foreach my $opt (sort keys %$source_volumes) {
5529         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5530
5531         if ($use_existing) {
5532             $nbd->{$opt}->{drivestr} = print_drive($drive);
5533             $nbd->{$opt}->{volid} = $volid;
5534             $nbd->{$opt}->{replicated} = 1;
5535             next;
5536         }
5537
5538         # storage mapping + volname = regular migration
5539         # storage mapping + format = remote migration
5540         # order of precedence, filtered by whether storage supports it:
5541         # 1. explicit requested format
5542         # 2. format of current volume
5543         # 3. default format of storage
5544         if (!$storagemap->{identity}) {
5545             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5546             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5547             if (!$format || !grep { $format eq $_ } @$validFormats) {
5548                 if ($volname) {
5549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5550                     my $fileFormat = qemu_img_format($scfg, $volname);
5551                     $format = $fileFormat
5552                         if grep { $fileFormat eq $_ } @$validFormats;
5553                 }
5554                 $format //= $defFormat;
5555             }
5556         } else {
5557             # can't happen for remote migration, so $volname is always defined
5558             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5559             $format = qemu_img_format($scfg, $volname);
5560         }
5561
5562         my $size = $drive->{size} / 1024;
5563         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5564         my $newdrive = $drive;
5565         $newdrive->{format} = $format;
5566         $newdrive->{file} = $newvolid;
5567         my $drivestr = print_drive($newdrive);
5568         $nbd->{$opt}->{drivestr} = $drivestr;
5569         $nbd->{$opt}->{volid} = $newvolid;
5570     }
5571
5572     return $nbd;
5573 }
5574
5575 # see vm_start_nolock for parameters, additionally:
5576 # migrate_opts:
5577 #   storagemap = parsed storage map for allocating NBD disks
5578 sub vm_start {
5579     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5580
5581     return PVE::QemuConfig->lock_config($vmid, sub {
5582         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5583
5584         die "you can't start a vm if it's a template\n"
5585             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5586
5587         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5588         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5589
5590         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5591
5592         if ($has_backup_lock && $running) {
5593             # a backup is currently running, attempt to start the guest in the
5594             # existing QEMU instance
5595             return vm_resume($vmid);
5596         }
5597
5598         PVE::QemuConfig->check_lock($conf)
5599             if !($params->{skiplock} || $has_suspended_lock);
5600
5601         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5602
5603         die "VM $vmid already running\n" if $running;
5604
5605         if (my $storagemap = $migrate_opts->{storagemap}) {
5606             my $replicated = $migrate_opts->{replicated_volumes};
5607             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5608             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5609
5610             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5611                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5612             }
5613         }
5614
5615         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5616     });
5617 }
5618
5619
5620 # params:
5621 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5622 #   skiplock => 0/1, skip checking for config lock
5623 #   skiptemplate => 0/1, skip checking whether VM is template
5624 #   forcemachine => to force Qemu machine (rollback/migration)
5625 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5626 #   timeout => in seconds
5627 #   paused => start VM in paused state (backup)
5628 #   resume => resume from hibernation
5629 #   pbs-backing => {
5630 #      sata0 => {
5631 #         repository
5632 #         snapshot
5633 #         keyfile
5634 #         archive
5635 #      },
5636 #      virtio2 => ...
5637 #   }
5638 # migrate_opts:
5639 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5640 #   migratedfrom => source node
5641 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5642 #   network => CIDR of migration network
5643 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5644 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5645 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5646 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5647 #       contained in config
5648 sub vm_start_nolock {
5649     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5650
5651     my $statefile = $params->{statefile};
5652     my $resume = $params->{resume};
5653
5654     my $migratedfrom = $migrate_opts->{migratedfrom};
5655     my $migration_type = $migrate_opts->{type};
5656
5657     my $res = {};
5658
5659     # clean up leftover reboot request files
5660     eval { clear_reboot_request($vmid); };
5661     warn $@ if $@;
5662
5663     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5664         vmconfig_apply_pending($vmid, $conf, $storecfg);
5665         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5666     }
5667
5668     # don't regenerate the ISO if the VM is started as part of a live migration
5669     # this way we can reuse the old ISO with the correct config
5670     if (!$migratedfrom) {
5671         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5672             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5673             # $conf->{cloudinit}, so we could just not do this?
5674             # But we do it above, so for now let's be consistent.
5675             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5676         }
5677     }
5678
5679     # override offline migrated volumes, conf is out of date still
5680     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5681         for my $key (sort keys $offline_volumes->%*) {
5682             my $parsed = parse_drive($key, $conf->{$key});
5683             $parsed->{file} = $offline_volumes->{$key};
5684             $conf->{$key} = print_drive($parsed);
5685         }
5686     }
5687
5688     my $defaults = load_defaults();
5689
5690     # set environment variable useful inside network script
5691     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5692
5693     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5694
5695     my $forcemachine = $params->{forcemachine};
5696     my $forcecpu = $params->{forcecpu};
5697     if ($resume) {
5698         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5699         $forcemachine = $conf->{runningmachine};
5700         $forcecpu = $conf->{runningcpu};
5701         print "Resuming suspended VM\n";
5702     }
5703
5704     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5705         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5706
5707     my $migration_ip;
5708     my $get_migration_ip = sub {
5709         my ($nodename) = @_;
5710
5711         return $migration_ip if defined($migration_ip);
5712
5713         my $cidr = $migrate_opts->{network};
5714
5715         if (!defined($cidr)) {
5716             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5717             $cidr = $dc_conf->{migration}->{network};
5718         }
5719
5720         if (defined($cidr)) {
5721             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5722
5723             die "could not get IP: no address configured on local " .
5724                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5725
5726             die "could not get IP: multiple addresses configured on local " .
5727                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5728
5729             $migration_ip = @$ips[0];
5730         }
5731
5732         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5733             if !defined($migration_ip);
5734
5735         return $migration_ip;
5736     };
5737
5738     if ($statefile) {
5739         if ($statefile eq 'tcp') {
5740             my $migrate = $res->{migrate} = { proto => 'tcp' };
5741             $migrate->{addr} = "localhost";
5742             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5743             my $nodename = nodename();
5744
5745             if (!defined($migration_type)) {
5746                 if (defined($datacenterconf->{migration}->{type})) {
5747                     $migration_type = $datacenterconf->{migration}->{type};
5748                 } else {
5749                     $migration_type = 'secure';
5750                 }
5751             }
5752
5753             if ($migration_type eq 'insecure') {
5754                 $migrate->{addr} = $get_migration_ip->($nodename);
5755                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5756             }
5757
5758             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5759             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5760             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5761             push @$cmd, '-incoming', $migrate->{uri};
5762             push @$cmd, '-S';
5763
5764         } elsif ($statefile eq 'unix') {
5765             # should be default for secure migrations as a ssh TCP forward
5766             # tunnel is not deterministic reliable ready and fails regurarly
5767             # to set up in time, so use UNIX socket forwards
5768             my $migrate = $res->{migrate} = { proto => 'unix' };
5769             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5770             unlink $migrate->{addr};
5771
5772             $migrate->{uri} = "unix:$migrate->{addr}";
5773             push @$cmd, '-incoming', $migrate->{uri};
5774             push @$cmd, '-S';
5775
5776         } elsif (-e $statefile) {
5777             push @$cmd, '-loadstate', $statefile;
5778         } else {
5779             my $statepath = PVE::Storage::path($storecfg, $statefile);
5780             push @$vollist, $statefile;
5781             push @$cmd, '-loadstate', $statepath;
5782         }
5783     } elsif ($params->{paused}) {
5784         push @$cmd, '-S';
5785     }
5786
5787     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5788
5789     my $pci_devices = {}; # host pci devices
5790     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5791         my $dev = $conf->{"hostpci$i"} or next;
5792         $pci_devices->{$i} = parse_hostpci($dev);
5793     }
5794
5795     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5796     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5797
5798     # map to a flat list of pci ids
5799     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5800
5801     # reserve all PCI IDs before actually doing anything with them
5802     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5803
5804     eval {
5805         my $uuid;
5806         for my $id (sort keys %$pci_devices) {
5807             my $d = $pci_devices->{$id};
5808             for my $dev ($d->{pciid}->@*) {
5809                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5810
5811                 # nvidia grid needs the uuid of the mdev as qemu parameter
5812                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5813                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5814                 }
5815             }
5816         }
5817         push @$cmd, '-uuid', $uuid if defined($uuid);
5818     };
5819     if (my $err = $@) {
5820         eval { cleanup_pci_devices($vmid, $conf) };
5821         warn $@ if $@;
5822         die $err;
5823     }
5824
5825     PVE::Storage::activate_volumes($storecfg, $vollist);
5826
5827     eval {
5828         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5829     };
5830     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5831     # timeout should be more than enough here...
5832     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5833
5834     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5835
5836     my %run_params = (
5837         timeout => $statefile ? undef : $start_timeout,
5838         umask => 0077,
5839         noerr => 1,
5840     );
5841
5842     # when migrating, prefix QEMU output so other side can pick up any
5843     # errors that might occur and show the user
5844     if ($migratedfrom) {
5845         $run_params{quiet} = 1;
5846         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5847     }
5848
5849     my %systemd_properties = (
5850         Slice => 'qemu.slice',
5851         KillMode => 'process',
5852         SendSIGKILL => 0,
5853         TimeoutStopUSec => ULONG_MAX, # infinity
5854     );
5855
5856     if (PVE::CGroup::cgroup_mode() == 2) {
5857         $systemd_properties{CPUWeight} = $cpuunits;
5858     } else {
5859         $systemd_properties{CPUShares} = $cpuunits;
5860     }
5861
5862     if (my $cpulimit = $conf->{cpulimit}) {
5863         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5864     }
5865     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5866
5867     my $run_qemu = sub {
5868         PVE::Tools::run_fork sub {
5869             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5870
5871             my $tpmpid;
5872             if (my $tpm = $conf->{tpmstate0}) {
5873                 # start the TPM emulator so QEMU can connect on start
5874                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5875             }
5876
5877             my $exitcode = run_command($cmd, %run_params);
5878             if ($exitcode) {
5879                 if ($tpmpid) {
5880                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5881                     kill 'TERM', $tpmpid;
5882                 }
5883                 die "QEMU exited with code $exitcode\n";
5884             }
5885         };
5886     };
5887
5888     if ($conf->{hugepages}) {
5889
5890         my $code = sub {
5891             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5892             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5893
5894             PVE::QemuServer::Memory::hugepages_mount();
5895             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5896
5897             eval { $run_qemu->() };
5898             if (my $err = $@) {
5899                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5900                     if !$conf->{keephugepages};
5901                 die $err;
5902             }
5903
5904             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5905                 if !$conf->{keephugepages};
5906         };
5907         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5908
5909     } else {
5910         eval { $run_qemu->() };
5911     }
5912
5913     if (my $err = $@) {
5914         # deactivate volumes if start fails
5915         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5916         warn $@ if $@;
5917         eval { cleanup_pci_devices($vmid, $conf) };
5918         warn $@ if $@;
5919
5920         die "start failed: $err";
5921     }
5922
5923     # re-reserve all PCI IDs now that we can know the actual VM PID
5924     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5925     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5926     warn $@ if $@;
5927
5928     if (defined($res->{migrate})) {
5929         print "migration listens on $res->{migrate}->{uri}\n";
5930     } elsif ($statefile) {
5931         eval { mon_cmd($vmid, "cont"); };
5932         warn $@ if $@;
5933     }
5934
5935     #start nbd server for storage migration
5936     if (my $nbd = $migrate_opts->{nbd}) {
5937         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5938
5939         my $migrate_storage_uri;
5940         # nbd_protocol_version > 0 for unix socket support
5941         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5942             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5943             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5944             $migrate_storage_uri = "nbd:unix:$socket_path";
5945             $res->{migrate}->{unix_sockets} = [$socket_path];
5946         } else {
5947             my $nodename = nodename();
5948             my $localip = $get_migration_ip->($nodename);
5949             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5950             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5951
5952             mon_cmd($vmid, "nbd-server-start", addr => {
5953                 type => 'inet',
5954                 data => {
5955                     host => "${localip}",
5956                     port => "${storage_migrate_port}",
5957                 },
5958             });
5959             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5960             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5961         }
5962
5963         foreach my $opt (sort keys %$nbd) {
5964             my $drivestr = $nbd->{$opt}->{drivestr};
5965             my $volid = $nbd->{$opt}->{volid};
5966             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5967             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5968             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5969             print "re-using replicated volume: $opt - $volid\n"
5970                 if $nbd->{$opt}->{replicated};
5971
5972             $res->{drives}->{$opt} = $nbd->{$opt};
5973             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5974         }
5975     }
5976
5977     if ($migratedfrom) {
5978         eval {
5979             set_migration_caps($vmid);
5980         };
5981         warn $@ if $@;
5982
5983         if ($spice_port) {
5984             print "spice listens on port $spice_port\n";
5985             $res->{spice_port} = $spice_port;
5986             if ($migrate_opts->{spice_ticket}) {
5987                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5988                     $migrate_opts->{spice_ticket});
5989                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5990             }
5991         }
5992
5993     } else {
5994         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5995             if !$statefile && $conf->{balloon};
5996
5997         foreach my $opt (keys %$conf) {
5998             next if $opt !~  m/^net\d+$/;
5999             my $nicconf = parse_net($conf->{$opt});
6000             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6001         }
6002         add_nets_bridge_fdb($conf, $vmid);
6003     }
6004
6005     mon_cmd($vmid, 'qom-set',
6006                 path => "machine/peripheral/balloon0",
6007                 property => "guest-stats-polling-interval",
6008                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6009
6010     if ($resume) {
6011         print "Resumed VM, removing state\n";
6012         if (my $vmstate = $conf->{vmstate}) {
6013             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6014             PVE::Storage::vdisk_free($storecfg, $vmstate);
6015         }
6016         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6017         PVE::QemuConfig->write_config($vmid, $conf);
6018     }
6019
6020     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6021
6022     return $res;
6023 }
6024
6025 sub vm_commandline {
6026     my ($storecfg, $vmid, $snapname) = @_;
6027
6028     my $conf = PVE::QemuConfig->load_config($vmid);
6029
6030     my ($forcemachine, $forcecpu);
6031     if ($snapname) {
6032         my $snapshot = $conf->{snapshots}->{$snapname};
6033         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6034
6035         # check for machine or CPU overrides in snapshot
6036         $forcemachine = $snapshot->{runningmachine};
6037         $forcecpu = $snapshot->{runningcpu};
6038
6039         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6040
6041         $conf = $snapshot;
6042     }
6043
6044     my $defaults = load_defaults();
6045
6046     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6047
6048     return PVE::Tools::cmd2string($cmd);
6049 }
6050
6051 sub vm_reset {
6052     my ($vmid, $skiplock) = @_;
6053
6054     PVE::QemuConfig->lock_config($vmid, sub {
6055
6056         my $conf = PVE::QemuConfig->load_config($vmid);
6057
6058         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6059
6060         mon_cmd($vmid, "system_reset");
6061     });
6062 }
6063
6064 sub get_vm_volumes {
6065     my ($conf) = @_;
6066
6067     my $vollist = [];
6068     foreach_volid($conf, sub {
6069         my ($volid, $attr) = @_;
6070
6071         return if $volid =~ m|^/|;
6072
6073         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6074         return if !$sid;
6075
6076         push @$vollist, $volid;
6077     });
6078
6079     return $vollist;
6080 }
6081
6082 sub cleanup_pci_devices {
6083     my ($vmid, $conf) = @_;
6084
6085     foreach my $key (keys %$conf) {
6086         next if $key !~ m/^hostpci(\d+)$/;
6087         my $hostpciindex = $1;
6088         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6089         my $d = parse_hostpci($conf->{$key});
6090         if ($d->{mdev}) {
6091             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6092             # don't want to break ABI just for this two liner
6093             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6094             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6095         }
6096     }
6097     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6098 }
6099
6100 sub vm_stop_cleanup {
6101     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6102
6103     eval {
6104
6105         if (!$keepActive) {
6106             my $vollist = get_vm_volumes($conf);
6107             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6108
6109             if (my $tpmdrive = $conf->{tpmstate0}) {
6110                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6111                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6112                 if ($storeid) {
6113                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6114                 }
6115             }
6116         }
6117
6118         foreach my $ext (qw(mon qmp pid vnc qga)) {
6119             unlink "/var/run/qemu-server/${vmid}.$ext";
6120         }
6121
6122         if ($conf->{ivshmem}) {
6123             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6124             # just delete it for now, VMs which have this already open do not
6125             # are affected, but new VMs will get a separated one. If this
6126             # becomes an issue we either add some sort of ref-counting or just
6127             # add a "don't delete on stop" flag to the ivshmem format.
6128             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6129         }
6130
6131         cleanup_pci_devices($vmid, $conf);
6132
6133         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6134     };
6135     warn $@ if $@; # avoid errors - just warn
6136 }
6137
6138 # call only in locked context
6139 sub _do_vm_stop {
6140     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6141
6142     my $pid = check_running($vmid, $nocheck);
6143     return if !$pid;
6144
6145     my $conf;
6146     if (!$nocheck) {
6147         $conf = PVE::QemuConfig->load_config($vmid);
6148         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6149         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6150             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6151             $timeout = $opts->{down} if $opts->{down};
6152         }
6153         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6154     }
6155
6156     eval {
6157         if ($shutdown) {
6158             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6159                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6160             } else {
6161                 mon_cmd($vmid, "system_powerdown");
6162             }
6163         } else {
6164             mon_cmd($vmid, "quit");
6165         }
6166     };
6167     my $err = $@;
6168
6169     if (!$err) {
6170         $timeout = 60 if !defined($timeout);
6171
6172         my $count = 0;
6173         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6174             $count++;
6175             sleep 1;
6176         }
6177
6178         if ($count >= $timeout) {
6179             if ($force) {
6180                 warn "VM still running - terminating now with SIGTERM\n";
6181                 kill 15, $pid;
6182             } else {
6183                 die "VM quit/powerdown failed - got timeout\n";
6184             }
6185         } else {
6186             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6187             return;
6188         }
6189     } else {
6190        if (!check_running($vmid, $nocheck)) {
6191             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6192             return;
6193        }
6194        if ($force) {
6195             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6196             kill 15, $pid;
6197         } else {
6198             die "VM quit/powerdown failed\n";
6199         }
6200     }
6201
6202     # wait again
6203     $timeout = 10;
6204
6205     my $count = 0;
6206     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6207         $count++;
6208         sleep 1;
6209     }
6210
6211     if ($count >= $timeout) {
6212         warn "VM still running - terminating now with SIGKILL\n";
6213         kill 9, $pid;
6214         sleep 1;
6215     }
6216
6217     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6218 }
6219
6220 # Note: use $nocheck to skip tests if VM configuration file exists.
6221 # We need that when migration VMs to other nodes (files already moved)
6222 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6223 sub vm_stop {
6224     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6225
6226     $force = 1 if !defined($force) && !$shutdown;
6227
6228     if ($migratedfrom){
6229         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6230         kill 15, $pid if $pid;
6231         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6232         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6233         return;
6234     }
6235
6236     PVE::QemuConfig->lock_config($vmid, sub {
6237         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6238    });
6239 }
6240
6241 sub vm_reboot {
6242     my ($vmid, $timeout) = @_;
6243
6244     PVE::QemuConfig->lock_config($vmid, sub {
6245         eval {
6246
6247             # only reboot if running, as qmeventd starts it again on a stop event
6248             return if !check_running($vmid);
6249
6250             create_reboot_request($vmid);
6251
6252             my $storecfg = PVE::Storage::config();
6253             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6254
6255         };
6256         if (my $err = $@) {
6257             # avoid that the next normal shutdown will be confused for a reboot
6258             clear_reboot_request($vmid);
6259             die $err;
6260         }
6261    });
6262 }
6263
6264 # note: if using the statestorage parameter, the caller has to check privileges
6265 sub vm_suspend {
6266     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6267
6268     my $conf;
6269     my $path;
6270     my $storecfg;
6271     my $vmstate;
6272
6273     PVE::QemuConfig->lock_config($vmid, sub {
6274
6275         $conf = PVE::QemuConfig->load_config($vmid);
6276
6277         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6278         PVE::QemuConfig->check_lock($conf)
6279             if !($skiplock || $is_backing_up);
6280
6281         die "cannot suspend to disk during backup\n"
6282             if $is_backing_up && $includestate;
6283
6284         if ($includestate) {
6285             $conf->{lock} = 'suspending';
6286             my $date = strftime("%Y-%m-%d", localtime(time()));
6287             $storecfg = PVE::Storage::config();
6288             if (!$statestorage) {
6289                 $statestorage = find_vmstate_storage($conf, $storecfg);
6290                 # check permissions for the storage
6291                 my $rpcenv = PVE::RPCEnvironment::get();
6292                 if ($rpcenv->{type} ne 'cli') {
6293                     my $authuser = $rpcenv->get_user();
6294                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6295                 }
6296             }
6297
6298
6299             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6300                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6301             $path = PVE::Storage::path($storecfg, $vmstate);
6302             PVE::QemuConfig->write_config($vmid, $conf);
6303         } else {
6304             mon_cmd($vmid, "stop");
6305         }
6306     });
6307
6308     if ($includestate) {
6309         # save vm state
6310         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6311
6312         eval {
6313             set_migration_caps($vmid, 1);
6314             mon_cmd($vmid, "savevm-start", statefile => $path);
6315             for(;;) {
6316                 my $state = mon_cmd($vmid, "query-savevm");
6317                 if (!$state->{status}) {
6318                     die "savevm not active\n";
6319                 } elsif ($state->{status} eq 'active') {
6320                     sleep(1);
6321                     next;
6322                 } elsif ($state->{status} eq 'completed') {
6323                     print "State saved, quitting\n";
6324                     last;
6325                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6326                     die "query-savevm failed with error '$state->{error}'\n"
6327                 } else {
6328                     die "query-savevm returned status '$state->{status}'\n";
6329                 }
6330             }
6331         };
6332         my $err = $@;
6333
6334         PVE::QemuConfig->lock_config($vmid, sub {
6335             $conf = PVE::QemuConfig->load_config($vmid);
6336             if ($err) {
6337                 # cleanup, but leave suspending lock, to indicate something went wrong
6338                 eval {
6339                     mon_cmd($vmid, "savevm-end");
6340                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6341                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6342                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6343                     PVE::QemuConfig->write_config($vmid, $conf);
6344                 };
6345                 warn $@ if $@;
6346                 die $err;
6347             }
6348
6349             die "lock changed unexpectedly\n"
6350                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6351
6352             mon_cmd($vmid, "quit");
6353             $conf->{lock} = 'suspended';
6354             PVE::QemuConfig->write_config($vmid, $conf);
6355         });
6356     }
6357 }
6358
6359 sub vm_resume {
6360     my ($vmid, $skiplock, $nocheck) = @_;
6361
6362     PVE::QemuConfig->lock_config($vmid, sub {
6363         my $res = mon_cmd($vmid, 'query-status');
6364         my $resume_cmd = 'cont';
6365         my $reset = 0;
6366         my $conf = PVE::QemuConfig->load_config($vmid);
6367
6368         if ($res->{status}) {
6369             return if $res->{status} eq 'running'; # job done, go home
6370             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6371             $reset = 1 if $res->{status} eq 'shutdown';
6372         }
6373
6374         if (!$nocheck) {
6375
6376             PVE::QemuConfig->check_lock($conf)
6377                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6378         }
6379
6380         if ($reset) {
6381             # required if a VM shuts down during a backup and we get a resume
6382             # request before the backup finishes for example
6383             mon_cmd($vmid, "system_reset");
6384         }
6385
6386         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6387
6388         mon_cmd($vmid, $resume_cmd);
6389     });
6390 }
6391
6392 sub vm_sendkey {
6393     my ($vmid, $skiplock, $key) = @_;
6394
6395     PVE::QemuConfig->lock_config($vmid, sub {
6396
6397         my $conf = PVE::QemuConfig->load_config($vmid);
6398
6399         # there is no qmp command, so we use the human monitor command
6400         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6401         die $res if $res ne '';
6402     });
6403 }
6404
6405 # vzdump restore implementaion
6406
6407 sub tar_archive_read_firstfile {
6408     my $archive = shift;
6409
6410     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6411
6412     # try to detect archive type first
6413     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6414         die "unable to open file '$archive'\n";
6415     my $firstfile = <$fh>;
6416     kill 15, $pid;
6417     close $fh;
6418
6419     die "ERROR: archive contaions no data\n" if !$firstfile;
6420     chomp $firstfile;
6421
6422     return $firstfile;
6423 }
6424
6425 sub tar_restore_cleanup {
6426     my ($storecfg, $statfile) = @_;
6427
6428     print STDERR "starting cleanup\n";
6429
6430     if (my $fd = IO::File->new($statfile, "r")) {
6431         while (defined(my $line = <$fd>)) {
6432             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6433                 my $volid = $2;
6434                 eval {
6435                     if ($volid =~ m|^/|) {
6436                         unlink $volid || die 'unlink failed\n';
6437                     } else {
6438                         PVE::Storage::vdisk_free($storecfg, $volid);
6439                     }
6440                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6441                 };
6442                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6443             } else {
6444                 print STDERR "unable to parse line in statfile - $line";
6445             }
6446         }
6447         $fd->close();
6448     }
6449 }
6450
6451 sub restore_file_archive {
6452     my ($archive, $vmid, $user, $opts) = @_;
6453
6454     return restore_vma_archive($archive, $vmid, $user, $opts)
6455         if $archive eq '-';
6456
6457     my $info = PVE::Storage::archive_info($archive);
6458     my $format = $opts->{format} // $info->{format};
6459     my $comp = $info->{compression};
6460
6461     # try to detect archive format
6462     if ($format eq 'tar') {
6463         return restore_tar_archive($archive, $vmid, $user, $opts);
6464     } else {
6465         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6466     }
6467 }
6468
6469 # hepler to remove disks that will not be used after restore
6470 my $restore_cleanup_oldconf = sub {
6471     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6472
6473     my $kept_disks = {};
6474
6475     PVE::QemuConfig->foreach_volume($oldconf, sub {
6476         my ($ds, $drive) = @_;
6477
6478         return if drive_is_cdrom($drive, 1);
6479
6480         my $volid = $drive->{file};
6481         return if !$volid || $volid =~ m|^/|;
6482
6483         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6484         return if !$path || !$owner || ($owner != $vmid);
6485
6486         # Note: only delete disk we want to restore
6487         # other volumes will become unused
6488         if ($virtdev_hash->{$ds}) {
6489             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6490             if (my $err = $@) {
6491                 warn $err;
6492             }
6493         } else {
6494             $kept_disks->{$volid} = 1;
6495         }
6496     });
6497
6498     # after the restore we have no snapshots anymore
6499     for my $snapname (keys $oldconf->{snapshots}->%*) {
6500         my $snap = $oldconf->{snapshots}->{$snapname};
6501         if ($snap->{vmstate}) {
6502             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6503             if (my $err = $@) {
6504                 warn $err;
6505             }
6506         }
6507
6508         for my $volid (keys $kept_disks->%*) {
6509             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6510             warn $@ if $@;
6511         }
6512     }
6513 };
6514
6515 # Helper to parse vzdump backup device hints
6516 #
6517 # $rpcenv: Environment, used to ckeck storage permissions
6518 # $user: User ID, to check storage permissions
6519 # $storecfg: Storage configuration
6520 # $fh: the file handle for reading the configuration
6521 # $devinfo: should contain device sizes for all backu-up'ed devices
6522 # $options: backup options (pool, default storage)
6523 #
6524 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6525 my $parse_backup_hints = sub {
6526     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6527
6528     my $check_storage = sub { # assert if an image can be allocate
6529         my ($storeid, $scfg) = @_;
6530         die "Content type 'images' is not available on storage '$storeid'\n"
6531             if !$scfg->{content}->{images};
6532         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6533             if $user ne 'root@pam';
6534     };
6535
6536     my $virtdev_hash = {};
6537     while (defined(my $line = <$fh>)) {
6538         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6539             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6540             die "archive does not contain data for drive '$virtdev'\n"
6541                 if !$devinfo->{$devname};
6542
6543             if (defined($options->{storage})) {
6544                 $storeid = $options->{storage} || 'local';
6545             } elsif (!$storeid) {
6546                 $storeid = 'local';
6547             }
6548             $format = 'raw' if !$format;
6549             $devinfo->{$devname}->{devname} = $devname;
6550             $devinfo->{$devname}->{virtdev} = $virtdev;
6551             $devinfo->{$devname}->{format} = $format;
6552             $devinfo->{$devname}->{storeid} = $storeid;
6553
6554             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6555             $check_storage->($storeid, $scfg); # permission and content type check
6556
6557             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6558         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6559             my $virtdev = $1;
6560             my $drive = parse_drive($virtdev, $2);
6561
6562             if (drive_is_cloudinit($drive)) {
6563                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6564                 $storeid = $options->{storage} if defined ($options->{storage});
6565                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6566                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6567
6568                 $check_storage->($storeid, $scfg); # permission and content type check
6569
6570                 $virtdev_hash->{$virtdev} = {
6571                     format => $format,
6572                     storeid => $storeid,
6573                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6574                     is_cloudinit => 1,
6575                 };
6576             }
6577         }
6578     }
6579
6580     return $virtdev_hash;
6581 };
6582
6583 # Helper to allocate and activate all volumes required for a restore
6584 #
6585 # $storecfg: Storage configuration
6586 # $virtdev_hash: as returned by parse_backup_hints()
6587 #
6588 # Returns: { $virtdev => $volid }
6589 my $restore_allocate_devices = sub {
6590     my ($storecfg, $virtdev_hash, $vmid) = @_;
6591
6592     my $map = {};
6593     foreach my $virtdev (sort keys %$virtdev_hash) {
6594         my $d = $virtdev_hash->{$virtdev};
6595         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6596         my $storeid = $d->{storeid};
6597         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6598
6599         # test if requested format is supported
6600         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6601         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6602         $d->{format} = $defFormat if !$supported;
6603
6604         my $name;
6605         if ($d->{is_cloudinit}) {
6606             $name = "vm-$vmid-cloudinit";
6607             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6608             if ($scfg->{path}) {
6609                 $name .= ".$d->{format}";
6610             }
6611         }
6612
6613         my $volid = PVE::Storage::vdisk_alloc(
6614             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6615
6616         print STDERR "new volume ID is '$volid'\n";
6617         $d->{volid} = $volid;
6618
6619         PVE::Storage::activate_volumes($storecfg, [$volid]);
6620
6621         $map->{$virtdev} = $volid;
6622     }
6623
6624     return $map;
6625 };
6626
6627 sub restore_update_config_line {
6628     my ($cookie, $map, $line, $unique) = @_;
6629
6630     return '' if $line =~ m/^\#qmdump\#/;
6631     return '' if $line =~ m/^\#vzdump\#/;
6632     return '' if $line =~ m/^lock:/;
6633     return '' if $line =~ m/^unused\d+:/;
6634     return '' if $line =~ m/^parent:/;
6635
6636     my $res = '';
6637
6638     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6639     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6640         # try to convert old 1.X settings
6641         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6642         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6643             my ($model, $macaddr) = split(/\=/, $devconfig);
6644             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6645             my $net = {
6646                 model => $model,
6647                 bridge => "vmbr$ind",
6648                 macaddr => $macaddr,
6649             };
6650             my $netstr = print_net($net);
6651
6652             $res .= "net$cookie->{netcount}: $netstr\n";
6653             $cookie->{netcount}++;
6654         }
6655     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6656         my ($id, $netstr) = ($1, $2);
6657         my $net = parse_net($netstr);
6658         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6659         $netstr = print_net($net);
6660         $res .= "$id: $netstr\n";
6661     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6662         my $virtdev = $1;
6663         my $value = $3;
6664         my $di = parse_drive($virtdev, $value);
6665         if (defined($di->{backup}) && !$di->{backup}) {
6666             $res .= "#$line";
6667         } elsif ($map->{$virtdev}) {
6668             delete $di->{format}; # format can change on restore
6669             $di->{file} = $map->{$virtdev};
6670             $value = print_drive($di);
6671             $res .= "$virtdev: $value\n";
6672         } else {
6673             $res .= $line;
6674         }
6675     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6676         my $vmgenid = $1;
6677         if ($vmgenid ne '0') {
6678             # always generate a new vmgenid if there was a valid one setup
6679             $vmgenid = generate_uuid();
6680         }
6681         $res .= "vmgenid: $vmgenid\n";
6682     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6683         my ($uuid, $uuid_str);
6684         UUID::generate($uuid);
6685         UUID::unparse($uuid, $uuid_str);
6686         my $smbios1 = parse_smbios1($2);
6687         $smbios1->{uuid} = $uuid_str;
6688         $res .= $1.print_smbios1($smbios1)."\n";
6689     } else {
6690         $res .= $line;
6691     }
6692
6693     return $res;
6694 }
6695
6696 my $restore_deactivate_volumes = sub {
6697     my ($storecfg, $virtdev_hash) = @_;
6698
6699     my $vollist = [];
6700     for my $dev (values $virtdev_hash->%*) {
6701         push $vollist->@*, $dev->{volid} if $dev->{volid};
6702     }
6703
6704     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6705     print STDERR $@ if $@;
6706 };
6707
6708 my $restore_destroy_volumes = sub {
6709     my ($storecfg, $virtdev_hash) = @_;
6710
6711     for my $dev (values $virtdev_hash->%*) {
6712         my $volid = $dev->{volid} or next;
6713         eval {
6714             PVE::Storage::vdisk_free($storecfg, $volid);
6715             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6716         };
6717         print STDERR "unable to cleanup '$volid' - $@" if $@;
6718     }
6719 };
6720
6721 my $restore_merge_config = sub {
6722     my ($filename, $backup_conf_raw, $override_conf) = @_;
6723
6724     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6725     for my $key (keys $override_conf->%*) {
6726         $backup_conf->{$key} = $override_conf->{$key};
6727     }
6728
6729     return $backup_conf;
6730 };
6731
6732 sub scan_volids {
6733     my ($cfg, $vmid) = @_;
6734
6735     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6736
6737     my $volid_hash = {};
6738     foreach my $storeid (keys %$info) {
6739         foreach my $item (@{$info->{$storeid}}) {
6740             next if !($item->{volid} && $item->{size});
6741             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6742             $volid_hash->{$item->{volid}} = $item;
6743         }
6744     }
6745
6746     return $volid_hash;
6747 }
6748
6749 sub update_disk_config {
6750     my ($vmid, $conf, $volid_hash) = @_;
6751
6752     my $changes;
6753     my $prefix = "VM $vmid";
6754
6755     # used and unused disks
6756     my $referenced = {};
6757
6758     # Note: it is allowed to define multiple storages with same path (alias), so
6759     # we need to check both 'volid' and real 'path' (two different volid can point
6760     # to the same path).
6761
6762     my $referencedpath = {};
6763
6764     # update size info
6765     PVE::QemuConfig->foreach_volume($conf, sub {
6766         my ($opt, $drive) = @_;
6767
6768         my $volid = $drive->{file};
6769         return if !$volid;
6770         my $volume = $volid_hash->{$volid};
6771
6772         # mark volid as "in-use" for next step
6773         $referenced->{$volid} = 1;
6774         if ($volume && (my $path = $volume->{path})) {
6775             $referencedpath->{$path} = 1;
6776         }
6777
6778         return if drive_is_cdrom($drive);
6779         return if !$volume;
6780
6781         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6782         if (defined($updated)) {
6783             $changes = 1;
6784             $conf->{$opt} = print_drive($updated);
6785             print "$prefix ($opt): $msg\n";
6786         }
6787     });
6788
6789     # remove 'unusedX' entry if volume is used
6790     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6791         my ($opt, $drive) = @_;
6792
6793         my $volid = $drive->{file};
6794         return if !$volid;
6795
6796         my $path;
6797         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6798         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6799             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6800             $changes = 1;
6801             delete $conf->{$opt};
6802         }
6803
6804         $referenced->{$volid} = 1;
6805         $referencedpath->{$path} = 1 if $path;
6806     });
6807
6808     foreach my $volid (sort keys %$volid_hash) {
6809         next if $volid =~ m/vm-$vmid-state-/;
6810         next if $referenced->{$volid};
6811         my $path = $volid_hash->{$volid}->{path};
6812         next if !$path; # just to be sure
6813         next if $referencedpath->{$path};
6814         $changes = 1;
6815         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6816         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6817         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6818     }
6819
6820     return $changes;
6821 }
6822
6823 sub rescan {
6824     my ($vmid, $nolock, $dryrun) = @_;
6825
6826     my $cfg = PVE::Storage::config();
6827
6828     print "rescan volumes...\n";
6829     my $volid_hash = scan_volids($cfg, $vmid);
6830
6831     my $updatefn =  sub {
6832         my ($vmid) = @_;
6833
6834         my $conf = PVE::QemuConfig->load_config($vmid);
6835
6836         PVE::QemuConfig->check_lock($conf);
6837
6838         my $vm_volids = {};
6839         foreach my $volid (keys %$volid_hash) {
6840             my $info = $volid_hash->{$volid};
6841             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6842         }
6843
6844         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6845
6846         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6847     };
6848
6849     if (defined($vmid)) {
6850         if ($nolock) {
6851             &$updatefn($vmid);
6852         } else {
6853             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6854         }
6855     } else {
6856         my $vmlist = config_list();
6857         foreach my $vmid (keys %$vmlist) {
6858             if ($nolock) {
6859                 &$updatefn($vmid);
6860             } else {
6861                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6862             }
6863         }
6864     }
6865 }
6866
6867 sub restore_proxmox_backup_archive {
6868     my ($archive, $vmid, $user, $options) = @_;
6869
6870     my $storecfg = PVE::Storage::config();
6871
6872     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6873     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6874
6875     my $fingerprint = $scfg->{fingerprint};
6876     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6877
6878     my $repo = PVE::PBSClient::get_repository($scfg);
6879     my $namespace = $scfg->{namespace};
6880
6881     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6882     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6883     local $ENV{PBS_PASSWORD} = $password;
6884     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6885
6886     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6887         PVE::Storage::parse_volname($storecfg, $archive);
6888
6889     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6890
6891     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6892
6893     my $tmpdir = "/var/tmp/vzdumptmp$$";
6894     rmtree $tmpdir;
6895     mkpath $tmpdir;
6896
6897     my $conffile = PVE::QemuConfig->config_file($vmid);
6898      # disable interrupts (always do cleanups)
6899     local $SIG{INT} =
6900         local $SIG{TERM} =
6901         local $SIG{QUIT} =
6902         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6903
6904     # Note: $oldconf is undef if VM does not exists
6905     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6906     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6907     my $new_conf_raw = '';
6908
6909     my $rpcenv = PVE::RPCEnvironment::get();
6910     my $devinfo = {}; # info about drives included in backup
6911     my $virtdev_hash = {}; # info about allocated drives
6912
6913     eval {
6914         # enable interrupts
6915         local $SIG{INT} =
6916             local $SIG{TERM} =
6917             local $SIG{QUIT} =
6918             local $SIG{HUP} =
6919             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6920
6921         my $cfgfn = "$tmpdir/qemu-server.conf";
6922         my $firewall_config_fn = "$tmpdir/fw.conf";
6923         my $index_fn = "$tmpdir/index.json";
6924
6925         my $cmd = "restore";
6926
6927         my $param = [$pbs_backup_name, "index.json", $index_fn];
6928         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6929         my $index = PVE::Tools::file_get_contents($index_fn);
6930         $index = decode_json($index);
6931
6932         foreach my $info (@{$index->{files}}) {
6933             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6934                 my $devname = $1;
6935                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6936                     $devinfo->{$devname}->{size} = $1;
6937                 } else {
6938                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6939                 }
6940             }
6941         }
6942
6943         my $is_qemu_server_backup = scalar(
6944             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6945         );
6946         if (!$is_qemu_server_backup) {
6947             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6948         }
6949         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6950
6951         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6952         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6953
6954         if ($has_firewall_config) {
6955             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6956             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6957
6958             my $pve_firewall_dir = '/etc/pve/firewall';
6959             mkdir $pve_firewall_dir; # make sure the dir exists
6960             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6961         }
6962
6963         my $fh = IO::File->new($cfgfn, "r") ||
6964             die "unable to read qemu-server.conf - $!\n";
6965
6966         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6967
6968         # fixme: rate limit?
6969
6970         # create empty/temp config
6971         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6972
6973         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6974
6975         # allocate volumes
6976         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6977
6978         foreach my $virtdev (sort keys %$virtdev_hash) {
6979             my $d = $virtdev_hash->{$virtdev};
6980             next if $d->{is_cloudinit}; # no need to restore cloudinit
6981
6982             # this fails if storage is unavailable
6983             my $volid = $d->{volid};
6984             my $path = PVE::Storage::path($storecfg, $volid);
6985
6986             # for live-restore we only want to preload the efidisk and TPM state
6987             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6988
6989             my @ns_arg;
6990             if (defined(my $ns = $scfg->{namespace})) {
6991                 @ns_arg = ('--ns', $ns);
6992             }
6993
6994             my $pbs_restore_cmd = [
6995                 '/usr/bin/pbs-restore',
6996                 '--repository', $repo,
6997                 @ns_arg,
6998                 $pbs_backup_name,
6999                 "$d->{devname}.img.fidx",
7000                 $path,
7001                 '--verbose',
7002                 ];
7003
7004             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7005             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7006
7007             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7008                 push @$pbs_restore_cmd, '--skip-zero';
7009             }
7010
7011             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7012             print "restore proxmox backup image: $dbg_cmdstring\n";
7013             run_command($pbs_restore_cmd);
7014         }
7015
7016         $fh->seek(0, 0) || die "seek failed - $!\n";
7017
7018         my $cookie = { netcount => 0 };
7019         while (defined(my $line = <$fh>)) {
7020             $new_conf_raw .= restore_update_config_line(
7021                 $cookie,
7022                 $map,
7023                 $line,
7024                 $options->{unique},
7025             );
7026         }
7027
7028         $fh->close();
7029     };
7030     my $err = $@;
7031
7032     if ($err || !$options->{live}) {
7033         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7034     }
7035
7036     rmtree $tmpdir;
7037
7038     if ($err) {
7039         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7040         die $err;
7041     }
7042
7043     if ($options->{live}) {
7044         # keep lock during live-restore
7045         $new_conf_raw .= "\nlock: create";
7046     }
7047
7048     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7049     PVE::QemuConfig->write_config($vmid, $new_conf);
7050
7051     eval { rescan($vmid, 1); };
7052     warn $@ if $@;
7053
7054     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7055
7056     if ($options->{live}) {
7057         # enable interrupts
7058         local $SIG{INT} =
7059             local $SIG{TERM} =
7060             local $SIG{QUIT} =
7061             local $SIG{HUP} =
7062             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7063
7064         my $conf = PVE::QemuConfig->load_config($vmid);
7065         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7066
7067         # these special drives are already restored before start
7068         delete $devinfo->{'drive-efidisk0'};
7069         delete $devinfo->{'drive-tpmstate0-backup'};
7070
7071         my $pbs_opts = {
7072             repo => $repo,
7073             keyfile => $keyfile,
7074             snapshot => $pbs_backup_name,
7075             namespace => $namespace,
7076         };
7077         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7078
7079         PVE::QemuConfig->remove_lock($vmid, "create");
7080     }
7081 }
7082
7083 sub pbs_live_restore {
7084     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7085
7086     print "starting VM for live-restore\n";
7087     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7088
7089     my $pbs_backing = {};
7090     for my $ds (keys %$restored_disks) {
7091         $ds =~ m/^drive-(.*)$/;
7092         my $confname = $1;
7093         $pbs_backing->{$confname} = {
7094             repository => $opts->{repo},
7095             snapshot => $opts->{snapshot},
7096             archive => "$ds.img.fidx",
7097         };
7098         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7099         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7100
7101         my $drive = parse_drive($confname, $conf->{$confname});
7102         print "restoring '$ds' to '$drive->{file}'\n";
7103     }
7104
7105     my $drives_streamed = 0;
7106     eval {
7107         # make sure HA doesn't interrupt our restore by stopping the VM
7108         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7109             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7110         }
7111
7112         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7113         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7114         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7115
7116         my $qmeventd_fd = register_qmeventd_handle($vmid);
7117
7118         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7119         # this will effectively collapse the backing image chain consisting of
7120         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7121         # removes itself once all backing images vanish with 'auto-remove=on')
7122         my $jobs = {};
7123         for my $ds (sort keys %$restored_disks) {
7124             my $job_id = "restore-$ds";
7125             mon_cmd($vmid, 'block-stream',
7126                 'job-id' => $job_id,
7127                 device => "$ds",
7128             );
7129             $jobs->{$job_id} = {};
7130         }
7131
7132         mon_cmd($vmid, 'cont');
7133         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7134
7135         print "restore-drive jobs finished successfully, removing all tracking block devices"
7136             ." to disconnect from Proxmox Backup Server\n";
7137
7138         for my $ds (sort keys %$restored_disks) {
7139             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7140         }
7141
7142         close($qmeventd_fd);
7143     };
7144
7145     my $err = $@;
7146
7147     if ($err) {
7148         warn "An error occurred during live-restore: $err\n";
7149         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7150         die "live-restore failed\n";
7151     }
7152 }
7153
7154 sub restore_vma_archive {
7155     my ($archive, $vmid, $user, $opts, $comp) = @_;
7156
7157     my $readfrom = $archive;
7158
7159     my $cfg = PVE::Storage::config();
7160     my $commands = [];
7161     my $bwlimit = $opts->{bwlimit};
7162
7163     my $dbg_cmdstring = '';
7164     my $add_pipe = sub {
7165         my ($cmd) = @_;
7166         push @$commands, $cmd;
7167         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7168         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7169         $readfrom = '-';
7170     };
7171
7172     my $input = undef;
7173     if ($archive eq '-') {
7174         $input = '<&STDIN';
7175     } else {
7176         # If we use a backup from a PVE defined storage we also consider that
7177         # storage's rate limit:
7178         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7179         if (defined($volid)) {
7180             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7181             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7182             if ($readlimit) {
7183                 print STDERR "applying read rate limit: $readlimit\n";
7184                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7185                 $add_pipe->($cstream);
7186             }
7187         }
7188     }
7189
7190     if ($comp) {
7191         my $info = PVE::Storage::decompressor_info('vma', $comp);
7192         my $cmd = $info->{decompressor};
7193         push @$cmd, $readfrom;
7194         $add_pipe->($cmd);
7195     }
7196
7197     my $tmpdir = "/var/tmp/vzdumptmp$$";
7198     rmtree $tmpdir;
7199
7200     # disable interrupts (always do cleanups)
7201     local $SIG{INT} =
7202         local $SIG{TERM} =
7203         local $SIG{QUIT} =
7204         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7205
7206     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7207     POSIX::mkfifo($mapfifo, 0600);
7208     my $fifofh;
7209     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7210
7211     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7212
7213     my $oldtimeout;
7214     my $timeout = 5;
7215
7216     my $devinfo = {}; # info about drives included in backup
7217     my $virtdev_hash = {}; # info about allocated drives
7218
7219     my $rpcenv = PVE::RPCEnvironment::get();
7220
7221     my $conffile = PVE::QemuConfig->config_file($vmid);
7222
7223     # Note: $oldconf is undef if VM does not exist
7224     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7225     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7226     my $new_conf_raw = '';
7227
7228     my %storage_limits;
7229
7230     my $print_devmap = sub {
7231         my $cfgfn = "$tmpdir/qemu-server.conf";
7232
7233         # we can read the config - that is already extracted
7234         my $fh = IO::File->new($cfgfn, "r") ||
7235             die "unable to read qemu-server.conf - $!\n";
7236
7237         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7238         if (-f $fwcfgfn) {
7239             my $pve_firewall_dir = '/etc/pve/firewall';
7240             mkdir $pve_firewall_dir; # make sure the dir exists
7241             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7242         }
7243
7244         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7245
7246         foreach my $info (values %{$virtdev_hash}) {
7247             my $storeid = $info->{storeid};
7248             next if defined($storage_limits{$storeid});
7249
7250             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7251             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7252             $storage_limits{$storeid} = $limit * 1024;
7253         }
7254
7255         foreach my $devname (keys %$devinfo) {
7256             die "found no device mapping information for device '$devname'\n"
7257                 if !$devinfo->{$devname}->{virtdev};
7258         }
7259
7260         # create empty/temp config
7261         if ($oldconf) {
7262             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7263             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7264         }
7265
7266         # allocate volumes
7267         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7268
7269         # print restore information to $fifofh
7270         foreach my $virtdev (sort keys %$virtdev_hash) {
7271             my $d = $virtdev_hash->{$virtdev};
7272             next if $d->{is_cloudinit}; # no need to restore cloudinit
7273
7274             my $storeid = $d->{storeid};
7275             my $volid = $d->{volid};
7276
7277             my $map_opts = '';
7278             if (my $limit = $storage_limits{$storeid}) {
7279                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7280             }
7281
7282             my $write_zeros = 1;
7283             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7284                 $write_zeros = 0;
7285             }
7286
7287             my $path = PVE::Storage::path($cfg, $volid);
7288
7289             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7290
7291             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7292         }
7293
7294         $fh->seek(0, 0) || die "seek failed - $!\n";
7295
7296         my $cookie = { netcount => 0 };
7297         while (defined(my $line = <$fh>)) {
7298             $new_conf_raw .= restore_update_config_line(
7299                 $cookie,
7300                 $map,
7301                 $line,
7302                 $opts->{unique},
7303             );
7304         }
7305
7306         $fh->close();
7307     };
7308
7309     eval {
7310         # enable interrupts
7311         local $SIG{INT} =
7312             local $SIG{TERM} =
7313             local $SIG{QUIT} =
7314             local $SIG{HUP} =
7315             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7316         local $SIG{ALRM} = sub { die "got timeout\n"; };
7317
7318         $oldtimeout = alarm($timeout);
7319
7320         my $parser = sub {
7321             my $line = shift;
7322
7323             print "$line\n";
7324
7325             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7326                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7327                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7328             } elsif ($line =~ m/^CTIME: /) {
7329                 # we correctly received the vma config, so we can disable
7330                 # the timeout now for disk allocation (set to 10 minutes, so
7331                 # that we always timeout if something goes wrong)
7332                 alarm(600);
7333                 &$print_devmap();
7334                 print $fifofh "done\n";
7335                 my $tmp = $oldtimeout || 0;
7336                 $oldtimeout = undef;
7337                 alarm($tmp);
7338                 close($fifofh);
7339                 $fifofh = undef;
7340             }
7341         };
7342
7343         print "restore vma archive: $dbg_cmdstring\n";
7344         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7345     };
7346     my $err = $@;
7347
7348     alarm($oldtimeout) if $oldtimeout;
7349
7350     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7351
7352     close($fifofh) if $fifofh;
7353     unlink $mapfifo;
7354     rmtree $tmpdir;
7355
7356     if ($err) {
7357         $restore_destroy_volumes->($cfg, $virtdev_hash);
7358         die $err;
7359     }
7360
7361     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7362     PVE::QemuConfig->write_config($vmid, $new_conf);
7363
7364     eval { rescan($vmid, 1); };
7365     warn $@ if $@;
7366
7367     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7368 }
7369
7370 sub restore_tar_archive {
7371     my ($archive, $vmid, $user, $opts) = @_;
7372
7373     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7374         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7375         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7376     }
7377
7378     if ($archive ne '-') {
7379         my $firstfile = tar_archive_read_firstfile($archive);
7380         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7381             if $firstfile ne 'qemu-server.conf';
7382     }
7383
7384     my $storecfg = PVE::Storage::config();
7385
7386     # avoid zombie disks when restoring over an existing VM -> cleanup first
7387     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7388     # skiplock=1 because qmrestore has set the 'create' lock itself already
7389     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7390     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7391
7392     my $tocmd = "/usr/lib/qemu-server/qmextract";
7393
7394     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7395     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7396     $tocmd .= ' --prealloc' if $opts->{prealloc};
7397     $tocmd .= ' --info' if $opts->{info};
7398
7399     # tar option "xf" does not autodetect compression when read from STDIN,
7400     # so we pipe to zcat
7401     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7402         PVE::Tools::shellquote("--to-command=$tocmd");
7403
7404     my $tmpdir = "/var/tmp/vzdumptmp$$";
7405     mkpath $tmpdir;
7406
7407     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7408     local $ENV{VZDUMP_VMID} = $vmid;
7409     local $ENV{VZDUMP_USER} = $user;
7410
7411     my $conffile = PVE::QemuConfig->config_file($vmid);
7412     my $new_conf_raw = '';
7413
7414     # disable interrupts (always do cleanups)
7415     local $SIG{INT} =
7416         local $SIG{TERM} =
7417         local $SIG{QUIT} =
7418         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7419
7420     eval {
7421         # enable interrupts
7422         local $SIG{INT} =
7423             local $SIG{TERM} =
7424             local $SIG{QUIT} =
7425             local $SIG{HUP} =
7426             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7427
7428         if ($archive eq '-') {
7429             print "extracting archive from STDIN\n";
7430             run_command($cmd, input => "<&STDIN");
7431         } else {
7432             print "extracting archive '$archive'\n";
7433             run_command($cmd);
7434         }
7435
7436         return if $opts->{info};
7437
7438         # read new mapping
7439         my $map = {};
7440         my $statfile = "$tmpdir/qmrestore.stat";
7441         if (my $fd = IO::File->new($statfile, "r")) {
7442             while (defined (my $line = <$fd>)) {
7443                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7444                     $map->{$1} = $2 if $1;
7445                 } else {
7446                     print STDERR "unable to parse line in statfile - $line\n";
7447                 }
7448             }
7449             $fd->close();
7450         }
7451
7452         my $confsrc = "$tmpdir/qemu-server.conf";
7453
7454         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7455
7456         my $cookie = { netcount => 0 };
7457         while (defined (my $line = <$srcfd>)) {
7458             $new_conf_raw .= restore_update_config_line(
7459                 $cookie,
7460                 $map,
7461                 $line,
7462                 $opts->{unique},
7463             );
7464         }
7465
7466         $srcfd->close();
7467     };
7468     if (my $err = $@) {
7469         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7470         die $err;
7471     }
7472
7473     rmtree $tmpdir;
7474
7475     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7476
7477     PVE::Cluster::cfs_update(); # make sure we read new file
7478
7479     eval { rescan($vmid, 1); };
7480     warn $@ if $@;
7481 };
7482
7483 sub foreach_storage_used_by_vm {
7484     my ($conf, $func) = @_;
7485
7486     my $sidhash = {};
7487
7488     PVE::QemuConfig->foreach_volume($conf, sub {
7489         my ($ds, $drive) = @_;
7490         return if drive_is_cdrom($drive);
7491
7492         my $volid = $drive->{file};
7493
7494         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7495         $sidhash->{$sid} = $sid if $sid;
7496     });
7497
7498     foreach my $sid (sort keys %$sidhash) {
7499         &$func($sid);
7500     }
7501 }
7502
7503 my $qemu_snap_storage = {
7504     rbd => 1,
7505 };
7506 sub do_snapshots_with_qemu {
7507     my ($storecfg, $volid, $deviceid) = @_;
7508
7509     return if $deviceid =~ m/tpmstate0/;
7510
7511     my $storage_name = PVE::Storage::parse_volume_id($volid);
7512     my $scfg = $storecfg->{ids}->{$storage_name};
7513     die "could not find storage '$storage_name'\n" if !defined($scfg);
7514
7515     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7516         return 1;
7517     }
7518
7519     if ($volid =~ m/\.(qcow2|qed)$/){
7520         return 1;
7521     }
7522
7523     return;
7524 }
7525
7526 sub qga_check_running {
7527     my ($vmid, $nowarn) = @_;
7528
7529     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7530     if ($@) {
7531         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7532         return 0;
7533     }
7534     return 1;
7535 }
7536
7537 sub template_create {
7538     my ($vmid, $conf, $disk) = @_;
7539
7540     my $storecfg = PVE::Storage::config();
7541
7542     PVE::QemuConfig->foreach_volume($conf, sub {
7543         my ($ds, $drive) = @_;
7544
7545         return if drive_is_cdrom($drive);
7546         return if $disk && $ds ne $disk;
7547
7548         my $volid = $drive->{file};
7549         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7550
7551         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7552         $drive->{file} = $voliddst;
7553         $conf->{$ds} = print_drive($drive);
7554         PVE::QemuConfig->write_config($vmid, $conf);
7555     });
7556 }
7557
7558 sub convert_iscsi_path {
7559     my ($path) = @_;
7560
7561     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7562         my $portal = $1;
7563         my $target = $2;
7564         my $lun = $3;
7565
7566         my $initiator_name = get_initiator_name();
7567
7568         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7569                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7570     }
7571
7572     die "cannot convert iscsi path '$path', unkown format\n";
7573 }
7574
7575 sub qemu_img_convert {
7576     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7577
7578     my $storecfg = PVE::Storage::config();
7579     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7580     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7581
7582     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7583
7584     my $cachemode;
7585     my $src_path;
7586     my $src_is_iscsi = 0;
7587     my $src_format;
7588
7589     if ($src_storeid) {
7590         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7591         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7592         $src_format = qemu_img_format($src_scfg, $src_volname);
7593         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7594         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7595         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7596     } elsif (-f $src_volid || -b $src_volid) {
7597         $src_path = $src_volid;
7598         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7599             $src_format = $1;
7600         }
7601     }
7602
7603     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7604
7605     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7606     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7607     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7608     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7609
7610     my $cmd = [];
7611     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7612     push @$cmd, '-l', "snapshot.name=$snapname"
7613         if $snapname && $src_format && $src_format eq "qcow2";
7614     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7615     push @$cmd, '-T', $cachemode if defined($cachemode);
7616
7617     if ($src_is_iscsi) {
7618         push @$cmd, '--image-opts';
7619         $src_path = convert_iscsi_path($src_path);
7620     } elsif ($src_format) {
7621         push @$cmd, '-f', $src_format;
7622     }
7623
7624     if ($dst_is_iscsi) {
7625         push @$cmd, '--target-image-opts';
7626         $dst_path = convert_iscsi_path($dst_path);
7627     } else {
7628         push @$cmd, '-O', $dst_format;
7629     }
7630
7631     push @$cmd, $src_path;
7632
7633     if (!$dst_is_iscsi && $is_zero_initialized) {
7634         push @$cmd, "zeroinit:$dst_path";
7635     } else {
7636         push @$cmd, $dst_path;
7637     }
7638
7639     my $parser = sub {
7640         my $line = shift;
7641         if($line =~ m/\((\S+)\/100\%\)/){
7642             my $percent = $1;
7643             my $transferred = int($size * $percent / 100);
7644             my $total_h = render_bytes($size, 1);
7645             my $transferred_h = render_bytes($transferred, 1);
7646
7647             print "transferred $transferred_h of $total_h ($percent%)\n";
7648         }
7649
7650     };
7651
7652     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7653     my $err = $@;
7654     die "copy failed: $err" if $err;
7655 }
7656
7657 sub qemu_img_format {
7658     my ($scfg, $volname) = @_;
7659
7660     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7661         return $1;
7662     } else {
7663         return "raw";
7664     }
7665 }
7666
7667 sub qemu_drive_mirror {
7668     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7669
7670     $jobs = {} if !$jobs;
7671
7672     my $qemu_target;
7673     my $format;
7674     $jobs->{"drive-$drive"} = {};
7675
7676     if ($dst_volid =~ /^nbd:/) {
7677         $qemu_target = $dst_volid;
7678         $format = "nbd";
7679     } else {
7680         my $storecfg = PVE::Storage::config();
7681         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7682
7683         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7684
7685         $format = qemu_img_format($dst_scfg, $dst_volname);
7686
7687         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7688
7689         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7690     }
7691
7692     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7693     $opts->{format} = $format if $format;
7694
7695     if (defined($src_bitmap)) {
7696         $opts->{sync} = 'incremental';
7697         $opts->{bitmap} = $src_bitmap;
7698         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7699     }
7700
7701     if (defined($bwlimit)) {
7702         $opts->{speed} = $bwlimit * 1024;
7703         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7704     } else {
7705         print "drive mirror is starting for drive-$drive\n";
7706     }
7707
7708     # if a job already runs for this device we get an error, catch it for cleanup
7709     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7710     if (my $err = $@) {
7711         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7712         warn "$@\n" if $@;
7713         die "mirroring error: $err\n";
7714     }
7715
7716     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7717 }
7718
7719 # $completion can be either
7720 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7721 # 'cancel': wait until all jobs are ready, block-job-cancel them
7722 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7723 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7724 sub qemu_drive_mirror_monitor {
7725     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7726
7727     $completion //= 'complete';
7728     $op //= "mirror";
7729
7730     eval {
7731         my $err_complete = 0;
7732
7733         my $starttime = time ();
7734         while (1) {
7735             die "block job ('$op') timed out\n" if $err_complete > 300;
7736
7737             my $stats = mon_cmd($vmid, "query-block-jobs");
7738             my $ctime = time();
7739
7740             my $running_jobs = {};
7741             for my $stat (@$stats) {
7742                 next if $stat->{type} ne $op;
7743                 $running_jobs->{$stat->{device}} = $stat;
7744             }
7745
7746             my $readycounter = 0;
7747
7748             for my $job_id (sort keys %$jobs) {
7749                 my $job = $running_jobs->{$job_id};
7750
7751                 my $vanished = !defined($job);
7752                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7753                 if($complete || ($vanished && $completion eq 'auto')) {
7754                     print "$job_id: $op-job finished\n";
7755                     delete $jobs->{$job_id};
7756                     next;
7757                 }
7758
7759                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7760
7761                 my $busy = $job->{busy};
7762                 my $ready = $job->{ready};
7763                 if (my $total = $job->{len}) {
7764                     my $transferred = $job->{offset} || 0;
7765                     my $remaining = $total - $transferred;
7766                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7767
7768                     my $duration = $ctime - $starttime;
7769                     my $total_h = render_bytes($total, 1);
7770                     my $transferred_h = render_bytes($transferred, 1);
7771
7772                     my $status = sprintf(
7773                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7774                         render_duration($duration),
7775                     );
7776
7777                     if ($ready) {
7778                         if ($busy) {
7779                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7780                         } else {
7781                             $status .= ", ready";
7782                         }
7783                     }
7784                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7785                     $jobs->{$job_id}->{ready} = $ready;
7786                 }
7787
7788                 $readycounter++ if $job->{ready};
7789             }
7790
7791             last if scalar(keys %$jobs) == 0;
7792
7793             if ($readycounter == scalar(keys %$jobs)) {
7794                 print "all '$op' jobs are ready\n";
7795
7796                 # do the complete later (or has already been done)
7797                 last if $completion eq 'skip' || $completion eq 'auto';
7798
7799                 if ($vmiddst && $vmiddst != $vmid) {
7800                     my $agent_running = $qga && qga_check_running($vmid);
7801                     if ($agent_running) {
7802                         print "freeze filesystem\n";
7803                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7804                         warn $@ if $@;
7805                     } else {
7806                         print "suspend vm\n";
7807                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7808                         warn $@ if $@;
7809                     }
7810
7811                     # if we clone a disk for a new target vm, we don't switch the disk
7812                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7813
7814                     if ($agent_running) {
7815                         print "unfreeze filesystem\n";
7816                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7817                         warn $@ if $@;
7818                     } else {
7819                         print "resume vm\n";
7820                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7821                         warn $@ if $@;
7822                     }
7823
7824                     last;
7825                 } else {
7826
7827                     for my $job_id (sort keys %$jobs) {
7828                         # try to switch the disk if source and destination are on the same guest
7829                         print "$job_id: Completing block job_id...\n";
7830
7831                         my $op;
7832                         if ($completion eq 'complete') {
7833                             $op = 'block-job-complete';
7834                         } elsif ($completion eq 'cancel') {
7835                             $op = 'block-job-cancel';
7836                         } else {
7837                             die "invalid completion value: $completion\n";
7838                         }
7839                         eval { mon_cmd($vmid, $op, device => $job_id) };
7840                         if ($@ =~ m/cannot be completed/) {
7841                             print "$job_id: block job cannot be completed, trying again.\n";
7842                             $err_complete++;
7843                         }else {
7844                             print "$job_id: Completed successfully.\n";
7845                             $jobs->{$job_id}->{complete} = 1;
7846                         }
7847                     }
7848                 }
7849             }
7850             sleep 1;
7851         }
7852     };
7853     my $err = $@;
7854
7855     if ($err) {
7856         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7857         die "block job ($op) error: $err";
7858     }
7859 }
7860
7861 sub qemu_blockjobs_cancel {
7862     my ($vmid, $jobs) = @_;
7863
7864     foreach my $job (keys %$jobs) {
7865         print "$job: Cancelling block job\n";
7866         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7867         $jobs->{$job}->{cancel} = 1;
7868     }
7869
7870     while (1) {
7871         my $stats = mon_cmd($vmid, "query-block-jobs");
7872
7873         my $running_jobs = {};
7874         foreach my $stat (@$stats) {
7875             $running_jobs->{$stat->{device}} = $stat;
7876         }
7877
7878         foreach my $job (keys %$jobs) {
7879
7880             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7881                 print "$job: Done.\n";
7882                 delete $jobs->{$job};
7883             }
7884         }
7885
7886         last if scalar(keys %$jobs) == 0;
7887
7888         sleep 1;
7889     }
7890 }
7891
7892 sub clone_disk {
7893     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7894
7895     my ($vmid, $running) = $source->@{qw(vmid running)};
7896     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7897
7898     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7899     my ($storage, $format) = $dest->@{qw(storage format)};
7900
7901     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7902
7903     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7904         die "cloning from/to EFI disk requires EFI disk\n"
7905             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7906         die "cloning from/to TPM state requires TPM state\n"
7907             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7908
7909         # This would lead to two device nodes in QEMU pointing to the same backing image!
7910         die "cannot change drive name when cloning disk from/to the same VM\n"
7911             if $use_drive_mirror && $vmid == $newvmid;
7912     }
7913
7914     die "cannot move TPM state while VM is running\n"
7915         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7916
7917     my $newvolid;
7918
7919     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7920     print "$src_drivename " if $src_drivename;
7921     print "($drive->{file})\n";
7922
7923     if (!$full) {
7924         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7925         push @$newvollist, $newvolid;
7926     } else {
7927
7928         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7929         $storeid = $storage if $storage;
7930
7931         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7932
7933         my $name = undef;
7934         my $size = undef;
7935         if (drive_is_cloudinit($drive)) {
7936             $name = "vm-$newvmid-cloudinit";
7937             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7938             if ($scfg->{path}) {
7939                 $name .= ".$dst_format";
7940             }
7941             $snapname = undef;
7942             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7943         } elsif ($dst_drivename eq 'efidisk0') {
7944             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7945         } elsif ($dst_drivename eq 'tpmstate0') {
7946             $dst_format = 'raw';
7947             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7948         } else {
7949             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7950         }
7951         $newvolid = PVE::Storage::vdisk_alloc(
7952             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7953         );
7954         push @$newvollist, $newvolid;
7955
7956         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7957
7958         if (drive_is_cloudinit($drive)) {
7959             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7960             # if this is the case, we have to complete any block-jobs still there from
7961             # previous drive-mirrors
7962             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7963                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7964             }
7965             goto no_data_clone;
7966         }
7967
7968         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7969         if ($use_drive_mirror) {
7970             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7971                 $completion, $qga, $bwlimit);
7972         } else {
7973             # TODO: handle bwlimits
7974             if ($dst_drivename eq 'efidisk0') {
7975                 # the relevant data on the efidisk may be smaller than the source
7976                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7977                 # that is given by the OVMF_VARS.fd
7978                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7979                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7980
7981                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7982
7983                 # better for Ceph if block size is not too small, see bug #3324
7984                 my $bs = 1024*1024;
7985
7986                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7987
7988                 if ($src_format eq 'qcow2' && $snapname) {
7989                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7990                         if !min_version(kvm_user_version(), 6, 2);
7991                     push $cmd->@*, '-l', $snapname;
7992                 }
7993                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7994                 run_command($cmd);
7995             } else {
7996                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7997             }
7998         }
7999     }
8000
8001 no_data_clone:
8002     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8003
8004     my $disk = dclone($drive);
8005     delete $disk->{format};
8006     $disk->{file} = $newvolid;
8007     $disk->{size} = $size if defined($size);
8008
8009     return $disk;
8010 }
8011
8012 sub get_running_qemu_version {
8013     my ($vmid) = @_;
8014     my $res = mon_cmd($vmid, "query-version");
8015     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8016 }
8017
8018 sub qemu_use_old_bios_files {
8019     my ($machine_type) = @_;
8020
8021     return if !$machine_type;
8022
8023     my $use_old_bios_files = undef;
8024
8025     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8026         $machine_type = $1;
8027         $use_old_bios_files = 1;
8028     } else {
8029         my $version = extract_version($machine_type, kvm_user_version());
8030         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8031         # load new efi bios files on migration. So this hack is required to allow
8032         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8033         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8034         $use_old_bios_files = !min_version($version, 2, 4);
8035     }
8036
8037     return ($use_old_bios_files, $machine_type);
8038 }
8039
8040 sub get_efivars_size {
8041     my ($conf, $efidisk) = @_;
8042
8043     my $arch = get_vm_arch($conf);
8044     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8045     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8046     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8047     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8048     return -s $ovmf_vars;
8049 }
8050
8051 sub update_efidisk_size {
8052     my ($conf) = @_;
8053
8054     return if !defined($conf->{efidisk0});
8055
8056     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8057     $disk->{size} = get_efivars_size($conf);
8058     $conf->{efidisk0} = print_drive($disk);
8059
8060     return;
8061 }
8062
8063 sub update_tpmstate_size {
8064     my ($conf) = @_;
8065
8066     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8067     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8068     $conf->{tpmstate0} = print_drive($disk);
8069 }
8070
8071 sub create_efidisk($$$$$$$) {
8072     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8073
8074     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8075     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8076
8077     my $vars_size_b = -s $ovmf_vars;
8078     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8079     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8080     PVE::Storage::activate_volumes($storecfg, [$volid]);
8081
8082     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8083     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8084
8085     return ($volid, $size/1024);
8086 }
8087
8088 sub vm_iothreads_list {
8089     my ($vmid) = @_;
8090
8091     my $res = mon_cmd($vmid, 'query-iothreads');
8092
8093     my $iothreads = {};
8094     foreach my $iothread (@$res) {
8095         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8096     }
8097
8098     return $iothreads;
8099 }
8100
8101 sub scsihw_infos {
8102     my ($conf, $drive) = @_;
8103
8104     my $maxdev = 0;
8105
8106     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8107         $maxdev = 7;
8108     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8109         $maxdev = 1;
8110     } else {
8111         $maxdev = 256;
8112     }
8113
8114     my $controller = int($drive->{index} / $maxdev);
8115     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8116         ? "virtioscsi"
8117         : "scsihw";
8118
8119     return ($maxdev, $controller, $controller_prefix);
8120 }
8121
8122 sub resolve_dst_disk_format {
8123         my ($storecfg, $storeid, $src_volname, $format) = @_;
8124         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8125
8126         if (!$format) {
8127             # if no target format is specified, use the source disk format as hint
8128             if ($src_volname) {
8129                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8130                 $format = qemu_img_format($scfg, $src_volname);
8131             } else {
8132                 return $defFormat;
8133             }
8134         }
8135
8136         # test if requested format is supported - else use default
8137         my $supported = grep { $_ eq $format } @$validFormats;
8138         $format = $defFormat if !$supported;
8139         return $format;
8140 }
8141
8142 # NOTE: if this logic changes, please update docs & possibly gui logic
8143 sub find_vmstate_storage {
8144     my ($conf, $storecfg) = @_;
8145
8146     # first, return storage from conf if set
8147     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8148
8149     my ($target, $shared, $local);
8150
8151     foreach_storage_used_by_vm($conf, sub {
8152         my ($sid) = @_;
8153         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8154         my $dst = $scfg->{shared} ? \$shared : \$local;
8155         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8156     });
8157
8158     # second, use shared storage where VM has at least one disk
8159     # third, use local storage where VM has at least one disk
8160     # fall back to local storage
8161     $target = $shared // $local // 'local';
8162
8163     return $target;
8164 }
8165
8166 sub generate_uuid {
8167     my ($uuid, $uuid_str);
8168     UUID::generate($uuid);
8169     UUID::unparse($uuid, $uuid_str);
8170     return $uuid_str;
8171 }
8172
8173 sub generate_smbios1_uuid {
8174     return "uuid=".generate_uuid();
8175 }
8176
8177 sub nbd_stop {
8178     my ($vmid) = @_;
8179
8180     mon_cmd($vmid, 'nbd-server-stop');
8181 }
8182
8183 sub create_reboot_request {
8184     my ($vmid) = @_;
8185     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8186         or die "failed to create reboot trigger file: $!\n";
8187     close($fh);
8188 }
8189
8190 sub clear_reboot_request {
8191     my ($vmid) = @_;
8192     my $path = "/run/qemu-server/$vmid.reboot";
8193     my $res = 0;
8194
8195     $res = unlink($path);
8196     die "could not remove reboot request for $vmid: $!"
8197         if !$res && $! != POSIX::ENOENT;
8198
8199     return $res;
8200 }
8201
8202 sub bootorder_from_legacy {
8203     my ($conf, $bootcfg) = @_;
8204
8205     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8206     my $bootindex_hash = {};
8207     my $i = 1;
8208     foreach my $o (split(//, $boot)) {
8209         $bootindex_hash->{$o} = $i*100;
8210         $i++;
8211     }
8212
8213     my $bootorder = {};
8214
8215     PVE::QemuConfig->foreach_volume($conf, sub {
8216         my ($ds, $drive) = @_;
8217
8218         if (drive_is_cdrom ($drive, 1)) {
8219             if ($bootindex_hash->{d}) {
8220                 $bootorder->{$ds} = $bootindex_hash->{d};
8221                 $bootindex_hash->{d} += 1;
8222             }
8223         } elsif ($bootindex_hash->{c}) {
8224             $bootorder->{$ds} = $bootindex_hash->{c}
8225                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8226             $bootindex_hash->{c} += 1;
8227         }
8228     });
8229
8230     if ($bootindex_hash->{n}) {
8231         for (my $i = 0; $i < $MAX_NETS; $i++) {
8232             my $netname = "net$i";
8233             next if !$conf->{$netname};
8234             $bootorder->{$netname} = $bootindex_hash->{n};
8235             $bootindex_hash->{n} += 1;
8236         }
8237     }
8238
8239     return $bootorder;
8240 }
8241
8242 # Generate default device list for 'boot: order=' property. Matches legacy
8243 # default boot order, but with explicit device names. This is important, since
8244 # the fallback for when neither 'order' nor the old format is specified relies
8245 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8246 sub get_default_bootdevices {
8247     my ($conf) = @_;
8248
8249     my @ret = ();
8250
8251     # harddisk
8252     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8253     push @ret, $first if $first;
8254
8255     # cdrom
8256     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8257     push @ret, $first if $first;
8258
8259     # network
8260     for (my $i = 0; $i < $MAX_NETS; $i++) {
8261         my $netname = "net$i";
8262         next if !$conf->{$netname};
8263         push @ret, $netname;
8264         last;
8265     }
8266
8267     return \@ret;
8268 }
8269
8270 sub device_bootorder {
8271     my ($conf) = @_;
8272
8273     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8274
8275     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8276
8277     my $bootorder = {};
8278     if (!defined($boot) || $boot->{legacy}) {
8279         $bootorder = bootorder_from_legacy($conf, $boot);
8280     } elsif ($boot->{order}) {
8281         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8282         for my $dev (PVE::Tools::split_list($boot->{order})) {
8283             $bootorder->{$dev} = $i++;
8284         }
8285     }
8286
8287     return $bootorder;
8288 }
8289
8290 sub register_qmeventd_handle {
8291     my ($vmid) = @_;
8292
8293     my $fh;
8294     my $peer = "/var/run/qmeventd.sock";
8295     my $count = 0;
8296
8297     for (;;) {
8298         $count++;
8299         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8300         last if $fh;
8301         if ($! != EINTR && $! != EAGAIN) {
8302             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8303         }
8304         if ($count > 4) {
8305             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8306               . "after $count retries\n";
8307         }
8308         usleep(25000);
8309     }
8310
8311     # send handshake to mark VM as backing up
8312     print $fh to_json({vzdump => {vmid => "$vmid"}});
8313
8314     # return handle to be closed later when inhibit is no longer required
8315     return $fh;
8316 }
8317
8318 # bash completion helper
8319
8320 sub complete_backup_archives {
8321     my ($cmdname, $pname, $cvalue) = @_;
8322
8323     my $cfg = PVE::Storage::config();
8324
8325     my $storeid;
8326
8327     if ($cvalue =~ m/^([^:]+):/) {
8328         $storeid = $1;
8329     }
8330
8331     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8332
8333     my $res = [];
8334     foreach my $id (keys %$data) {
8335         foreach my $item (@{$data->{$id}}) {
8336             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8337             push @$res, $item->{volid} if defined($item->{volid});
8338         }
8339     }
8340
8341     return $res;
8342 }
8343
8344 my $complete_vmid_full = sub {
8345     my ($running) = @_;
8346
8347     my $idlist = vmstatus();
8348
8349     my $res = [];
8350
8351     foreach my $id (keys %$idlist) {
8352         my $d = $idlist->{$id};
8353         if (defined($running)) {
8354             next if $d->{template};
8355             next if $running && $d->{status} ne 'running';
8356             next if !$running && $d->{status} eq 'running';
8357         }
8358         push @$res, $id;
8359
8360     }
8361     return $res;
8362 };
8363
8364 sub complete_vmid {
8365     return &$complete_vmid_full();
8366 }
8367
8368 sub complete_vmid_stopped {
8369     return &$complete_vmid_full(0);
8370 }
8371
8372 sub complete_vmid_running {
8373     return &$complete_vmid_full(1);
8374 }
8375
8376 sub complete_storage {
8377
8378     my $cfg = PVE::Storage::config();
8379     my $ids = $cfg->{ids};
8380
8381     my $res = [];
8382     foreach my $sid (keys %$ids) {
8383         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8384         next if !$ids->{$sid}->{content}->{images};
8385         push @$res, $sid;
8386     }
8387
8388     return $res;
8389 }
8390
8391 sub complete_migration_storage {
8392     my ($cmd, $param, $current_value, $all_args) = @_;
8393
8394     my $targetnode = @$all_args[1];
8395
8396     my $cfg = PVE::Storage::config();
8397     my $ids = $cfg->{ids};
8398
8399     my $res = [];
8400     foreach my $sid (keys %$ids) {
8401         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8402         next if !$ids->{$sid}->{content}->{images};
8403         push @$res, $sid;
8404     }
8405
8406     return $res;
8407 }
8408
8409 sub vm_is_paused {
8410     my ($vmid) = @_;
8411     my $qmpstatus = eval {
8412         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8413         mon_cmd($vmid, "query-status");
8414     };
8415     warn "$@\n" if $@;
8416     return $qmpstatus && $qmpstatus->{status} eq "paused";
8417 }
8418
8419 sub check_volume_storage_type {
8420     my ($storecfg, $vol) = @_;
8421
8422     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8423     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8424     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8425
8426     die "storage '$storeid' does not support content-type '$vtype'\n"
8427         if !$scfg->{content}->{$vtype};
8428
8429     return 1;
8430 }
8431
8432 sub add_nets_bridge_fdb {
8433     my ($conf, $vmid) = @_;
8434
8435     for my $opt (keys %$conf) {
8436         next if $opt !~ m/^net(\d+)$/;
8437         my $iface = "tap${vmid}i$1";
8438         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8439         my $net = parse_net($conf->{$opt}, 1) or next;
8440
8441         my $mac = $net->{macaddr};
8442         if (!$mac) {
8443             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8444                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8445             next;
8446         }
8447
8448         if ($have_sdn) {
8449             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8450         } else {
8451             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8452         }
8453     }
8454 }
8455
8456 sub del_nets_bridge_fdb {
8457     my ($conf, $vmid) = @_;
8458
8459     for my $opt (keys %$conf) {
8460         next if $opt !~ m/^net(\d+)$/;
8461         my $iface = "tap${vmid}i$1";
8462
8463         my $net = parse_net($conf->{$opt}) or next;
8464         my $mac = $net->{macaddr} or next;
8465
8466         if ($have_sdn) {
8467             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8468         } else {
8469             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8470         }
8471     }
8472 }
8473
8474 1;